Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
954be9e868 | ||
|
|
afcccf811e | ||
|
|
0aa7244cf4 | ||
|
|
1d8919532b | ||
|
|
ea9bdf9598 | ||
|
|
c52016d727 | ||
|
|
588188edb5 |
@@ -1 +1 @@
|
|||||||
dev
|
1.9.0
|
||||||
|
|||||||
@@ -0,0 +1,503 @@
|
|||||||
|
/**
|
||||||
|
* Auto-Restart Manager - Per-container restart policies with retry tracking
|
||||||
|
*
|
||||||
|
* When a container goes down, attempts automatic restart up to N times
|
||||||
|
* (configurable per-service). Sends notifications on each attempt and
|
||||||
|
* when max retries are exceeded. Integrates with HealthChecker events.
|
||||||
|
*
|
||||||
|
* @module auto-restart-manager
|
||||||
|
*/
|
||||||
|
|
||||||
|
const EventEmitter = require('events');
|
||||||
|
const path = require('path');
|
||||||
|
const { readJsonFile, writeJsonFile } = require('./fs-helpers');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Default policy values applied when a new policy is created.
|
||||||
|
* @readonly
|
||||||
|
*/
|
||||||
|
const DEFAULT_POLICY = {
|
||||||
|
enabled: true,
|
||||||
|
maxRetries: 3,
|
||||||
|
retryIntervalMs: 5000,
|
||||||
|
windowMinutes: 10,
|
||||||
|
currentRetries: 0,
|
||||||
|
lastRestartAt: null,
|
||||||
|
cooldownUntil: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manages automatic container restart policies and execution.
|
||||||
|
*
|
||||||
|
* @extends EventEmitter
|
||||||
|
*
|
||||||
|
* @fires AutoRestartManager#auto-restart-attempt
|
||||||
|
* @fires AutoRestartManager#auto-restart-success
|
||||||
|
* @fires AutoRestartManager#auto-restart-failed
|
||||||
|
* @fires AutoRestartManager#auto-restart-max-reached
|
||||||
|
*/
|
||||||
|
class AutoRestartManager extends EventEmitter {
|
||||||
|
/**
|
||||||
|
* @param {Object} ctx - Shared application context
|
||||||
|
* @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode })
|
||||||
|
* @param {Object} ctx.healthChecker - HealthChecker singleton
|
||||||
|
* @param {Object} ctx.notification - NotificationManager instance
|
||||||
|
* @param {Object} ctx.log - Logger instance
|
||||||
|
* @param {Function} ctx.logError - Error logging function
|
||||||
|
* @param {string} ctx.SERVICES_FILE - Path to services.json (used to derive data dir)
|
||||||
|
*/
|
||||||
|
constructor(ctx) {
|
||||||
|
super();
|
||||||
|
this.ctx = ctx;
|
||||||
|
this.log = ctx.log || console;
|
||||||
|
this.logError = ctx.logError || ((_ctx, err) => console.error(err));
|
||||||
|
this.docker = ctx.docker;
|
||||||
|
this.healthChecker = ctx.healthChecker;
|
||||||
|
this.notification = ctx.notification;
|
||||||
|
|
||||||
|
/** @type {Map<string, Object>} serviceId -> policy */
|
||||||
|
this.policies = new Map();
|
||||||
|
|
||||||
|
/** Path to the JSON file that persists policies */
|
||||||
|
this.policiesFile = path.join(path.dirname(ctx.SERVICES_FILE), 'auto-restart-policies.json');
|
||||||
|
|
||||||
|
/** Track previous health status per service for transition detection */
|
||||||
|
this._previousHealth = new Map();
|
||||||
|
|
||||||
|
/** Bound handlers so we can remove them on stop() */
|
||||||
|
this._onStatusCheck = this._handleStatusCheck.bind(this);
|
||||||
|
this._started = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Lifecycle ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load persisted policies, then wire into HealthChecker events.
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
*/
|
||||||
|
async start() {
|
||||||
|
if (this._started) return;
|
||||||
|
|
||||||
|
// Load persisted policies from disk
|
||||||
|
try {
|
||||||
|
const data = await readJsonFile(this.policiesFile, {});
|
||||||
|
for (const [serviceId, policy] of Object.entries(data)) {
|
||||||
|
this.policies.set(serviceId, { ...DEFAULT_POLICY, ...policy });
|
||||||
|
}
|
||||||
|
this.log.info('auto-restart', 'Policies loaded', { count: this.policies.size });
|
||||||
|
} catch (err) {
|
||||||
|
this.log.error('auto-restart', 'Failed to load policies', { error: err.message });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Listen to health checker status transitions
|
||||||
|
if (this.healthChecker) {
|
||||||
|
this.healthChecker.on('status-check', this._onStatusCheck);
|
||||||
|
}
|
||||||
|
|
||||||
|
this._started = true;
|
||||||
|
this.log.info('auto-restart', 'Manager started');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove event listeners and stop processing health events.
|
||||||
|
*/
|
||||||
|
stop() {
|
||||||
|
if (!this._started) return;
|
||||||
|
|
||||||
|
if (this.healthChecker) {
|
||||||
|
this.healthChecker.removeListener('status-check', this._onStatusCheck);
|
||||||
|
}
|
||||||
|
|
||||||
|
this._started = false;
|
||||||
|
this.log.info('auto-restart', 'Manager stopped');
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Policy CRUD ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create or update a restart policy for a service.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId - Unique service identifier
|
||||||
|
* @param {Object} policy - Partial policy fields to merge
|
||||||
|
* @param {boolean} [policy.enabled=true]
|
||||||
|
* @param {number} [policy.maxRetries=3]
|
||||||
|
* @param {number} [policy.retryIntervalMs=5000]
|
||||||
|
* @param {number} [policy.windowMinutes=10]
|
||||||
|
* @returns {Promise<Object>} The resulting policy
|
||||||
|
* @throws {Error} If serviceId is invalid
|
||||||
|
*/
|
||||||
|
async setPolicy(serviceId, policy) {
|
||||||
|
if (!serviceId || typeof serviceId !== 'string') {
|
||||||
|
throw new Error('serviceId is required');
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = this.policies.get(serviceId) || { ...DEFAULT_POLICY, serviceId };
|
||||||
|
|
||||||
|
const merged = {
|
||||||
|
...existing,
|
||||||
|
...policy,
|
||||||
|
serviceId,
|
||||||
|
// Never allow caller to override runtime counters directly
|
||||||
|
currentRetries: existing.currentRetries || 0,
|
||||||
|
lastRestartAt: existing.lastRestartAt,
|
||||||
|
cooldownUntil: existing.cooldownUntil,
|
||||||
|
};
|
||||||
|
|
||||||
|
this.policies.set(serviceId, merged);
|
||||||
|
await this._savePolicies();
|
||||||
|
|
||||||
|
this.log.info('auto-restart', 'Policy set', { serviceId, enabled: merged.enabled });
|
||||||
|
return { ...merged };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Retrieve the policy for a service.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId
|
||||||
|
* @returns {Object|null} Policy object or null if none exists
|
||||||
|
*/
|
||||||
|
getPolicy(serviceId) {
|
||||||
|
const policy = this.policies.get(serviceId);
|
||||||
|
return policy ? { ...policy } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return all policies as an array.
|
||||||
|
* @returns {Object[]}
|
||||||
|
*/
|
||||||
|
listPolicies() {
|
||||||
|
return Array.from(this.policies.values()).map(p => ({ ...p }));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove a service's restart policy.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId
|
||||||
|
* @returns {Promise<boolean>} true if a policy was removed
|
||||||
|
*/
|
||||||
|
async removePolicy(serviceId) {
|
||||||
|
if (!this.policies.has(serviceId)) return false;
|
||||||
|
|
||||||
|
this.policies.delete(serviceId);
|
||||||
|
await this._savePolicies();
|
||||||
|
|
||||||
|
this.log.info('auto-restart', 'Policy removed', { serviceId });
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Core Restart Logic ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called when a container is detected as down.
|
||||||
|
*
|
||||||
|
* Checks policy, cooldown, and retry count, then either attempts a
|
||||||
|
* Docker restart or notifies that max retries were exceeded.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId - Service identifier
|
||||||
|
* @param {string} containerId - Docker container ID to restart
|
||||||
|
* @returns {Promise<Object>} Result of the operation
|
||||||
|
*/
|
||||||
|
async handleContainerDown(serviceId, containerId) {
|
||||||
|
const policy = this.policies.get(serviceId);
|
||||||
|
if (!policy) {
|
||||||
|
return { action: 'ignored', reason: 'no-policy' };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!policy.enabled) {
|
||||||
|
return { action: 'ignored', reason: 'disabled' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check cooldown window
|
||||||
|
const now = Date.now();
|
||||||
|
if (policy.cooldownUntil && now < policy.cooldownUntil) {
|
||||||
|
this.log.info('auto-restart', 'Skipping — cooldown active', {
|
||||||
|
serviceId,
|
||||||
|
cooldownUntil: new Date(policy.cooldownUntil).toISOString(),
|
||||||
|
});
|
||||||
|
return { action: 'skipped', reason: 'cooldown' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Max retries exceeded — notify and enter cooldown
|
||||||
|
if (policy.currentRetries >= policy.maxRetries) {
|
||||||
|
const cooldownMs = policy.windowMinutes * 60 * 1000;
|
||||||
|
policy.cooldownUntil = now + cooldownMs;
|
||||||
|
policy.currentRetries = 0; // Reset so next window can try again
|
||||||
|
await this._savePolicies();
|
||||||
|
|
||||||
|
const eventData = {
|
||||||
|
serviceId,
|
||||||
|
containerId,
|
||||||
|
maxRetries: policy.maxRetries,
|
||||||
|
cooldownUntil: policy.cooldownUntil,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @event AutoRestartManager#auto-restart-max-reached
|
||||||
|
* @type {Object}
|
||||||
|
*/
|
||||||
|
this.emit('auto-restart-max-reached', eventData);
|
||||||
|
|
||||||
|
// Send notification
|
||||||
|
try {
|
||||||
|
await this._notify('auto-restart', {
|
||||||
|
containerName: serviceId,
|
||||||
|
message: `⛔ Max auto-restart retries (${policy.maxRetries}) exceeded for "${serviceId}". Cooldown until ${new Date(policy.cooldownUntil).toISOString()}.`,
|
||||||
|
...eventData,
|
||||||
|
});
|
||||||
|
} catch (notifErr) {
|
||||||
|
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||||
|
}
|
||||||
|
|
||||||
|
return { action: 'max-reached', ...eventData };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for the configured retry interval before attempting
|
||||||
|
if (policy.retryIntervalMs > 0 && policy.lastRestartAt) {
|
||||||
|
const elapsed = now - new Date(policy.lastRestartAt).getTime();
|
||||||
|
if (elapsed < policy.retryIntervalMs) {
|
||||||
|
const waitMs = policy.retryIntervalMs - elapsed;
|
||||||
|
this.log.info('auto-restart', 'Waiting for retry interval', { serviceId, waitMs });
|
||||||
|
await new Promise(resolve => setTimeout(resolve, waitMs));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt restart
|
||||||
|
policy.currentRetries += 1;
|
||||||
|
const attemptNum = policy.currentRetries;
|
||||||
|
const maxRetries = policy.maxRetries;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @event AutoRestartManager#auto-restart-attempt
|
||||||
|
* @type {Object}
|
||||||
|
*/
|
||||||
|
this.emit('auto-restart-attempt', {
|
||||||
|
serviceId,
|
||||||
|
containerId,
|
||||||
|
attempt: attemptNum,
|
||||||
|
maxRetries,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!this.docker?.client) {
|
||||||
|
throw new Error('Docker client not available');
|
||||||
|
}
|
||||||
|
|
||||||
|
const container = this.docker.client.getContainer(containerId);
|
||||||
|
await container.start();
|
||||||
|
|
||||||
|
policy.lastRestartAt = new Date().toISOString();
|
||||||
|
await this._savePolicies();
|
||||||
|
|
||||||
|
const successData = {
|
||||||
|
serviceId,
|
||||||
|
containerId,
|
||||||
|
attempt: attemptNum,
|
||||||
|
maxRetries,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @event AutoRestartManager#auto-restart-success
|
||||||
|
* @type {Object}
|
||||||
|
*/
|
||||||
|
this.emit('auto-restart-success', successData);
|
||||||
|
|
||||||
|
// Notify
|
||||||
|
try {
|
||||||
|
await this._notify('auto-restart', {
|
||||||
|
containerName: serviceId,
|
||||||
|
message: `🔄 Auto-restart attempt ${attemptNum}/${maxRetries} succeeded for "${serviceId}".`,
|
||||||
|
...successData,
|
||||||
|
});
|
||||||
|
} catch (notifErr) {
|
||||||
|
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||||
|
}
|
||||||
|
|
||||||
|
this.log.info('auto-restart', 'Container restarted', {
|
||||||
|
serviceId,
|
||||||
|
attempt: attemptNum,
|
||||||
|
maxRetries,
|
||||||
|
});
|
||||||
|
|
||||||
|
return { action: 'restarted', ...successData };
|
||||||
|
} catch (restartErr) {
|
||||||
|
policy.lastRestartAt = new Date().toISOString();
|
||||||
|
await this._savePolicies();
|
||||||
|
|
||||||
|
const failData = {
|
||||||
|
serviceId,
|
||||||
|
containerId,
|
||||||
|
attempt: attemptNum,
|
||||||
|
maxRetries,
|
||||||
|
error: restartErr.message,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @event AutoRestartManager#auto-restart-failed
|
||||||
|
* @type {Object}
|
||||||
|
*/
|
||||||
|
this.emit('auto-restart-failed', failData);
|
||||||
|
|
||||||
|
// Notify
|
||||||
|
try {
|
||||||
|
await this._notify('auto-restart', {
|
||||||
|
containerName: serviceId,
|
||||||
|
message: `❌ Auto-restart attempt ${attemptNum}/${maxRetries} failed for "${serviceId}": ${restartErr.message}`,
|
||||||
|
...failData,
|
||||||
|
});
|
||||||
|
} catch (notifErr) {
|
||||||
|
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||||
|
}
|
||||||
|
|
||||||
|
this.log.error('auto-restart', 'Restart failed', {
|
||||||
|
serviceId,
|
||||||
|
attempt: attemptNum,
|
||||||
|
error: restartErr.message,
|
||||||
|
});
|
||||||
|
|
||||||
|
return { action: 'failed', ...failData };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called when a container recovers to healthy state.
|
||||||
|
* Resets the retry counter for the associated service.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
*/
|
||||||
|
async handleContainerUp(serviceId) {
|
||||||
|
const policy = this.policies.get(serviceId);
|
||||||
|
if (!policy) return;
|
||||||
|
|
||||||
|
if (policy.currentRetries > 0) {
|
||||||
|
policy.currentRetries = 0;
|
||||||
|
policy.cooldownUntil = null;
|
||||||
|
await this._savePolicies();
|
||||||
|
|
||||||
|
this.log.info('auto-restart', 'Retries reset after recovery', { serviceId });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Health Event Bridge ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Internal handler for HealthChecker `status-check` events.
|
||||||
|
* Detects healthy→unhealthy and unhealthy→healthy transitions for tracked services.
|
||||||
|
*
|
||||||
|
* @param {Object} status - HealthChecker status object
|
||||||
|
* @param {string} status.serviceId
|
||||||
|
* @param {string} status.status - "up" or "down"
|
||||||
|
* @private
|
||||||
|
*/
|
||||||
|
async _handleStatusCheck(status) {
|
||||||
|
const { serviceId, status: currentStatus } = status;
|
||||||
|
if (!serviceId) return;
|
||||||
|
|
||||||
|
// Only process services that have a restart policy
|
||||||
|
if (!this.policies.has(serviceId)) return;
|
||||||
|
|
||||||
|
const previousStatus = this._previousHealth.get(serviceId);
|
||||||
|
this._previousHealth.set(serviceId, currentStatus);
|
||||||
|
|
||||||
|
// Transition: healthy → unhealthy
|
||||||
|
if (previousStatus === 'up' && currentStatus === 'down') {
|
||||||
|
// Find the containerId from the health checker config or status details
|
||||||
|
const containerId = this._resolveContainerId(serviceId, status);
|
||||||
|
if (containerId) {
|
||||||
|
try {
|
||||||
|
await this.handleContainerDown(serviceId, containerId);
|
||||||
|
} catch (err) {
|
||||||
|
this.logError('auto-restart-health-bridge', err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Transition: unhealthy → healthy (recovery)
|
||||||
|
if (previousStatus === 'down' && currentStatus === 'up') {
|
||||||
|
try {
|
||||||
|
await this.handleContainerUp(serviceId);
|
||||||
|
} catch (err) {
|
||||||
|
this.logError('auto-restart-health-bridge', err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attempt to find the containerId for a service from various sources.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId
|
||||||
|
* @param {Object} status - The status-check event data
|
||||||
|
* @returns {string|null}
|
||||||
|
* @private
|
||||||
|
*/
|
||||||
|
_resolveContainerId(serviceId, status) {
|
||||||
|
// Check if it's in the status details (some health checks embed it)
|
||||||
|
if (status.details?.containerId) return status.details.containerId;
|
||||||
|
|
||||||
|
// Look in the health checker config
|
||||||
|
const hcService = this.healthChecker?.config?.services?.[serviceId];
|
||||||
|
if (hcService?.containerId) return hcService.containerId;
|
||||||
|
|
||||||
|
// Try to look it up from the services state manager
|
||||||
|
try {
|
||||||
|
const servicesStateManager = this.ctx.servicesStateManager;
|
||||||
|
if (servicesStateManager) {
|
||||||
|
const readResult = servicesStateManager.read();
|
||||||
|
if (readResult && typeof readResult.then === 'function') {
|
||||||
|
// It returns a promise — fire-and-forget lookup
|
||||||
|
readResult.then(list => {
|
||||||
|
const found = (list || []).find(s => s.id === serviceId);
|
||||||
|
return found?.containerId || null;
|
||||||
|
}).catch(() => null);
|
||||||
|
} else {
|
||||||
|
const found = (readResult || []).find(s => s.id === serviceId);
|
||||||
|
if (found?.containerId) return found.containerId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (_) { /* best effort */ }
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Persistence ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Persist current policies to disk.
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
* @private
|
||||||
|
*/
|
||||||
|
async _savePolicies() {
|
||||||
|
try {
|
||||||
|
const obj = {};
|
||||||
|
for (const [serviceId, policy] of this.policies.entries()) {
|
||||||
|
obj[serviceId] = { ...policy };
|
||||||
|
}
|
||||||
|
await writeJsonFile(this.policiesFile, obj);
|
||||||
|
} catch (err) {
|
||||||
|
this.log.error('auto-restart', 'Failed to save policies', { error: err.message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send a notification via the notification manager.
|
||||||
|
*
|
||||||
|
* @param {string} event - Event type (e.g. 'auto-restart')
|
||||||
|
* @param {Object} data - Notification payload
|
||||||
|
* @returns {Promise<Object>}
|
||||||
|
* @private
|
||||||
|
*/
|
||||||
|
async _notify(event, data) {
|
||||||
|
if (this.notification?.send) {
|
||||||
|
return this.notification.send(event, data);
|
||||||
|
}
|
||||||
|
return { success: false, reason: 'no-notification-manager' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { AutoRestartManager, DEFAULT_POLICY };
|
||||||
@@ -0,0 +1,376 @@
|
|||||||
|
/**
|
||||||
|
* Config Drift Detector - Compares services.json with live Docker state
|
||||||
|
*
|
||||||
|
* Detects discrepancies between the configured service list and what is
|
||||||
|
* actually running in Docker, including missing containers, unknown
|
||||||
|
* containers, port mismatches, state mismatches, and stale records.
|
||||||
|
*
|
||||||
|
* @module config-drift-detector
|
||||||
|
*/
|
||||||
|
|
||||||
|
const EventEmitter = require('events');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} DriftReport
|
||||||
|
* @property {string} checkedAt - ISO timestamp of the check
|
||||||
|
* @property {Object[]} missingContainers - Services with containerId but container absent in Docker
|
||||||
|
* @property {Object[]} unknownContainers - Running Docker containers with sami.managed label but not in services.json
|
||||||
|
* @property {Object[]} portMismatch - Service port != container mapped port
|
||||||
|
* @property {Object[]} stateMismatch - Service expected up but container stopped/absent
|
||||||
|
* @property {Object[]} staleRecords - Services with containerId pointing to removed containers
|
||||||
|
* @property {boolean} hasDrift - Whether any drift category is non-empty
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detects and reports configuration drift between services.json and Docker.
|
||||||
|
*
|
||||||
|
* @extends EventEmitter
|
||||||
|
*
|
||||||
|
* @fires ConfigDriftDetector#drift-detected
|
||||||
|
*/
|
||||||
|
class ConfigDriftDetector extends EventEmitter {
|
||||||
|
/**
|
||||||
|
* @param {Object} ctx - Shared application context
|
||||||
|
* @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode })
|
||||||
|
* @param {Object} ctx.servicesStateManager - StateManager for services.json
|
||||||
|
* @param {Object} ctx.notification - NotificationManager instance
|
||||||
|
* @param {Object} ctx.log - Logger instance
|
||||||
|
* @param {Function} ctx.logError - Error logging function
|
||||||
|
*/
|
||||||
|
constructor(ctx) {
|
||||||
|
super();
|
||||||
|
this.ctx = ctx;
|
||||||
|
this.log = ctx.log || console;
|
||||||
|
this.logError = ctx.logError || ((_c, err) => console.error(err));
|
||||||
|
this.docker = ctx.docker;
|
||||||
|
this.servicesStateManager = ctx.servicesStateManager;
|
||||||
|
this.notification = ctx.notification;
|
||||||
|
|
||||||
|
/** @type {DriftReport|null} Cached report from last detection */
|
||||||
|
this.lastReport = null;
|
||||||
|
|
||||||
|
/** @type {NodeJS.Timeout|null} Polling timer reference */
|
||||||
|
this._pollTimer = null;
|
||||||
|
|
||||||
|
/** Whether polling is currently active */
|
||||||
|
this._polling = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Detection ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a full drift detection and return the report.
|
||||||
|
*
|
||||||
|
* Reads services from servicesStateManager and live containers from Docker,
|
||||||
|
* then compares them across five drift categories.
|
||||||
|
*
|
||||||
|
* @returns {Promise<DriftReport>}
|
||||||
|
*/
|
||||||
|
async detect() {
|
||||||
|
const checkedAt = new Date().toISOString();
|
||||||
|
|
||||||
|
// Gather configured services
|
||||||
|
let services = [];
|
||||||
|
try {
|
||||||
|
const data = await this.servicesStateManager.read();
|
||||||
|
services = Array.isArray(data) ? data : (data.services || []);
|
||||||
|
} catch (err) {
|
||||||
|
this.log.error('drift', 'Failed to read services', { error: err.message });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gather live Docker containers
|
||||||
|
let containers = [];
|
||||||
|
try {
|
||||||
|
containers = await this.docker.client.listContainers({ all: true });
|
||||||
|
} catch (err) {
|
||||||
|
this.log.error('drift', 'Failed to list containers', { error: err.message });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build lookup maps
|
||||||
|
const containerById = new Map(); // containerId (short or long) → container info
|
||||||
|
const containerByName = new Map(); // container name → container info
|
||||||
|
|
||||||
|
for (const c of containers) {
|
||||||
|
// Store by full ID
|
||||||
|
containerById.set(c.Id, c);
|
||||||
|
// Store by short ID (first 12 chars)
|
||||||
|
if (c.Id && c.Id.length >= 12) {
|
||||||
|
containerById.set(c.Id.substring(0, 12), c);
|
||||||
|
}
|
||||||
|
// Store by name (strip leading /)
|
||||||
|
for (const name of (c.Names || [])) {
|
||||||
|
containerByName.set(name.replace(/^\//, ''), c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build set of service containerIds for reverse lookup
|
||||||
|
const serviceContainerIds = new Set();
|
||||||
|
const serviceByContainerId = new Map();
|
||||||
|
|
||||||
|
for (const svc of services) {
|
||||||
|
if (svc.containerId) {
|
||||||
|
serviceContainerIds.add(svc.containerId);
|
||||||
|
// Index by both full and short ID
|
||||||
|
serviceByContainerId.set(svc.containerId, svc);
|
||||||
|
if (svc.containerId.length >= 12) {
|
||||||
|
serviceByContainerId.set(svc.containerId.substring(0, 12), svc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const missingContainers = [];
|
||||||
|
const portMismatch = [];
|
||||||
|
const stateMismatch = [];
|
||||||
|
const staleRecords = [];
|
||||||
|
|
||||||
|
for (const svc of services) {
|
||||||
|
if (!svc.containerId) continue;
|
||||||
|
|
||||||
|
// Look up the container
|
||||||
|
const container = containerById.get(svc.containerId)
|
||||||
|
|| containerById.get(svc.containerId.substring(0, 12));
|
||||||
|
|
||||||
|
if (!container) {
|
||||||
|
// Container ID referenced but not found in Docker at all
|
||||||
|
staleRecords.push({
|
||||||
|
serviceId: svc.id,
|
||||||
|
name: svc.name,
|
||||||
|
containerId: svc.containerId,
|
||||||
|
reason: 'Container not found in Docker',
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Missing container — service expects it but it's not running
|
||||||
|
if (container.State !== 'running') {
|
||||||
|
missingContainers.push({
|
||||||
|
serviceId: svc.id,
|
||||||
|
name: svc.name,
|
||||||
|
containerId: svc.containerId,
|
||||||
|
containerState: container.State,
|
||||||
|
containerStatus: container.Status,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Also a state mismatch if the service is expected to be up
|
||||||
|
stateMismatch.push({
|
||||||
|
serviceId: svc.id,
|
||||||
|
name: svc.name,
|
||||||
|
expectedState: 'running',
|
||||||
|
actualState: container.State,
|
||||||
|
containerId: svc.containerId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Port mismatch detection
|
||||||
|
if (svc.port && container.State === 'running') {
|
||||||
|
const actualPorts = this._extractContainerPorts(container);
|
||||||
|
if (actualPorts.length > 0 && !actualPorts.includes(svc.port)) {
|
||||||
|
portMismatch.push({
|
||||||
|
serviceId: svc.id,
|
||||||
|
name: svc.name,
|
||||||
|
configuredPort: svc.port,
|
||||||
|
actualPorts,
|
||||||
|
containerId: svc.containerId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unknown managed containers: Docker containers with sami.managed label
|
||||||
|
// that are NOT in services.json
|
||||||
|
const unknownContainers = [];
|
||||||
|
for (const c of containers) {
|
||||||
|
const isManaged = c.Labels && c.Labels['sami.managed'] === 'true';
|
||||||
|
if (!isManaged) continue;
|
||||||
|
|
||||||
|
const isInServices = serviceByContainerId.has(c.Id)
|
||||||
|
|| serviceByContainerId.has(c.Id.substring(0, 12));
|
||||||
|
|
||||||
|
if (!isInServices) {
|
||||||
|
unknownContainers.push({
|
||||||
|
containerId: c.Id,
|
||||||
|
name: (c.Names && c.Names[0] || '').replace(/^\//, ''),
|
||||||
|
image: c.Image,
|
||||||
|
state: c.State,
|
||||||
|
status: c.Status,
|
||||||
|
app: c.Labels?.['sami.app'] || null,
|
||||||
|
subdomain: c.Labels?.['sami.subdomain'] || null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const report = {
|
||||||
|
checkedAt,
|
||||||
|
missingContainers,
|
||||||
|
unknownContainers,
|
||||||
|
portMismatch,
|
||||||
|
stateMismatch,
|
||||||
|
staleRecords,
|
||||||
|
hasDrift: missingContainers.length > 0
|
||||||
|
|| unknownContainers.length > 0
|
||||||
|
|| portMismatch.length > 0
|
||||||
|
|| stateMismatch.length > 0
|
||||||
|
|| staleRecords.length > 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Cache for quick API access
|
||||||
|
this.lastReport = report;
|
||||||
|
|
||||||
|
// Emit and notify if drift detected
|
||||||
|
if (report.hasDrift) {
|
||||||
|
/**
|
||||||
|
* @event ConfigDriftDetector#drift-detected
|
||||||
|
* @type {DriftReport}
|
||||||
|
*/
|
||||||
|
this.emit('drift-detected', report);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this._sendDriftNotification(report);
|
||||||
|
} catch (notifErr) {
|
||||||
|
this.log.error('drift', 'Failed to send drift notification', {
|
||||||
|
error: notifErr.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.log.info('drift', 'Detection complete', {
|
||||||
|
hasDrift: report.hasDrift,
|
||||||
|
missing: report.missingContainers.length,
|
||||||
|
unknown: report.unknownContainers.length,
|
||||||
|
portMismatch: report.portMismatch.length,
|
||||||
|
stateMismatch: report.stateMismatch.length,
|
||||||
|
stale: report.staleRecords.length,
|
||||||
|
});
|
||||||
|
|
||||||
|
return report;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Auto-fix ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attempt to auto-fix drift:
|
||||||
|
* - Remove stale records (services referencing removed containers)
|
||||||
|
* - Flag unknown containers for review
|
||||||
|
*
|
||||||
|
* @returns {Promise<{ staleRemoved: number, unknownFlagged: number }>}
|
||||||
|
*/
|
||||||
|
async autoFix() {
|
||||||
|
const report = await this.detect();
|
||||||
|
let staleRemoved = 0;
|
||||||
|
|
||||||
|
// Remove stale records from services.json
|
||||||
|
if (report.staleRecords.length > 0) {
|
||||||
|
const staleIds = new Set(report.staleRecords.map(r => r.serviceId));
|
||||||
|
await this.servicesStateManager.update(services => {
|
||||||
|
const before = services.length;
|
||||||
|
const cleaned = services.filter(s => !staleIds.has(s.id));
|
||||||
|
staleRemoved = before - cleaned.length;
|
||||||
|
return cleaned;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const unknownFlagged = report.unknownContainers.length;
|
||||||
|
|
||||||
|
this.log.info('drift', 'Auto-fix applied', { staleRemoved, unknownFlagged });
|
||||||
|
|
||||||
|
return { staleRemoved, unknownFlagged };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Polling ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start periodic drift detection.
|
||||||
|
*
|
||||||
|
* @param {number} [intervalMs=300000] - Polling interval in milliseconds (default 5 min)
|
||||||
|
*/
|
||||||
|
startPolling(intervalMs = 300000) {
|
||||||
|
this.stopPolling();
|
||||||
|
|
||||||
|
this._polling = true;
|
||||||
|
this._pollTimer = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
await this.detect();
|
||||||
|
} catch (err) {
|
||||||
|
this.logError('drift-poll', err);
|
||||||
|
}
|
||||||
|
}, intervalMs);
|
||||||
|
|
||||||
|
this.log.info('drift', 'Polling started', { intervalMs });
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop periodic drift detection.
|
||||||
|
*/
|
||||||
|
stopPolling() {
|
||||||
|
if (this._pollTimer) {
|
||||||
|
clearInterval(this._pollTimer);
|
||||||
|
this._pollTimer = null;
|
||||||
|
}
|
||||||
|
this._polling = false;
|
||||||
|
this.log.info('drift', 'Polling stopped');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether polling is currently active.
|
||||||
|
* @returns {boolean}
|
||||||
|
*/
|
||||||
|
isPolling() {
|
||||||
|
return this._polling;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract mapped host ports from a Docker container info object.
|
||||||
|
*
|
||||||
|
* @param {Object} container - Dockerode container info
|
||||||
|
* @returns {number[]} Array of host port numbers
|
||||||
|
* @private
|
||||||
|
*/
|
||||||
|
_extractContainerPorts(container) {
|
||||||
|
const ports = [];
|
||||||
|
if (!container.Ports) return ports;
|
||||||
|
|
||||||
|
for (const p of container.Ports) {
|
||||||
|
if (p.PublicPort) {
|
||||||
|
ports.push(p.PublicPort);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ports;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send a notification about detected drift.
|
||||||
|
*
|
||||||
|
* @param {DriftReport} report
|
||||||
|
* @returns {Promise<Object>}
|
||||||
|
* @private
|
||||||
|
*/
|
||||||
|
async _sendDriftNotification(report) {
|
||||||
|
if (!this.notification?.send) {
|
||||||
|
return { success: false, reason: 'no-notification-manager' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const parts = [];
|
||||||
|
if (report.missingContainers.length > 0) {
|
||||||
|
parts.push(`Missing containers: ${report.missingContainers.map(c => c.name).join(', ')}`);
|
||||||
|
}
|
||||||
|
if (report.unknownContainers.length > 0) {
|
||||||
|
parts.push(`Unknown managed containers: ${report.unknownContainers.map(c => c.name).join(', ')}`);
|
||||||
|
}
|
||||||
|
if (report.portMismatch.length > 0) {
|
||||||
|
parts.push(`Port mismatches: ${report.portMismatch.map(c => c.name).join(', ')}`);
|
||||||
|
}
|
||||||
|
if (report.staleRecords.length > 0) {
|
||||||
|
parts.push(`Stale records: ${report.staleRecords.map(c => c.name).join(', ')}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.notification.send('drift-detected', {
|
||||||
|
text: `⚠️ Configuration drift detected:\n${parts.join('\n')}`,
|
||||||
|
report,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { ConfigDriftDetector };
|
||||||
@@ -0,0 +1,605 @@
|
|||||||
|
/**
|
||||||
|
* Dependency Manager - Service dependency tracking with ordered restart chains
|
||||||
|
*
|
||||||
|
* Manages directed acyclic graph (DAG) of service dependencies. Services can
|
||||||
|
* declare which other services they depend on, and this manager provides:
|
||||||
|
* - Full dependency graph inspection
|
||||||
|
* - Topological ordering for safe restart chains
|
||||||
|
* - Circular dependency detection
|
||||||
|
* - Health-aware restart with per-service polling
|
||||||
|
*
|
||||||
|
* Dependencies are stored directly on service objects in services.json:
|
||||||
|
* { id, name, ..., dependsOn: ['service-id-1', 'service-id-2'] }
|
||||||
|
*
|
||||||
|
* @module dependency-manager
|
||||||
|
*/
|
||||||
|
|
||||||
|
const EventEmitter = require('events');
|
||||||
|
|
||||||
|
/** Maximum seconds to wait for a single container to become healthy after restart */
|
||||||
|
const HEALTH_CHECK_TIMEOUT_MS = 30_000;
|
||||||
|
|
||||||
|
/** Interval between container health polls */
|
||||||
|
const HEALTH_CHECK_INTERVAL_MS = 1_000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} ServiceNode
|
||||||
|
* @property {string} serviceId
|
||||||
|
* @property {string} name
|
||||||
|
* @property {string|null} containerId
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} DependencyEdge
|
||||||
|
* @property {string} from - The service that depends
|
||||||
|
* @property {string} to - The service being depended upon
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} DependencyGraph
|
||||||
|
* @property {ServiceNode[]} nodes
|
||||||
|
* @property {DependencyEdge[]} edges
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} DependencyStatusEntry
|
||||||
|
* @property {string} serviceId
|
||||||
|
* @property {string} name
|
||||||
|
* @property {boolean} isUp
|
||||||
|
* @property {string} [error]
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DependencyManager — tracks service dependencies and orchestrates ordered restarts.
|
||||||
|
*
|
||||||
|
* Events emitted:
|
||||||
|
* - `dependency-restart-start` ({ serviceId, chain: string[] })
|
||||||
|
* - `dependency-restart-progress` ({ serviceId, currentServiceId, index, total })
|
||||||
|
* - `dependency-restart-complete` ({ serviceId, chain: string[], results: Array })
|
||||||
|
* - `dependency-restart-failed` ({ serviceId, failedServiceId, error, chain: string[] })
|
||||||
|
*
|
||||||
|
* @extends EventEmitter
|
||||||
|
*/
|
||||||
|
class DependencyManager extends EventEmitter {
|
||||||
|
/**
|
||||||
|
* @param {Object} ctx - Application context
|
||||||
|
* @param {Object} ctx.servicesStateManager - StateManager for services.json
|
||||||
|
* @param {Object} ctx.docker - Docker context ({ client: Dockerode })
|
||||||
|
* @param {Object} ctx.notification - NotificationManager instance
|
||||||
|
* @param {Object} ctx.log - Logger instance
|
||||||
|
*/
|
||||||
|
constructor(ctx) {
|
||||||
|
super();
|
||||||
|
/** @private */
|
||||||
|
this.ctx = ctx;
|
||||||
|
/** @private */
|
||||||
|
this._servicesStateManager = ctx.servicesStateManager;
|
||||||
|
/** @private */
|
||||||
|
this._docker = ctx.docker;
|
||||||
|
/** @private */
|
||||||
|
this._notification = ctx.notification;
|
||||||
|
/** @private */
|
||||||
|
this._log = ctx.log || console;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Core helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load all services from the state manager.
|
||||||
|
* @private
|
||||||
|
* @returns {Promise<Object[]>}
|
||||||
|
*/
|
||||||
|
async _loadServices() {
|
||||||
|
const data = await this._servicesStateManager.read();
|
||||||
|
return Array.isArray(data) ? data : (data.services || []);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find a single service by ID.
|
||||||
|
* @private
|
||||||
|
* @param {string} serviceId
|
||||||
|
* @returns {Promise<Object|null>}
|
||||||
|
*/
|
||||||
|
async _findService(serviceId) {
|
||||||
|
const services = await this._loadServices();
|
||||||
|
return services.find(s => s.id === serviceId) || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Graph queries
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the full dependency graph for visualisation.
|
||||||
|
*
|
||||||
|
* @returns {Promise<DependencyGraph>}
|
||||||
|
*/
|
||||||
|
async getDependencyGraph() {
|
||||||
|
const services = await this._loadServices();
|
||||||
|
|
||||||
|
const nodes = services.map(s => ({
|
||||||
|
serviceId: s.id,
|
||||||
|
name: s.name,
|
||||||
|
containerId: s.containerId || null,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const edges = [];
|
||||||
|
for (const service of services) {
|
||||||
|
const deps = service.dependsOn || [];
|
||||||
|
for (const depId of deps) {
|
||||||
|
edges.push({ from: service.id, to: depId });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { nodes, edges };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the services that depend on the given service (reverse deps).
|
||||||
|
*
|
||||||
|
* @param {string} serviceId
|
||||||
|
* @returns {Promise<Object[]>} Services whose `dependsOn` includes `serviceId`.
|
||||||
|
*/
|
||||||
|
async getDependents(serviceId) {
|
||||||
|
const services = await this._loadServices();
|
||||||
|
return services.filter(s => (s.dependsOn || []).includes(serviceId));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the direct dependencies for a service.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId
|
||||||
|
* @returns {Promise<Object[]>} Services that `serviceId` depends on.
|
||||||
|
*/
|
||||||
|
async getDependencies(serviceId) {
|
||||||
|
const services = await this._loadServices();
|
||||||
|
const service = services.find(s => s.id === serviceId);
|
||||||
|
if (!service) return [];
|
||||||
|
const depIds = service.dependsOn || [];
|
||||||
|
return services.filter(s => depIds.includes(s.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Topological sort
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build an adjacency list for the current dependency graph.
|
||||||
|
* Edge direction: service → its dependencies (i.e. what it depends on).
|
||||||
|
*
|
||||||
|
* @private
|
||||||
|
* @param {Object[]} services
|
||||||
|
* @returns {Map<string, string[]>}
|
||||||
|
*/
|
||||||
|
_buildAdjacencyList(services) {
|
||||||
|
const adj = new Map();
|
||||||
|
for (const service of services) {
|
||||||
|
adj.set(service.id, (service.dependsOn || []).slice());
|
||||||
|
}
|
||||||
|
return adj;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DFS-based topological sort with cycle detection (white/gray/black coloring).
|
||||||
|
*
|
||||||
|
* Returns services in restart order: dependencies first, dependents last.
|
||||||
|
* The target service is included at the end.
|
||||||
|
*
|
||||||
|
* @private
|
||||||
|
* @param {string} serviceId - Target service (will be last in the result).
|
||||||
|
* @param {Object[]} services - All services.
|
||||||
|
* @param {Map<string, string[]>} adj - Adjacency list (service → deps).
|
||||||
|
* @returns {string[]} Ordered service IDs for restart.
|
||||||
|
* @throws {Error} If a circular dependency is detected.
|
||||||
|
*/
|
||||||
|
_topologicalSort(serviceId, services, adj) {
|
||||||
|
// Collect only the reachable sub-graph from serviceId
|
||||||
|
const visited = new Set();
|
||||||
|
const reachable = new Set();
|
||||||
|
|
||||||
|
const collectReachable = (id) => {
|
||||||
|
if (reachable.has(id)) return;
|
||||||
|
reachable.add(id);
|
||||||
|
for (const dep of (adj.get(id) || [])) {
|
||||||
|
collectReachable(dep);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
collectReachable(serviceId);
|
||||||
|
|
||||||
|
// DFS topological sort on the reachable sub-graph
|
||||||
|
const WHITE = 0, GRAY = 1, BLACK = 2;
|
||||||
|
const color = new Map();
|
||||||
|
for (const id of reachable) color.set(id, WHITE);
|
||||||
|
|
||||||
|
const result = [];
|
||||||
|
|
||||||
|
const dfs = (id) => {
|
||||||
|
if (color.get(id) === BLACK) return;
|
||||||
|
if (color.get(id) === GRAY) {
|
||||||
|
throw new Error(`Circular dependency detected involving service "${id}"`);
|
||||||
|
}
|
||||||
|
color.set(id, GRAY);
|
||||||
|
for (const dep of (adj.get(id) || [])) {
|
||||||
|
dfs(dep);
|
||||||
|
}
|
||||||
|
color.set(id, BLACK);
|
||||||
|
result.push(id);
|
||||||
|
};
|
||||||
|
|
||||||
|
// Visit the target last so it ends up at the end of the result
|
||||||
|
// Actually, we want deps *first* then the target.
|
||||||
|
// The DFS naturally puts deps before dependents, so starting from
|
||||||
|
// serviceId will place it last (which is correct for restart order).
|
||||||
|
dfs(serviceId);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the topologically ordered restart chain for a service.
|
||||||
|
*
|
||||||
|
* The returned array lists all services that must be restarted,
|
||||||
|
* starting with leaf dependencies and ending with the target service.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId - The service to build the chain for.
|
||||||
|
* @returns {Promise<string[]>} Ordered service IDs.
|
||||||
|
* @throws {Error} If `serviceId` doesn't exist or a circular dependency is found.
|
||||||
|
*/
|
||||||
|
async getOrderedRestartChain(serviceId) {
|
||||||
|
const services = await this._loadServices();
|
||||||
|
const service = services.find(s => s.id === serviceId);
|
||||||
|
if (!service) {
|
||||||
|
throw new Error(`Service "${serviceId}" not found`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const adj = this._buildAdjacencyList(services);
|
||||||
|
return this._topologicalSort(serviceId, services, adj);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Validation
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate a proposed set of dependencies for a service.
|
||||||
|
*
|
||||||
|
* Checks:
|
||||||
|
* - All referenced service IDs exist.
|
||||||
|
* - Adding these dependencies would not create a circular dependency.
|
||||||
|
* - A service cannot depend on itself.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId - The service to set dependencies on.
|
||||||
|
* @param {string[]} dependsOn - Proposed dependency IDs.
|
||||||
|
* @returns {Promise<{ valid: boolean, errors: string[] }>}
|
||||||
|
*/
|
||||||
|
async validateDependencies(serviceId, dependsOn) {
|
||||||
|
const errors = [];
|
||||||
|
|
||||||
|
if (!Array.isArray(dependsOn)) {
|
||||||
|
return { valid: false, errors: ['dependsOn must be an array'] };
|
||||||
|
}
|
||||||
|
|
||||||
|
const services = await this._loadServices();
|
||||||
|
const allIds = new Set(services.map(s => s.id));
|
||||||
|
|
||||||
|
// Service must exist
|
||||||
|
if (!allIds.has(serviceId)) {
|
||||||
|
return { valid: false, errors: [`Service "${serviceId}" not found`] };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Self-dependency
|
||||||
|
if (dependsOn.includes(serviceId)) {
|
||||||
|
errors.push(`Service "${serviceId}" cannot depend on itself`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Existence check
|
||||||
|
for (const depId of dependsOn) {
|
||||||
|
if (!allIds.has(depId)) {
|
||||||
|
errors.push(`Dependency service "${depId}" does not exist`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (errors.length > 0) {
|
||||||
|
return { valid: false, errors };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Circular dependency check: temporarily set the proposed dependsOn
|
||||||
|
// and attempt a topological sort.
|
||||||
|
const tempServices = services.map(s => {
|
||||||
|
if (s.id === serviceId) {
|
||||||
|
return { ...s, dependsOn: dependsOn.slice() };
|
||||||
|
}
|
||||||
|
return { ...s };
|
||||||
|
});
|
||||||
|
|
||||||
|
const adj = this._buildAdjacencyList(tempServices);
|
||||||
|
|
||||||
|
// Check every node for cycles with the new edges
|
||||||
|
try {
|
||||||
|
const WHITE = 0, GRAY = 1, BLACK = 2;
|
||||||
|
const color = new Map();
|
||||||
|
for (const s of tempServices) color.set(s.id, WHITE);
|
||||||
|
|
||||||
|
const dfs = (id) => {
|
||||||
|
if (color.get(id) === BLACK) return;
|
||||||
|
if (color.get(id) === GRAY) {
|
||||||
|
throw new Error(`Circular dependency detected involving service "${id}"`);
|
||||||
|
}
|
||||||
|
color.set(id, GRAY);
|
||||||
|
for (const dep of (adj.get(id) || [])) {
|
||||||
|
dfs(dep);
|
||||||
|
}
|
||||||
|
color.set(id, BLACK);
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const s of tempServices) {
|
||||||
|
if (color.get(s.id) === WHITE) {
|
||||||
|
dfs(s.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
errors.push(err.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { valid: errors.length === 0, errors };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Health status
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the current container status for a service and all its transitive dependencies.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId
|
||||||
|
* @returns {Promise<DependencyStatusEntry[]>}
|
||||||
|
* @throws {Error} If `serviceId` doesn't exist.
|
||||||
|
*/
|
||||||
|
async getDependencyStatus(serviceId) {
|
||||||
|
const services = await this._loadServices();
|
||||||
|
const service = services.find(s => s.id === serviceId);
|
||||||
|
if (!service) {
|
||||||
|
throw new Error(`Service "${serviceId}" not found`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect all transitive dependencies via BFS
|
||||||
|
const serviceMap = new Map(services.map(s => [s.id, s]));
|
||||||
|
const visited = new Set();
|
||||||
|
const queue = [serviceId];
|
||||||
|
const allRelated = [];
|
||||||
|
|
||||||
|
while (queue.length > 0) {
|
||||||
|
const currentId = queue.shift();
|
||||||
|
if (visited.has(currentId)) continue;
|
||||||
|
visited.add(currentId);
|
||||||
|
|
||||||
|
const svc = serviceMap.get(currentId);
|
||||||
|
if (!svc) continue;
|
||||||
|
|
||||||
|
allRelated.push(svc);
|
||||||
|
|
||||||
|
for (const depId of (svc.dependsOn || [])) {
|
||||||
|
if (!visited.has(depId)) {
|
||||||
|
queue.push(depId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query container status for each
|
||||||
|
const results = [];
|
||||||
|
for (const svc of allRelated) {
|
||||||
|
const entry = {
|
||||||
|
serviceId: svc.id,
|
||||||
|
name: svc.name,
|
||||||
|
isUp: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!svc.containerId) {
|
||||||
|
entry.error = 'No container associated with this service';
|
||||||
|
results.push(entry);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const container = this._docker.client.getContainer(svc.containerId);
|
||||||
|
const info = await container.inspect();
|
||||||
|
entry.isUp = info.State?.Running === true;
|
||||||
|
} catch (err) {
|
||||||
|
entry.error = err.message || 'Unable to inspect container';
|
||||||
|
}
|
||||||
|
|
||||||
|
results.push(entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Restart with dependencies
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for a container to report as running after a restart.
|
||||||
|
*
|
||||||
|
* @private
|
||||||
|
* @param {string} containerId
|
||||||
|
* @param {number} [timeoutMs=30000]
|
||||||
|
* @returns {Promise<boolean>} `true` if healthy, `false` if timed out.
|
||||||
|
*/
|
||||||
|
async _waitForContainerHealthy(containerId, timeoutMs = HEALTH_CHECK_TIMEOUT_MS) {
|
||||||
|
const start = Date.now();
|
||||||
|
while (Date.now() - start < timeoutMs) {
|
||||||
|
try {
|
||||||
|
const container = this._docker.client.getContainer(containerId);
|
||||||
|
const info = await container.inspect();
|
||||||
|
if (info.State?.Running === true) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Container might not be inspectable during restart — keep polling
|
||||||
|
}
|
||||||
|
await new Promise(r => setTimeout(r, HEALTH_CHECK_INTERVAL_MS));
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Restart a service and all its dependencies in topological order.
|
||||||
|
*
|
||||||
|
* Emits progress events and sends a notification on completion/failure.
|
||||||
|
* This method is designed to be called from the route handler and
|
||||||
|
* **does not throw** — errors are reported via events and notifications.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId - Target service to restart (with deps).
|
||||||
|
* @returns {Promise<{ success: boolean, chain: string[], results: Array }>}
|
||||||
|
*/
|
||||||
|
async restartWithDependencies(serviceId) {
|
||||||
|
const service = await this._findService(serviceId);
|
||||||
|
if (!service) {
|
||||||
|
const err = new Error(`Service "${serviceId}" not found`);
|
||||||
|
this.emit('dependency-restart-failed', {
|
||||||
|
serviceId,
|
||||||
|
failedServiceId: serviceId,
|
||||||
|
error: err.message,
|
||||||
|
chain: [],
|
||||||
|
});
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
let chain;
|
||||||
|
try {
|
||||||
|
chain = await this.getOrderedRestartChain(serviceId);
|
||||||
|
} catch (err) {
|
||||||
|
this.emit('dependency-restart-failed', {
|
||||||
|
serviceId,
|
||||||
|
failedServiceId: serviceId,
|
||||||
|
error: err.message,
|
||||||
|
chain: [],
|
||||||
|
});
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
const services = await this._loadServices();
|
||||||
|
const serviceMap = new Map(services.map(s => [s.id, s]));
|
||||||
|
|
||||||
|
this._log.info('dependency', 'Starting dependency restart chain', {
|
||||||
|
serviceId,
|
||||||
|
chain,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.emit('dependency-restart-start', { serviceId, chain });
|
||||||
|
|
||||||
|
const results = [];
|
||||||
|
const total = chain.length;
|
||||||
|
|
||||||
|
for (let i = 0; i < total; i++) {
|
||||||
|
const currentId = chain[i];
|
||||||
|
const svc = serviceMap.get(currentId);
|
||||||
|
|
||||||
|
this.emit('dependency-restart-progress', {
|
||||||
|
serviceId,
|
||||||
|
currentServiceId: currentId,
|
||||||
|
index: i,
|
||||||
|
total,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!svc || !svc.containerId) {
|
||||||
|
const msg = !svc
|
||||||
|
? `Service "${currentId}" not found in state`
|
||||||
|
: `Service "${currentId}" has no container — skipping restart`;
|
||||||
|
this._log.warn('dependency', msg);
|
||||||
|
results.push({ serviceId: currentId, restarted: false, skipped: true, reason: msg });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const container = this._docker.client.getContainer(svc.containerId);
|
||||||
|
this._log.info('dependency', `Restarting container for service "${currentId}"`, {
|
||||||
|
containerId: svc.containerId,
|
||||||
|
});
|
||||||
|
await container.restart();
|
||||||
|
|
||||||
|
// Wait for it to come back up
|
||||||
|
const healthy = await this._waitForContainerHealthy(svc.containerId);
|
||||||
|
if (!healthy) {
|
||||||
|
const msg = `Container for service "${currentId}" did not become healthy within ${HEALTH_CHECK_TIMEOUT_MS / 1000}s`;
|
||||||
|
this._log.warn('dependency', msg);
|
||||||
|
results.push({ serviceId: currentId, restarted: true, healthy: false, error: msg });
|
||||||
|
|
||||||
|
// Abort chain — dependency didn't come back
|
||||||
|
this.emit('dependency-restart-failed', {
|
||||||
|
serviceId,
|
||||||
|
failedServiceId: currentId,
|
||||||
|
error: msg,
|
||||||
|
chain,
|
||||||
|
});
|
||||||
|
await this._notifyRestartResult(serviceId, false, chain, results, currentId);
|
||||||
|
return { success: false, chain, results };
|
||||||
|
}
|
||||||
|
|
||||||
|
this._log.info('dependency', `Service "${currentId}" is healthy after restart`);
|
||||||
|
results.push({ serviceId: currentId, restarted: true, healthy: true });
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err.message || 'Unknown error during restart';
|
||||||
|
this._log.error('dependency', `Failed to restart service "${currentId}"`, {
|
||||||
|
error: msg,
|
||||||
|
});
|
||||||
|
results.push({ serviceId: currentId, restarted: false, error: msg });
|
||||||
|
|
||||||
|
this.emit('dependency-restart-failed', {
|
||||||
|
serviceId,
|
||||||
|
failedServiceId: currentId,
|
||||||
|
error: msg,
|
||||||
|
chain,
|
||||||
|
});
|
||||||
|
await this._notifyRestartResult(serviceId, false, chain, results, currentId);
|
||||||
|
return { success: false, chain, results };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.emit('dependency-restart-complete', { serviceId, chain, results });
|
||||||
|
await this._notifyRestartResult(serviceId, true, chain, results);
|
||||||
|
return { success: true, chain, results };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send a notification about the restart result.
|
||||||
|
*
|
||||||
|
* @private
|
||||||
|
* @param {string} serviceId
|
||||||
|
* @param {boolean} success
|
||||||
|
* @param {string[]} chain
|
||||||
|
* @param {Array} results
|
||||||
|
* @param {string} [failedServiceId]
|
||||||
|
*/
|
||||||
|
async _notifyRestartResult(serviceId, success, chain, results, failedServiceId) {
|
||||||
|
if (!this._notification) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (success) {
|
||||||
|
await this._notification.send('dependency-restart-complete', {
|
||||||
|
text: `✅ Dependency restart chain completed for "${serviceId}". Restarted: ${chain.join(' → ')}`,
|
||||||
|
serviceId,
|
||||||
|
chain,
|
||||||
|
results,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
await this._notification.send('dependency-restart-failed', {
|
||||||
|
text: `❌ Dependency restart chain failed for "${serviceId}" at "${failedServiceId}". Chain: ${chain.join(' → ')}`,
|
||||||
|
serviceId,
|
||||||
|
failedServiceId,
|
||||||
|
chain,
|
||||||
|
results,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
this._log.error('dependency', 'Failed to send restart notification', {
|
||||||
|
error: err.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = DependencyManager;
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
/**
|
||||||
|
* DNS Propagation Checker
|
||||||
|
* Verifies DNS record propagation by querying multiple resolvers.
|
||||||
|
* Runs as background jobs with configurable timeout and interval.
|
||||||
|
*
|
||||||
|
* @module dns-propagation
|
||||||
|
*/
|
||||||
|
|
||||||
|
const dns = require('dns').promises;
|
||||||
|
const EventEmitter = require('events');
|
||||||
|
|
||||||
|
/** Default verification options */
|
||||||
|
const DEFAULT_OPTIONS = {
|
||||||
|
timeout: 300000, // 5 minutes
|
||||||
|
interval: 10000, // 10 seconds
|
||||||
|
resolvers: ['1.1.1.1', '8.8.8.8', '9.9.9.9']
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Maximum age for stored verification results (1 hour) */
|
||||||
|
const MAX_RESULT_AGE_MS = 3600000;
|
||||||
|
|
||||||
|
class DNSPropagationChecker extends EventEmitter {
|
||||||
|
/**
|
||||||
|
* Create a DNSPropagationChecker instance.
|
||||||
|
* @param {Object} ctx - Shared application context
|
||||||
|
* @param {Object} ctx.notification - NotificationManager instance
|
||||||
|
* @param {Object} ctx.log - Logger instance
|
||||||
|
*/
|
||||||
|
constructor(ctx) {
|
||||||
|
super();
|
||||||
|
this.ctx = ctx;
|
||||||
|
this.log = ctx.log || console;
|
||||||
|
|
||||||
|
/** @type {Map<string, Object>} domain → verification status */
|
||||||
|
this.verifications = new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verify that a DNS record has propagated by querying multiple resolvers.
|
||||||
|
* Retries every `interval` ms until `timeout` is reached.
|
||||||
|
*
|
||||||
|
* @param {string} domain - The domain to check (e.g., 'test.sami')
|
||||||
|
* @param {string} expectedIp - The expected IP address
|
||||||
|
* @param {Object} [options={}] - Verification options
|
||||||
|
* @param {number} [options.timeout=300000] - Maximum time to wait (ms)
|
||||||
|
* @param {number} [options.interval=10000] - Time between retries (ms)
|
||||||
|
* @param {string[]} [options.resolvers] - DNS resolvers to query
|
||||||
|
* @returns {Promise<Object>} Verification result
|
||||||
|
*/
|
||||||
|
async verifyRecord(domain, expectedIp, options = {}) {
|
||||||
|
const startTime = Date.now();
|
||||||
|
const {
|
||||||
|
timeout = DEFAULT_OPTIONS.timeout,
|
||||||
|
interval = DEFAULT_OPTIONS.interval,
|
||||||
|
resolvers = DEFAULT_OPTIONS.resolvers
|
||||||
|
} = options;
|
||||||
|
|
||||||
|
const allResults = [];
|
||||||
|
let propagated = false;
|
||||||
|
|
||||||
|
while (Date.now() - startTime < timeout) {
|
||||||
|
const roundResults = [];
|
||||||
|
|
||||||
|
for (const resolver of resolvers) {
|
||||||
|
const checkStart = Date.now();
|
||||||
|
try {
|
||||||
|
// Use dns.resolve4 with a custom resolver
|
||||||
|
const resolverInstance = new dns.Resolver();
|
||||||
|
resolverInstance.setServers([resolver]);
|
||||||
|
resolverInstance.setTimeout(5000);
|
||||||
|
|
||||||
|
const addresses = await resolverInstance.resolve4(domain);
|
||||||
|
const matched = addresses.includes(expectedIp);
|
||||||
|
|
||||||
|
const result = {
|
||||||
|
resolver,
|
||||||
|
ips: addresses,
|
||||||
|
matched,
|
||||||
|
checkedAt: new Date().toISOString(),
|
||||||
|
responseTime: Date.now() - checkStart
|
||||||
|
};
|
||||||
|
|
||||||
|
roundResults.push(result);
|
||||||
|
|
||||||
|
if (matched) {
|
||||||
|
propagated = true;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
roundResults.push({
|
||||||
|
resolver,
|
||||||
|
ips: [],
|
||||||
|
matched: false,
|
||||||
|
checkedAt: new Date().toISOString(),
|
||||||
|
error: err.code || err.message,
|
||||||
|
responseTime: Date.now() - checkStart
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
allResults.push(...roundResults);
|
||||||
|
|
||||||
|
// Emit progress event
|
||||||
|
this.emit('propagation-check', {
|
||||||
|
domain,
|
||||||
|
expectedIp,
|
||||||
|
roundResults,
|
||||||
|
elapsed: Date.now() - startTime,
|
||||||
|
propagated
|
||||||
|
});
|
||||||
|
|
||||||
|
if (propagated) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait before next attempt
|
||||||
|
await new Promise(resolve => setTimeout(resolve, interval));
|
||||||
|
}
|
||||||
|
|
||||||
|
const totalTime = Date.now() - startTime;
|
||||||
|
|
||||||
|
return {
|
||||||
|
domain,
|
||||||
|
expectedIp,
|
||||||
|
propagated,
|
||||||
|
results: allResults,
|
||||||
|
totalTime,
|
||||||
|
checkedAt: new Date().toISOString()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start a background DNS propagation verification.
|
||||||
|
* Does not block — returns immediately with the job reference.
|
||||||
|
*
|
||||||
|
* @param {string} domain - The domain to verify
|
||||||
|
* @param {string} expectedIp - The expected IP address
|
||||||
|
* @param {Object} [options={}] - Verification options
|
||||||
|
* @returns {Object} Job status object
|
||||||
|
*/
|
||||||
|
startVerification(domain, expectedIp, options = {}) {
|
||||||
|
// If there's already a running verification for this domain, return it
|
||||||
|
const existing = this.verifications.get(domain);
|
||||||
|
if (existing && existing.status === 'running') {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
const job = {
|
||||||
|
domain,
|
||||||
|
expectedIp,
|
||||||
|
status: 'running',
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
progress: [],
|
||||||
|
result: null
|
||||||
|
};
|
||||||
|
|
||||||
|
this.verifications.set(domain, job);
|
||||||
|
|
||||||
|
// Run verification in background (non-blocking)
|
||||||
|
this.verifyRecord(domain, expectedIp, options)
|
||||||
|
.then(result => {
|
||||||
|
job.status = 'completed';
|
||||||
|
job.result = result;
|
||||||
|
job.completedAt = new Date().toISOString();
|
||||||
|
|
||||||
|
if (result.propagated) {
|
||||||
|
this.emit('propagation-complete', result);
|
||||||
|
|
||||||
|
if (this.ctx.notification) {
|
||||||
|
this.ctx.notification.send('dns-propagation', {
|
||||||
|
text: `✅ DNS record for ${domain} propagated successfully to ${expectedIp}`,
|
||||||
|
domain,
|
||||||
|
expectedIp,
|
||||||
|
totalTime: result.totalTime
|
||||||
|
}, 'success').catch(err => {
|
||||||
|
this.log.error('dns-propagation', 'Failed to send propagation notification', {
|
||||||
|
error: err.message
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.emit('propagation-timeout', result);
|
||||||
|
|
||||||
|
if (this.ctx.notification) {
|
||||||
|
this.ctx.notification.send('dns-propagation', {
|
||||||
|
text: `⏱️ DNS propagation timeout for ${domain} — expected ${expectedIp} not found after ${Math.round(result.totalTime / 1000)}s`,
|
||||||
|
domain,
|
||||||
|
expectedIp,
|
||||||
|
totalTime: result.totalTime
|
||||||
|
}, 'warning').catch(err => {
|
||||||
|
this.log.error('dns-propagation', 'Failed to send timeout notification', {
|
||||||
|
error: err.message
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
job.status = 'error';
|
||||||
|
job.error = err.message;
|
||||||
|
job.completedAt = new Date().toISOString();
|
||||||
|
|
||||||
|
this.log.error('dns-propagation', `Verification failed for ${domain}`, {
|
||||||
|
error: err.message
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the current verification status for a domain.
|
||||||
|
*
|
||||||
|
* @param {string} domain - The domain to look up
|
||||||
|
* @returns {Object|null} Verification status or null if not found
|
||||||
|
*/
|
||||||
|
getVerificationStatus(domain) {
|
||||||
|
const job = this.verifications.get(domain);
|
||||||
|
if (!job) return null;
|
||||||
|
return {
|
||||||
|
domain: job.domain,
|
||||||
|
expectedIp: job.expectedIp,
|
||||||
|
status: job.status,
|
||||||
|
startedAt: job.startedAt,
|
||||||
|
completedAt: job.completedAt || null,
|
||||||
|
result: job.result || null,
|
||||||
|
error: job.error || null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all recent verifications.
|
||||||
|
*
|
||||||
|
* @returns {Object[]} Array of verification statuses
|
||||||
|
*/
|
||||||
|
getAllVerifications() {
|
||||||
|
const results = [];
|
||||||
|
for (const [domain, job] of this.verifications.entries()) {
|
||||||
|
results.push({
|
||||||
|
domain,
|
||||||
|
expectedIp: job.expectedIp,
|
||||||
|
status: job.status,
|
||||||
|
startedAt: job.startedAt,
|
||||||
|
completedAt: job.completedAt || null,
|
||||||
|
propagated: job.result?.propagated || null,
|
||||||
|
totalTime: job.result?.totalTime || null,
|
||||||
|
error: job.error || null
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove verifications older than 1 hour.
|
||||||
|
*/
|
||||||
|
cleanup() {
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [domain, job] of this.verifications.entries()) {
|
||||||
|
const completedAt = job.completedAt ? new Date(job.completedAt).getTime() : null;
|
||||||
|
const startedAt = new Date(job.startedAt).getTime();
|
||||||
|
|
||||||
|
// Clean up completed/error jobs older than 1 hour
|
||||||
|
// Also clean up stale running jobs that started over 2 hours ago
|
||||||
|
const age = completedAt ? (now - completedAt) : (now - startedAt);
|
||||||
|
const maxAge = job.status === 'running' ? MAX_RESULT_AGE_MS * 2 : MAX_RESULT_AGE_MS;
|
||||||
|
|
||||||
|
if (age > maxAge) {
|
||||||
|
this.verifications.delete(domain);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = DNSPropagationChecker;
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "dashcaddy-api",
|
"name": "dashcaddy-api",
|
||||||
"version": "1.6.0",
|
"version": "1.9.0",
|
||||||
"description": "DashCaddy API server - Dashboard backend for Docker, Caddy & DNS management",
|
"description": "DashCaddy API server - Dashboard backend for Docker, Caddy & DNS management",
|
||||||
"main": "server.js",
|
"main": "server.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -0,0 +1,164 @@
|
|||||||
|
/**
|
||||||
|
* Auto-Restart Policy Routes
|
||||||
|
*
|
||||||
|
* CRUD endpoints for per-container auto-restart policies.
|
||||||
|
* Also provides a dry-run test endpoint.
|
||||||
|
*
|
||||||
|
* @module routes/auto-restart
|
||||||
|
*/
|
||||||
|
|
||||||
|
const express = require('express');
|
||||||
|
const { success } = require('../response-helpers');
|
||||||
|
const { ValidationError, NotFoundError } = require('../errors');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Auto-restart route factory
|
||||||
|
*
|
||||||
|
* @param {Object} deps - Explicit dependencies
|
||||||
|
* @param {Object} deps.autoRestartManager - AutoRestartManager instance
|
||||||
|
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||||
|
* @param {Function} deps.logError - Error logging function
|
||||||
|
* @returns {express.Router}
|
||||||
|
*/
|
||||||
|
module.exports = function ({ autoRestartManager, asyncHandler, logError }) {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /auto-restart/policies
|
||||||
|
* List all configured auto-restart policies.
|
||||||
|
*/
|
||||||
|
router.get('/policies', asyncHandler(async (_req, res) => {
|
||||||
|
const policies = autoRestartManager.listPolicies();
|
||||||
|
success(res, { policies });
|
||||||
|
}, 'auto-restart-list'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /auto-restart/policies/:serviceId
|
||||||
|
* Get the restart policy for a single service.
|
||||||
|
*/
|
||||||
|
router.get('/policies/:serviceId', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
|
||||||
|
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
|
||||||
|
throw new ValidationError('Invalid service ID format');
|
||||||
|
}
|
||||||
|
|
||||||
|
const policy = autoRestartManager.getPolicy(serviceId);
|
||||||
|
if (!policy) {
|
||||||
|
throw new NotFoundError(`Auto-restart policy for "${serviceId}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
success(res, { policy });
|
||||||
|
}, 'auto-restart-get'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /auto-restart/policies/:serviceId
|
||||||
|
* Create or update a restart policy.
|
||||||
|
*
|
||||||
|
* Body: { enabled, maxRetries, retryIntervalMs, windowMinutes }
|
||||||
|
*/
|
||||||
|
router.post('/policies/:serviceId', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
|
||||||
|
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
|
||||||
|
throw new ValidationError('Invalid service ID format');
|
||||||
|
}
|
||||||
|
|
||||||
|
const { enabled, maxRetries, retryIntervalMs, windowMinutes } = req.body;
|
||||||
|
|
||||||
|
// Validate inputs
|
||||||
|
if (enabled !== undefined && typeof enabled !== 'boolean') {
|
||||||
|
throw new ValidationError('enabled must be a boolean');
|
||||||
|
}
|
||||||
|
if (maxRetries !== undefined) {
|
||||||
|
if (!Number.isInteger(maxRetries) || maxRetries < 0 || maxRetries > 100) {
|
||||||
|
throw new ValidationError('maxRetries must be an integer between 0 and 100');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (retryIntervalMs !== undefined) {
|
||||||
|
if (!Number.isInteger(retryIntervalMs) || retryIntervalMs < 0 || retryIntervalMs > 3600000) {
|
||||||
|
throw new ValidationError('retryIntervalMs must be an integer between 0 and 3600000');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (windowMinutes !== undefined) {
|
||||||
|
if (!Number.isInteger(windowMinutes) || windowMinutes < 0 || windowMinutes > 1440) {
|
||||||
|
throw new ValidationError('windowMinutes must be an integer between 0 and 1440');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const policy = await autoRestartManager.setPolicy(serviceId, {
|
||||||
|
...(enabled !== undefined && { enabled }),
|
||||||
|
...(maxRetries !== undefined && { maxRetries }),
|
||||||
|
...(retryIntervalMs !== undefined && { retryIntervalMs }),
|
||||||
|
...(windowMinutes !== undefined && { windowMinutes }),
|
||||||
|
});
|
||||||
|
|
||||||
|
success(res, { policy, message: `Policy ${serviceId} saved` });
|
||||||
|
}, 'auto-restart-set'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DELETE /auto-restart/policies/:serviceId
|
||||||
|
* Remove a restart policy.
|
||||||
|
*/
|
||||||
|
router.delete('/policies/:serviceId', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
|
||||||
|
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
|
||||||
|
throw new ValidationError('Invalid service ID format');
|
||||||
|
}
|
||||||
|
|
||||||
|
const removed = await autoRestartManager.removePolicy(serviceId);
|
||||||
|
if (!removed) {
|
||||||
|
throw new NotFoundError(`Auto-restart policy for "${serviceId}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
success(res, { message: `Policy for "${serviceId}" removed` });
|
||||||
|
}, 'auto-restart-delete'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /auto-restart/policies/:serviceId/test
|
||||||
|
* Dry-run: simulate a restart attempt without actually restarting.
|
||||||
|
* Returns what *would* happen given the current policy state.
|
||||||
|
*/
|
||||||
|
router.post('/policies/:serviceId/test', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
|
||||||
|
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
|
||||||
|
throw new ValidationError('Invalid service ID format');
|
||||||
|
}
|
||||||
|
|
||||||
|
const policy = autoRestartManager.getPolicy(serviceId);
|
||||||
|
if (!policy) {
|
||||||
|
throw new NotFoundError(`Auto-restart policy for "${serviceId}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
const inCooldown = policy.cooldownUntil && now < policy.cooldownUntil;
|
||||||
|
const wouldRetry = !inCooldown && policy.currentRetries < policy.maxRetries;
|
||||||
|
const nextAttempt = policy.currentRetries + 1;
|
||||||
|
|
||||||
|
success(res, {
|
||||||
|
dryRun: true,
|
||||||
|
serviceId,
|
||||||
|
policy: {
|
||||||
|
enabled: policy.enabled,
|
||||||
|
currentRetries: policy.currentRetries,
|
||||||
|
maxRetries: policy.maxRetries,
|
||||||
|
cooldownUntil: policy.cooldownUntil,
|
||||||
|
inCooldown,
|
||||||
|
},
|
||||||
|
wouldRestart: policy.enabled && wouldRetry,
|
||||||
|
wouldMaxOut: !wouldRetry && !inCooldown,
|
||||||
|
nextAttempt: wouldRetry ? nextAttempt : null,
|
||||||
|
message: !policy.enabled
|
||||||
|
? 'Policy is disabled — no restart would occur'
|
||||||
|
: inCooldown
|
||||||
|
? `In cooldown until ${new Date(policy.cooldownUntil).toISOString()} — would skip`
|
||||||
|
: wouldRetry
|
||||||
|
? `Would attempt restart ${nextAttempt}/${policy.maxRetries}`
|
||||||
|
: `Max retries (${policy.maxRetries}) already reached — would enter cooldown`,
|
||||||
|
});
|
||||||
|
}, 'auto-restart-test'));
|
||||||
|
|
||||||
|
return router;
|
||||||
|
};
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
/**
|
||||||
|
* Config Drift Detection Routes
|
||||||
|
*
|
||||||
|
* API endpoints for running drift detection, reading cached reports,
|
||||||
|
* auto-fixing drift, and controlling periodic polling.
|
||||||
|
*
|
||||||
|
* @module routes/config-drift
|
||||||
|
*/
|
||||||
|
|
||||||
|
const express = require('express');
|
||||||
|
const { success } = require('../response-helpers');
|
||||||
|
const { ValidationError, NotFoundError } = require('../errors');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Config-drift route factory
|
||||||
|
*
|
||||||
|
* @param {Object} deps - Explicit dependencies
|
||||||
|
* @param {Object} deps.driftDetector - ConfigDriftDetector instance
|
||||||
|
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||||
|
* @param {Function} deps.logError - Error logging function
|
||||||
|
* @returns {express.Router}
|
||||||
|
*/
|
||||||
|
module.exports = function ({ driftDetector, asyncHandler, logError }) {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /config-drift/report
|
||||||
|
* Run a fresh drift detection and return the full report.
|
||||||
|
*/
|
||||||
|
router.get('/report', asyncHandler(async (_req, res) => {
|
||||||
|
const report = await driftDetector.detect();
|
||||||
|
success(res, { report });
|
||||||
|
}, 'drift-report'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /config-drift/last
|
||||||
|
* Return the last cached drift report (no re-detection).
|
||||||
|
*/
|
||||||
|
router.get('/last', asyncHandler(async (_req, res) => {
|
||||||
|
if (!driftDetector.lastReport) {
|
||||||
|
throw new NotFoundError('No cached drift report — run detection first');
|
||||||
|
}
|
||||||
|
|
||||||
|
success(res, { report: driftDetector.lastReport });
|
||||||
|
}, 'drift-last'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /config-drift/fix
|
||||||
|
* Auto-fix detected drift: remove stale records, flag unknown containers.
|
||||||
|
*/
|
||||||
|
router.post('/fix', asyncHandler(async (_req, res) => {
|
||||||
|
const result = await driftDetector.autoFix();
|
||||||
|
success(res, {
|
||||||
|
message: 'Auto-fix applied',
|
||||||
|
staleRemoved: result.staleRemoved,
|
||||||
|
unknownFlagged: result.unknownFlagged,
|
||||||
|
});
|
||||||
|
}, 'drift-fix'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /config-drift/polling
|
||||||
|
* Enable or disable periodic drift detection polling.
|
||||||
|
*
|
||||||
|
* Body: { enabled: boolean, intervalMs?: number }
|
||||||
|
*/
|
||||||
|
router.post('/polling', asyncHandler(async (req, res) => {
|
||||||
|
const { enabled, intervalMs } = req.body;
|
||||||
|
|
||||||
|
if (typeof enabled !== 'boolean') {
|
||||||
|
throw new ValidationError('enabled must be a boolean');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (intervalMs !== undefined) {
|
||||||
|
if (!Number.isInteger(intervalMs) || intervalMs < 10000 || intervalMs > 86400000) {
|
||||||
|
throw new ValidationError('intervalMs must be an integer between 10000 and 86400000 (10s – 24h)');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (enabled) {
|
||||||
|
driftDetector.startPolling(intervalMs || 300000);
|
||||||
|
success(res, {
|
||||||
|
message: 'Drift polling enabled',
|
||||||
|
intervalMs: intervalMs || 300000,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
driftDetector.stopPolling();
|
||||||
|
success(res, { message: 'Drift polling disabled' });
|
||||||
|
}
|
||||||
|
}, 'drift-polling'));
|
||||||
|
|
||||||
|
return router;
|
||||||
|
};
|
||||||
@@ -10,9 +10,10 @@ const { success } = require('../response-helpers');
|
|||||||
* @param {Object} deps.docker - Docker client wrapper (client, pull methods)
|
* @param {Object} deps.docker - Docker client wrapper (client, pull methods)
|
||||||
* @param {Object} deps.log - Logger instance
|
* @param {Object} deps.log - Logger instance
|
||||||
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||||
|
* @param {Object} deps.workflowEngine - WorkflowEngine instance (optional)
|
||||||
* @returns {express.Router}
|
* @returns {express.Router}
|
||||||
*/
|
*/
|
||||||
module.exports = function({ docker, log, asyncHandler }) {
|
module.exports = function({ docker, log, asyncHandler, workflowEngine }) {
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
|
|
||||||
// Helper: verify container exists before operating on it
|
// Helper: verify container exists before operating on it
|
||||||
@@ -66,6 +67,11 @@ module.exports = function({ docker, log, asyncHandler }) {
|
|||||||
log.info('docker', `Pulling latest image: ${imageName}`);
|
log.info('docker', `Pulling latest image: ${imageName}`);
|
||||||
await docker.pull(imageName);
|
await docker.pull(imageName);
|
||||||
|
|
||||||
|
// Trigger pre-update workflow (backup before update)
|
||||||
|
if (workflowEngine) {
|
||||||
|
try { await workflowEngine.triggerEvent('pre-update', { containerId: containerId, containerName, imageName }); } catch (w) { log.warn('workflow', 'pre-update trigger failed: ' + w.message); }
|
||||||
|
}
|
||||||
|
|
||||||
// Get current container config for recreation
|
// Get current container config for recreation
|
||||||
const hostConfig = containerInfo.HostConfig;
|
const hostConfig = containerInfo.HostConfig;
|
||||||
const config = {
|
const config = {
|
||||||
@@ -138,6 +144,11 @@ module.exports = function({ docker, log, asyncHandler }) {
|
|||||||
message: `Container ${containerName} updated successfully`,
|
message: `Container ${containerName} updated successfully`,
|
||||||
newContainerId: newContainerInfo.Id
|
newContainerId: newContainerInfo.Id
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Trigger post-update workflow
|
||||||
|
if (workflowEngine) {
|
||||||
|
try { await workflowEngine.triggerEvent('post-update', { containerId: containerId, containerName, imageName, newContainerId: newContainerInfo.Id }); } catch (w) { log.warn('workflow', 'post-update trigger failed: ' + w.message); }
|
||||||
|
}
|
||||||
}, 'container-update'));
|
}, 'container-update'));
|
||||||
|
|
||||||
// Check for available updates (compares local and remote image digests)
|
// Check for available updates (compares local and remote image digests)
|
||||||
|
|||||||
@@ -0,0 +1,235 @@
|
|||||||
|
/**
|
||||||
|
* Dependencies Route — REST API for service dependency tracking
|
||||||
|
*
|
||||||
|
* Endpoints:
|
||||||
|
* GET /dependencies/graph Full dependency graph
|
||||||
|
* GET /dependencies/validate Validate a proposed dep chain
|
||||||
|
* GET /dependencies/:serviceId Direct deps for one service
|
||||||
|
* GET /dependencies/:serviceId/chain Ordered restart chain
|
||||||
|
* GET /dependencies/:serviceId/status Dependency health status
|
||||||
|
* POST /dependencies/:serviceId Set dependencies
|
||||||
|
* DELETE /dependencies/:serviceId Remove all dependencies
|
||||||
|
* POST /dependencies/:serviceId/restart Restart with dependency chain
|
||||||
|
*
|
||||||
|
* @module routes/dependencies
|
||||||
|
*/
|
||||||
|
|
||||||
|
const express = require('express');
|
||||||
|
const { success, error: errorResponse } = require('../response-helpers');
|
||||||
|
const { NotFoundError, ValidationError } = require('../errors');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dependencies route factory
|
||||||
|
*
|
||||||
|
* @param {Object} deps - Explicit dependencies
|
||||||
|
* @param {Object} deps.dependencyManager - DependencyManager instance
|
||||||
|
* @param {Object} deps.servicesStateManager - State manager for services.json
|
||||||
|
* @param {Object} deps.docker - Docker client wrapper
|
||||||
|
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||||
|
* @param {Function} deps.logError - Error logging function
|
||||||
|
* @param {Function} deps.resyncHealthChecker - Health checker resync function
|
||||||
|
* @param {Object} deps.log - Logger instance
|
||||||
|
* @returns {express.Router}
|
||||||
|
*/
|
||||||
|
module.exports = function({
|
||||||
|
dependencyManager,
|
||||||
|
servicesStateManager,
|
||||||
|
docker,
|
||||||
|
asyncHandler,
|
||||||
|
logError,
|
||||||
|
resyncHealthChecker,
|
||||||
|
log,
|
||||||
|
}) {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// GET /dependencies/graph — Full dependency graph
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
router.get('/graph', asyncHandler(async (req, res) => {
|
||||||
|
const graph = await dependencyManager.getDependencyGraph();
|
||||||
|
success(res, { graph });
|
||||||
|
}, 'dep-graph'));
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// GET /dependencies/validate — Validate a proposed dep chain (query params)
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
router.get('/validate', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId, dependsOn } = req.query;
|
||||||
|
|
||||||
|
if (!serviceId) {
|
||||||
|
throw new ValidationError('serviceId query parameter is required');
|
||||||
|
}
|
||||||
|
|
||||||
|
// dependsOn may be a comma-separated string or already an array
|
||||||
|
let parsed;
|
||||||
|
if (Array.isArray(dependsOn)) {
|
||||||
|
parsed = dependsOn;
|
||||||
|
} else if (typeof dependsOn === 'string' && dependsOn.length > 0) {
|
||||||
|
parsed = dependsOn.split(',').map(s => s.trim()).filter(Boolean);
|
||||||
|
} else {
|
||||||
|
parsed = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await dependencyManager.validateDependencies(serviceId, parsed);
|
||||||
|
success(res, result);
|
||||||
|
}, 'dep-validate'));
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// GET /dependencies/:serviceId — Direct deps for one service
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
router.get('/:serviceId', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
const dependencies = await dependencyManager.getDependencies(serviceId);
|
||||||
|
const dependents = await dependencyManager.getDependents(serviceId);
|
||||||
|
|
||||||
|
// Read the service's current dependsOn array
|
||||||
|
const services = await servicesStateManager.read();
|
||||||
|
const allServices = Array.isArray(services) ? services : (services.services || []);
|
||||||
|
const service = allServices.find(s => s.id === serviceId);
|
||||||
|
|
||||||
|
if (!service) {
|
||||||
|
throw new NotFoundError(`Service "${serviceId}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
success(res, {
|
||||||
|
serviceId,
|
||||||
|
dependsOn: service.dependsOn || [],
|
||||||
|
dependencies,
|
||||||
|
dependents: dependents.map(d => ({ id: d.id, name: d.name })),
|
||||||
|
});
|
||||||
|
}, 'dep-get'));
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// GET /dependencies/:serviceId/chain — Ordered restart chain
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
router.get('/:serviceId/chain', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
const chain = await dependencyManager.getOrderedRestartChain(serviceId);
|
||||||
|
success(res, { serviceId, chain });
|
||||||
|
}, 'dep-chain'));
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// GET /dependencies/:serviceId/status — Dependency health status
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
router.get('/:serviceId/status', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
const statuses = await dependencyManager.getDependencyStatus(serviceId);
|
||||||
|
success(res, { serviceId, statuses });
|
||||||
|
}, 'dep-status'));
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// POST /dependencies/:serviceId — Set dependencies
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
router.post('/:serviceId', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
const { dependsOn } = req.body;
|
||||||
|
|
||||||
|
if (!Array.isArray(dependsOn)) {
|
||||||
|
throw new ValidationError('Request body must include dependsOn as an array of service IDs');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate first
|
||||||
|
const validation = await dependencyManager.validateDependencies(serviceId, dependsOn);
|
||||||
|
if (!validation.valid) {
|
||||||
|
return errorResponse(res, validation.errors.join('; '), 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update the service
|
||||||
|
let found = false;
|
||||||
|
await servicesStateManager.update(services => {
|
||||||
|
const arr = Array.isArray(services) ? services : [];
|
||||||
|
return arr.map(s => {
|
||||||
|
if (s.id === serviceId) {
|
||||||
|
found = true;
|
||||||
|
return { ...s, dependsOn: dependsOn.slice() };
|
||||||
|
}
|
||||||
|
return s;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!found) {
|
||||||
|
throw new NotFoundError(`Service "${serviceId}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info('dependency', 'Dependencies updated', { serviceId, dependsOn });
|
||||||
|
|
||||||
|
success(res, {
|
||||||
|
message: `Dependencies updated for "${serviceId}"`,
|
||||||
|
serviceId,
|
||||||
|
dependsOn,
|
||||||
|
});
|
||||||
|
}, 'dep-set'));
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// DELETE /dependencies/:serviceId — Remove all dependencies for a service
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
router.delete('/:serviceId', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
|
||||||
|
let found = false;
|
||||||
|
await servicesStateManager.update(services => {
|
||||||
|
const arr = Array.isArray(services) ? services : [];
|
||||||
|
return arr.map(s => {
|
||||||
|
if (s.id === serviceId) {
|
||||||
|
found = true;
|
||||||
|
const updated = { ...s };
|
||||||
|
delete updated.dependsOn;
|
||||||
|
return updated;
|
||||||
|
}
|
||||||
|
return s;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!found) {
|
||||||
|
throw new NotFoundError(`Service "${serviceId}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info('dependency', 'Dependencies removed', { serviceId });
|
||||||
|
|
||||||
|
success(res, {
|
||||||
|
message: `All dependencies removed for "${serviceId}"`,
|
||||||
|
serviceId,
|
||||||
|
});
|
||||||
|
}, 'dep-delete'));
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// POST /dependencies/:serviceId/restart — Restart with dependency chain
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
router.post('/:serviceId/restart', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
|
||||||
|
// Verify the service exists
|
||||||
|
const services = await servicesStateManager.read();
|
||||||
|
const allServices = Array.isArray(services) ? services : (services.services || []);
|
||||||
|
if (!allServices.find(s => s.id === serviceId)) {
|
||||||
|
throw new NotFoundError(`Service "${serviceId}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the chain first for the response (before async restart begins)
|
||||||
|
let chain;
|
||||||
|
try {
|
||||||
|
chain = await dependencyManager.getOrderedRestartChain(serviceId);
|
||||||
|
} catch (err) {
|
||||||
|
return errorResponse(res, err.message, 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Respond immediately with the chain order
|
||||||
|
success(res, {
|
||||||
|
message: `Dependency restart initiated for "${serviceId}"`,
|
||||||
|
serviceId,
|
||||||
|
chain,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Run the restart chain asynchronously so the client doesn't block
|
||||||
|
dependencyManager.restartWithDependencies(serviceId).catch(err => {
|
||||||
|
if (log) {
|
||||||
|
log.error('dependency', 'Async dependency restart failed', {
|
||||||
|
serviceId,
|
||||||
|
error: err.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, 'dep-restart'));
|
||||||
|
|
||||||
|
return router;
|
||||||
|
};
|
||||||
@@ -26,7 +26,8 @@ module.exports = function({
|
|||||||
log,
|
log,
|
||||||
safeErrorMessage,
|
safeErrorMessage,
|
||||||
fetchT,
|
fetchT,
|
||||||
credentialManager
|
credentialManager,
|
||||||
|
dnsPropagationChecker
|
||||||
}) {
|
}) {
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
|
|
||||||
@@ -139,6 +140,14 @@ module.exports = function({
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (result.status === 'ok') {
|
if (result.status === 'ok') {
|
||||||
|
// Start DNS propagation verification in background
|
||||||
|
if (dnsPropagationChecker && ip) {
|
||||||
|
const fullDomain = domain;
|
||||||
|
dnsPropagationChecker.startVerification(fullDomain, ip).catch(err => {
|
||||||
|
log('DNS propagation check start failed:', err.message);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
success(res, { message: `DNS record ${domain} -> ${ip} created` });
|
success(res, { message: `DNS record ${domain} -> ${ip} created` });
|
||||||
} else {
|
} else {
|
||||||
// Error handled by middleware
|
// Error handled by middleware
|
||||||
@@ -641,5 +650,68 @@ module.exports = function({
|
|||||||
}
|
}
|
||||||
}, 'dns-update'));
|
}, 'dns-update'));
|
||||||
|
|
||||||
|
// ===== DNS PROPAGATION =====
|
||||||
|
|
||||||
|
// GET /propagation — Get all recent DNS propagation checks
|
||||||
|
router.get('/propagation', asyncHandler(async (req, res) => {
|
||||||
|
if (!dnsPropagationChecker) {
|
||||||
|
return success(res, { verifications: [], message: 'DNS propagation checker not available' });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cleanup old entries
|
||||||
|
dnsPropagationChecker.cleanup();
|
||||||
|
|
||||||
|
const verifications = dnsPropagationChecker.getAllVerifications();
|
||||||
|
success(res, { verifications });
|
||||||
|
}, 'dns-propagation-all'));
|
||||||
|
|
||||||
|
// POST /propagation/verify — Manually trigger DNS propagation verification
|
||||||
|
router.post('/propagation/verify', asyncHandler(async (req, res) => {
|
||||||
|
if (!dnsPropagationChecker) {
|
||||||
|
return errorResponse(res, 'DNS propagation checker not available', 503);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { domain, expectedIp } = req.body;
|
||||||
|
|
||||||
|
if (!domain || !expectedIp) {
|
||||||
|
throw new ValidationError('domain and expectedIp are required');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate domain format
|
||||||
|
if (!REGEX.DOMAIN.test(domain)) {
|
||||||
|
throw new ValidationError('[DC-301] Invalid domain format');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate IP address
|
||||||
|
const validatorLib = require('validator');
|
||||||
|
if (!validatorLib.isIP(expectedIp)) {
|
||||||
|
throw new ValidationError('[DC-210] Invalid IP address');
|
||||||
|
}
|
||||||
|
|
||||||
|
const job = dnsPropagationChecker.startVerification(domain, expectedIp);
|
||||||
|
success(res, {
|
||||||
|
message: 'DNS propagation verification started',
|
||||||
|
domain,
|
||||||
|
expectedIp,
|
||||||
|
status: job.status
|
||||||
|
});
|
||||||
|
}, 'dns-propagation-verify'));
|
||||||
|
|
||||||
|
// GET /propagation/:domain — Get propagation status for a specific domain
|
||||||
|
router.get('/propagation/:domain', asyncHandler(async (req, res) => {
|
||||||
|
if (!dnsPropagationChecker) {
|
||||||
|
return success(res, { verification: null, message: 'DNS propagation checker not available' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const { domain } = req.params;
|
||||||
|
const status = dnsPropagationChecker.getVerificationStatus(domain);
|
||||||
|
|
||||||
|
if (!status) {
|
||||||
|
throw new NotFoundError(`No propagation check found for domain: ${domain}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
success(res, { verification: status });
|
||||||
|
}, 'dns-propagation-domain'));
|
||||||
|
|
||||||
return router;
|
return router;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -8,9 +8,10 @@ const express = require('express');
|
|||||||
* @param {Object} deps.healthChecker - Health checker
|
* @param {Object} deps.healthChecker - Health checker
|
||||||
* @param {Object} deps.updateManager - Update manager
|
* @param {Object} deps.updateManager - Update manager
|
||||||
* @param {Function} deps.logError - Error logging function
|
* @param {Function} deps.logError - Error logging function
|
||||||
|
* @param {Object} deps.dependencyManager - Dependency manager for restart chain events
|
||||||
* @returns {express.Router}
|
* @returns {express.Router}
|
||||||
*/
|
*/
|
||||||
module.exports = function({ resourceMonitor, healthChecker, updateManager, logError }) {
|
module.exports = function({ resourceMonitor, healthChecker, updateManager, logError, dependencyManager, autoRestartManager, driftDetector, sslMonitor, dnsPropagationChecker }) {
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
const clients = new Set();
|
const clients = new Set();
|
||||||
|
|
||||||
@@ -74,6 +75,48 @@ module.exports = function({ resourceMonitor, healthChecker, updateManager, logEr
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Dependency manager events
|
||||||
|
if (dependencyManager) {
|
||||||
|
dependencyManager.on('dependency-restart-start', (data) => {
|
||||||
|
broadcast('dependency-restart-start', data);
|
||||||
|
});
|
||||||
|
dependencyManager.on('dependency-restart-progress', (data) => {
|
||||||
|
broadcast('dependency-restart-progress', data);
|
||||||
|
});
|
||||||
|
dependencyManager.on('dependency-restart-complete', (data) => {
|
||||||
|
broadcast('dependency-restart-complete', data);
|
||||||
|
});
|
||||||
|
dependencyManager.on('dependency-restart-failed', (data) => {
|
||||||
|
broadcast('dependency-restart-failed', data);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Auto-restart manager events
|
||||||
|
if (autoRestartManager) {
|
||||||
|
autoRestartManager.on('auto-restart-attempt', (data) => broadcast('auto-restart-attempt', data));
|
||||||
|
autoRestartManager.on('auto-restart-success', (data) => broadcast('auto-restart-success', data));
|
||||||
|
autoRestartManager.on('auto-restart-failed', (data) => broadcast('auto-restart-failed', data));
|
||||||
|
autoRestartManager.on('auto-restart-max-reached', (data) => broadcast('auto-restart-max-reached', data));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config drift detector events
|
||||||
|
if (driftDetector) {
|
||||||
|
driftDetector.on('drift-detected', (data) => broadcast('drift-detected', data));
|
||||||
|
}
|
||||||
|
|
||||||
|
// SSL monitor events
|
||||||
|
if (sslMonitor) {
|
||||||
|
sslMonitor.on('cert-expiring', (data) => broadcast('cert-expiring', data));
|
||||||
|
sslMonitor.on('cert-critical', (data) => broadcast('cert-critical', data));
|
||||||
|
}
|
||||||
|
|
||||||
|
// DNS propagation checker events
|
||||||
|
if (dnsPropagationChecker) {
|
||||||
|
dnsPropagationChecker.on('propagation-check', (data) => broadcast('dns-propagation-check', data));
|
||||||
|
dnsPropagationChecker.on('propagation-complete', (data) => broadcast('dns-propagation-complete', data));
|
||||||
|
dnsPropagationChecker.on('propagation-timeout', (data) => broadcast('dns-propagation-timeout', data));
|
||||||
|
}
|
||||||
|
|
||||||
// SSE endpoint
|
// SSE endpoint
|
||||||
router.get('/stream', (req, res) => {
|
router.get('/stream', (req, res) => {
|
||||||
res.writeHead(200, {
|
res.writeHead(200, {
|
||||||
|
|||||||
@@ -372,7 +372,7 @@ module.exports = function({
|
|||||||
// Add a new service
|
// Add a new service
|
||||||
router.post('/services', asyncHandler(async (req, res) => {
|
router.post('/services', asyncHandler(async (req, res) => {
|
||||||
try {
|
try {
|
||||||
const { id, name, logo } = req.body;
|
const { id, name, logo, category, containerId, port, ip, tailscaleOnly } = req.body;
|
||||||
|
|
||||||
if (!id || !name) {
|
if (!id || !name) {
|
||||||
throw new ValidationError('id and name are required');
|
throw new ValidationError('id and name are required');
|
||||||
@@ -391,7 +391,14 @@ module.exports = function({
|
|||||||
throw new ConflictError(`Service "${id}" already exists`, id);
|
throw new ConflictError(`Service "${id}" already exists`, id);
|
||||||
}
|
}
|
||||||
|
|
||||||
services.push({ id, name, logo: logo || `/assets/${id}.png` });
|
const newService = { id, name, logo: logo || `/assets/${id}.png` };
|
||||||
|
// Persist optional metadata fields if provided
|
||||||
|
if (category) newService.category = category;
|
||||||
|
if (containerId) newService.containerId = containerId;
|
||||||
|
if (port) newService.port = port;
|
||||||
|
if (ip) newService.ip = ip;
|
||||||
|
if (typeof tailscaleOnly === 'boolean') newService.tailscaleOnly = tailscaleOnly;
|
||||||
|
services.push(newService);
|
||||||
return services;
|
return services;
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -542,6 +549,8 @@ module.exports = function({
|
|||||||
};
|
};
|
||||||
if (name) services[serviceIndex].name = name;
|
if (name) services[serviceIndex].name = name;
|
||||||
if (logo) services[serviceIndex].logo = logo;
|
if (logo) services[serviceIndex].logo = logo;
|
||||||
|
// Allow category update via update endpoint too (optional body field)
|
||||||
|
if (req.body.category !== undefined) services[serviceIndex].category = req.body.category || undefined;
|
||||||
results.services = 'updated';
|
results.services = 'updated';
|
||||||
} else {
|
} else {
|
||||||
results.services = 'not found';
|
results.services = 'not found';
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
/**
|
||||||
|
* SSL Monitor Routes
|
||||||
|
* REST API endpoints for SSL certificate monitoring.
|
||||||
|
*
|
||||||
|
* @module routes/ssl-monitor
|
||||||
|
*/
|
||||||
|
|
||||||
|
const express = require('express');
|
||||||
|
const { success, error: errorResponse, notFound } = require('../response-helpers');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SSL Monitor route factory
|
||||||
|
* @param {Object} deps - Explicit dependencies
|
||||||
|
* @param {Object} deps.sslMonitor - SSLMonitor instance
|
||||||
|
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||||
|
* @param {Function} deps.logError - Error logging function
|
||||||
|
* @returns {express.Router}
|
||||||
|
*/
|
||||||
|
module.exports = function({ sslMonitor, asyncHandler, logError }) {
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /ssl/certificates
|
||||||
|
* Get all SSL certificate statuses
|
||||||
|
*/
|
||||||
|
router.get('/certificates', asyncHandler(async (req, res) => {
|
||||||
|
const status = sslMonitor.getStatus();
|
||||||
|
success(res, { certificates: status });
|
||||||
|
}, 'ssl-certificates'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /ssl/certificates/:serviceId
|
||||||
|
* Get SSL certificate status for a specific service
|
||||||
|
*/
|
||||||
|
router.get('/certificates/:serviceId', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
const certStatus = sslMonitor.getServiceCertStatus(serviceId);
|
||||||
|
|
||||||
|
if (!certStatus) {
|
||||||
|
return notFound(res, `No SSL certificate status found for service: ${serviceId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
success(res, { certificate: certStatus });
|
||||||
|
}, 'ssl-certificate-service'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /ssl/check
|
||||||
|
* Trigger an on-demand check of all SSL certificates
|
||||||
|
*/
|
||||||
|
router.post('/check', asyncHandler(async (req, res) => {
|
||||||
|
const results = await sslMonitor.checkAll();
|
||||||
|
success(res, { certificates: results, message: 'SSL check completed' });
|
||||||
|
}, 'ssl-check-all'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /ssl/check/:serviceId
|
||||||
|
* Check the SSL certificate for a specific service
|
||||||
|
*/
|
||||||
|
router.post('/check/:serviceId', asyncHandler(async (req, res) => {
|
||||||
|
const { serviceId } = req.params;
|
||||||
|
|
||||||
|
// Look up the existing cert status to find the hostname
|
||||||
|
const existingCert = sslMonitor.getServiceCertStatus(serviceId);
|
||||||
|
if (!existingCert) {
|
||||||
|
return notFound(res, `No HTTPS URL found for service: ${serviceId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await sslMonitor.checkCert(existingCert.hostname, existingCert.port);
|
||||||
|
success(res, { certificate: { ...result, serviceId } });
|
||||||
|
} catch (err) {
|
||||||
|
errorResponse(res, `Failed to check SSL certificate: ${err.message}`, 500);
|
||||||
|
}
|
||||||
|
}, 'ssl-check-service'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /ssl/config
|
||||||
|
* Get current SSL monitoring configuration
|
||||||
|
*/
|
||||||
|
router.get('/config', asyncHandler(async (req, res) => {
|
||||||
|
const config = sslMonitor.getConfig();
|
||||||
|
success(res, { config });
|
||||||
|
}, 'ssl-config-get'));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /ssl/config
|
||||||
|
* Update SSL monitoring configuration
|
||||||
|
* Body: { enabled: boolean, intervalMs: number }
|
||||||
|
*/
|
||||||
|
router.post('/config', asyncHandler(async (req, res) => {
|
||||||
|
const { enabled, intervalMs } = req.body;
|
||||||
|
|
||||||
|
// Validate inputs
|
||||||
|
if (enabled !== undefined && typeof enabled !== 'boolean') {
|
||||||
|
return errorResponse(res, 'enabled must be a boolean', 400);
|
||||||
|
}
|
||||||
|
if (intervalMs !== undefined) {
|
||||||
|
if (typeof intervalMs !== 'number' || intervalMs < 60000) {
|
||||||
|
return errorResponse(res, 'intervalMs must be a number >= 60000 (1 minute)', 400);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const updates = {};
|
||||||
|
if (enabled !== undefined) updates.enabled = enabled;
|
||||||
|
if (intervalMs !== undefined) updates.intervalMs = intervalMs;
|
||||||
|
|
||||||
|
sslMonitor.updateConfig(updates);
|
||||||
|
const config = sslMonitor.getConfig();
|
||||||
|
success(res, { config, message: 'SSL monitoring config updated' });
|
||||||
|
}, 'ssl-config-update'));
|
||||||
|
|
||||||
|
return router;
|
||||||
|
};
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# DashCaddy Host-Side Updater
|
# DashCaddy Host-Side Updater
|
||||||
# Triggered by systemd path unit when the container writes trigger.json.
|
# Triggered by systemd path unit when the container writes trigger.json.
|
||||||
# Reads the trigger, backs up current API, copies new files, rebuilds container.
|
# Reads the trigger, backs up current API + data/, copies new files, rebuilds container.
|
||||||
# Writes result.json so the new container knows the outcome.
|
# Writes result.json so the new container knows the outcome.
|
||||||
#
|
#
|
||||||
# This runs on the HOST, outside the container.
|
# This runs on the HOST, outside the container.
|
||||||
@@ -16,6 +16,10 @@ readonly CONTAINER_NAME="dashcaddy-api"
|
|||||||
readonly MAX_BACKUPS=3
|
readonly MAX_BACKUPS=3
|
||||||
readonly HEALTH_TIMEOUT=60
|
readonly HEALTH_TIMEOUT=60
|
||||||
|
|
||||||
|
# Data directory backup — stored alongside code backups so everything rolls back together
|
||||||
|
readonly DATA_SOURCE_DIR="/opt/dashcaddy/dashcaddy-api/data"
|
||||||
|
readonly DATA_BACKUP_PREFIX="data-backup"
|
||||||
|
|
||||||
log() { echo "[dashcaddy-update] $(date '+%Y-%m-%d %H:%M:%S') $*"; }
|
log() { echo "[dashcaddy-update] $(date '+%Y-%m-%d %H:%M:%S') $*"; }
|
||||||
|
|
||||||
write_result() {
|
write_result() {
|
||||||
@@ -56,6 +60,34 @@ cleanup_old_backups() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Data backup (rsync for efficiency + permissions) ──────────────────────────
|
||||||
|
backup_data_dir() {
|
||||||
|
local backup_dir="$1"
|
||||||
|
if [[ -d "$DATA_SOURCE_DIR" ]]; then
|
||||||
|
log "Backing up data/ to ${backup_dir}/${DATA_BACKUP_PREFIX}/"
|
||||||
|
mkdir -p "${backup_dir}/${DATA_BACKUP_PREFIX}"
|
||||||
|
rsync -a --delete "$DATA_SOURCE_DIR/" "${backup_dir}/${DATA_BACKUP_PREFIX}/" 2>/dev/null \
|
||||||
|
|| cp -a "$DATA_SOURCE_DIR" "${backup_dir}/${DATA_BACKUP_PREFIX}"
|
||||||
|
log "Data backup complete ($(du -sh "${backup_dir}/${DATA_BACKUP_PREFIX}" 2>/dev/null | cut -f1))"
|
||||||
|
else
|
||||||
|
log "WARNING: Data source dir $DATA_SOURCE_DIR not found — skipping data backup"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Data restore ──────────────────────────────────────────────────────────────
|
||||||
|
restore_data_dir() {
|
||||||
|
local backup_dir="$1"
|
||||||
|
local data_backup="${backup_dir}/${DATA_BACKUP_PREFIX}"
|
||||||
|
if [[ -d "$data_backup" ]]; then
|
||||||
|
log "Restoring data/ from backup..."
|
||||||
|
rsync -a --delete "$data_backup/" "$DATA_SOURCE_DIR/" 2>/dev/null \
|
||||||
|
|| cp -a "$data_backup" "$DATA_SOURCE_DIR"
|
||||||
|
log "Data restored successfully"
|
||||||
|
else
|
||||||
|
log "WARNING: No data backup found at ${data_backup} — data/ not restored"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
wait_for_health() {
|
wait_for_health() {
|
||||||
local port="${1:-3001}"
|
local port="${1:-3001}"
|
||||||
local timeout="$HEALTH_TIMEOUT"
|
local timeout="$HEALTH_TIMEOUT"
|
||||||
@@ -75,6 +107,59 @@ wait_for_health() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Shared rollback: restore code + data ────────────────────────────────────
|
||||||
|
rollback_restore() {
|
||||||
|
local backup_dir="$1"
|
||||||
|
log "Rolling back: restoring code files..."
|
||||||
|
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||||
|
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
if [[ -d "$backup_dir/routes" ]]; then
|
||||||
|
rm -rf "$api_source_dir/routes"
|
||||||
|
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||||
|
fi
|
||||||
|
if [[ -d "$backup_dir/src" ]]; then
|
||||||
|
rm -rf "$api_source_dir/src"
|
||||||
|
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||||
|
fi
|
||||||
|
restore_data_dir "$backup_dir"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Shared container restart (preserves SERVICES_FILE env var) ───────────────
|
||||||
|
# Uses rm + run so new env vars (e.g. SERVICES_FILE) take effect.
|
||||||
|
# If docker-compose is not configured, falls back to docker start.
|
||||||
|
restart_container() {
|
||||||
|
local image="$1"
|
||||||
|
log "Restarting container (rm + run to pick up env vars)..."
|
||||||
|
# Stop and remove existing container so new env var is applied
|
||||||
|
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
||||||
|
|
||||||
|
# Re-create with same volumes and the SERVICES_FILE env var
|
||||||
|
docker run -d --restart unless-stopped --name "$CONTAINER_NAME" \
|
||||||
|
-p 127.0.0.1:3001:3001 \
|
||||||
|
-v /opt/dashcaddy/dashcaddy-api/data:/app/data \
|
||||||
|
-e SERVICES_FILE=/app/data/services.json \
|
||||||
|
"$image"
|
||||||
|
log "Container restarted with fresh env"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Code-only restore (used after failed build when data hasn't changed yet) ──
|
||||||
|
code_restore() {
|
||||||
|
local backup_dir="$1"
|
||||||
|
log "Restoring code files..."
|
||||||
|
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||||
|
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
if [[ -d "$backup_dir/routes" ]]; then
|
||||||
|
rm -rf "$api_source_dir/routes"
|
||||||
|
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||||
|
fi
|
||||||
|
if [[ -d "$backup_dir/src" ]]; then
|
||||||
|
rm -rf "$api_source_dir/src"
|
||||||
|
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
local start_time
|
local start_time
|
||||||
start_time=$(date +%s)
|
start_time=$(date +%s)
|
||||||
@@ -94,45 +179,69 @@ main() {
|
|||||||
staging_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}'))['stagingDir'])")
|
staging_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}'))['stagingDir'])")
|
||||||
api_source_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}'))['apiSourceDir'])")
|
api_source_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}'))['apiSourceDir'])")
|
||||||
commit=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('commit') or '')")
|
commit=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('commit') or '')")
|
||||||
# Frontend paths — optional (older self-updaters don't write these). When
|
|
||||||
# present, this script also syncs the dashboard files (Caddy serves them
|
|
||||||
# directly from the host; the container path /app/dashboard isn't mounted).
|
|
||||||
frontend_staging_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('frontendStagingDir') or '')")
|
frontend_staging_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('frontendStagingDir') or '')")
|
||||||
frontend_target_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('frontendTargetDir') or '')")
|
frontend_target_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('frontendTargetDir') or '')")
|
||||||
|
# Handle action=rollback (no new version to deploy)
|
||||||
|
local to_version="${version}"
|
||||||
|
|
||||||
log "=== ${action^^}: v${from_version} -> v${version} ==="
|
log "=== ${action^^}: v${from_version} -> v${to_version} ==="
|
||||||
log "Staging: ${staging_dir}"
|
log "Staging: ${staging_dir}"
|
||||||
log "API source: ${api_source_dir}"
|
log "API source: ${api_source_dir}"
|
||||||
|
|
||||||
# Consume the trigger immediately so we don't re-process on failure
|
# Consume the trigger immediately so we don't re-process on failure
|
||||||
mv "$TRIGGER_FILE" "${TRIGGER_FILE}.processing"
|
mv "$TRIGGER_FILE" "${TRIGGER_FILE}.processing"
|
||||||
|
|
||||||
# 2. Validate staging directory
|
# ── Handle rollback ────────────────────────────────────────────────────────
|
||||||
if [[ ! -d "$staging_dir" ]]; then
|
if [[ "$action" == "rollback" ]]; then
|
||||||
log "ERROR: Staging directory not found: ${staging_dir}"
|
local backup_dir="${BACKUPS_DIR}/${version}"
|
||||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "Staging directory not found"
|
if [[ ! -d "$backup_dir" ]]; then
|
||||||
|
log "ERROR: No backup found for version ${version}"
|
||||||
|
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "No backup found for version ${version}"
|
||||||
rm -f "${TRIGGER_FILE}.processing"
|
rm -f "${TRIGGER_FILE}.processing"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 3. Backup current API files
|
log "Performing rollback to v${version}..."
|
||||||
|
rollback_restore "$backup_dir"
|
||||||
|
|
||||||
|
# Rebuild old code
|
||||||
|
log "Rebuilding container..."
|
||||||
|
cd "$api_source_dir"
|
||||||
|
docker build -t dashcaddy-dashcaddy-api:latest . 2>&1 | tail -1 || true
|
||||||
|
|
||||||
|
restart_container "dashcaddy-dashcaddy-api:latest"
|
||||||
|
wait_for_health || log "WARNING: Health check failed after rollback"
|
||||||
|
|
||||||
|
write_result "true" "$version" "$(( $(date +%s) - start_time ))"
|
||||||
|
rm -f "${TRIGGER_FILE}.processing"
|
||||||
|
log "=== Rollback complete ==="
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Handle update ───────────────────────────────────────────────────────────
|
||||||
|
if [[ ! -d "$staging_dir" ]]; then
|
||||||
|
log "ERROR: Staging directory not found: ${staging_dir}"
|
||||||
|
write_result "false" "$to_version" "$(( $(date +%s) - start_time ))" "Staging directory not found"
|
||||||
|
rm -f "${TRIGGER_FILE}.processing"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 2. Backup current API code + data/
|
||||||
local backup_dir="${BACKUPS_DIR}/${from_version}"
|
local backup_dir="${BACKUPS_DIR}/${from_version}"
|
||||||
mkdir -p "$backup_dir"
|
mkdir -p "$backup_dir"
|
||||||
log "Backing up current API files to ${backup_dir}"
|
log "Backing up current API files to ${backup_dir}"
|
||||||
|
|
||||||
# Copy all JS files, package.json, Dockerfile, and tracked subdirs
|
|
||||||
for item in "$api_source_dir"/*.js "$api_source_dir"/package.json "$api_source_dir"/package-lock.json "$api_source_dir"/Dockerfile "$api_source_dir"/openapi.yaml "$api_source_dir"/VERSION; do
|
for item in "$api_source_dir"/*.js "$api_source_dir"/package.json "$api_source_dir"/package-lock.json "$api_source_dir"/Dockerfile "$api_source_dir"/openapi.yaml "$api_source_dir"/VERSION; do
|
||||||
[[ -f "$item" ]] && cp -f "$item" "$backup_dir/" 2>/dev/null || true
|
[[ -f "$item" ]] && cp -f "$item" "$backup_dir/" 2>/dev/null || true
|
||||||
done
|
done
|
||||||
[[ -d "$api_source_dir/routes" ]] && cp -rf "$api_source_dir/routes" "$backup_dir/"
|
[[ -d "$api_source_dir/routes" ]] && cp -rf "$api_source_dir/routes" "$backup_dir/"
|
||||||
[[ -d "$api_source_dir/src" ]] && cp -rf "$api_source_dir/src" "$backup_dir/"
|
[[ -d "$api_source_dir/src" ]] && cp -rf "$api_source_dir/src" "$backup_dir/"
|
||||||
# VERSION (commit hash) was copied from api_source_dir above; preserve as-is
|
|
||||||
# so a rollback restores the original commit marker. The version *string* is
|
# Backup data/ directory (services.json, config.json, credentials, etc.)
|
||||||
# already encoded in the backup dir name (${from_version}).
|
backup_data_dir "$backup_dir"
|
||||||
|
|
||||||
cleanup_old_backups
|
cleanup_old_backups
|
||||||
|
|
||||||
# 4. Copy new files from staging to API source
|
# 3. Copy new files from staging to API source
|
||||||
log "Deploying new API files..."
|
log "Deploying new API files..."
|
||||||
for item in "$staging_dir"/*.js "$staging_dir"/package.json "$staging_dir"/package-lock.json "$staging_dir"/Dockerfile "$staging_dir"/openapi.yaml "$staging_dir"/VERSION; do
|
for item in "$staging_dir"/*.js "$staging_dir"/package.json "$staging_dir"/package-lock.json "$staging_dir"/Dockerfile "$staging_dir"/openapi.yaml "$staging_dir"/VERSION; do
|
||||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||||
@@ -145,19 +254,11 @@ main() {
|
|||||||
rm -rf "$api_source_dir/src"
|
rm -rf "$api_source_dir/src"
|
||||||
cp -rf "$staging_dir/src" "$api_source_dir/src"
|
cp -rf "$staging_dir/src" "$api_source_dir/src"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Belt-and-suspenders: always write the commit from trigger.json to VERSION,
|
|
||||||
# even if the tarball didn't include one. The container's self-updater uses
|
|
||||||
# this to detect the "same version, different commit" case.
|
|
||||||
if [[ -n "$commit" ]]; then
|
if [[ -n "$commit" ]]; then
|
||||||
echo "$commit" > "$api_source_dir/VERSION"
|
echo "$commit" > "$api_source_dir/VERSION"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 4b. Sync frontend. Caddy serves the dashboard directly from the host
|
# 3b. Sync frontend
|
||||||
# filesystem; the container-side copy in older self-updater.js builds wrote
|
|
||||||
# to /app/dashboard which isn't always mounted, so the real sync happens
|
|
||||||
# here. Trigger fields take precedence; if absent (older self-updater),
|
|
||||||
# fall back to: staging dir's sibling status/ + first existing known target.
|
|
||||||
if [[ -z "$frontend_staging_dir" ]]; then
|
if [[ -z "$frontend_staging_dir" ]]; then
|
||||||
parent_staging=$(dirname "$staging_dir")
|
parent_staging=$(dirname "$staging_dir")
|
||||||
[[ -d "$parent_staging/status" ]] && frontend_staging_dir="$parent_staging/status"
|
[[ -d "$parent_staging/status" ]] && frontend_staging_dir="$parent_staging/status"
|
||||||
@@ -175,107 +276,55 @@ main() {
|
|||||||
for sub in dist css vendor js; do
|
for sub in dist css vendor js; do
|
||||||
if [[ -d "$frontend_staging_dir/$sub" ]]; then
|
if [[ -d "$frontend_staging_dir/$sub" ]]; then
|
||||||
mkdir -p "$frontend_target_dir/$sub"
|
mkdir -p "$frontend_target_dir/$sub"
|
||||||
cp -rf "$frontend_staging_dir/$sub"/* "$frontend_target_dir/$sub/" 2>/dev/null || true
|
cp -rf "$frontend_staging_dir/$sub/"* "$frontend_target_dir/$sub/" 2>/dev/null || true
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
# assets/ is mounted into the container; usually already in sync via bind
|
|
||||||
# mount, but if a release ships new assets we want them on disk too.
|
|
||||||
if [[ -d "$frontend_staging_dir/assets" ]]; then
|
if [[ -d "$frontend_staging_dir/assets" ]]; then
|
||||||
mkdir -p "$frontend_target_dir/assets"
|
mkdir -p "$frontend_target_dir/assets"
|
||||||
cp -rf "$frontend_staging_dir/assets"/* "$frontend_target_dir/assets/" 2>/dev/null || true
|
cp -rf "$frontend_staging_dir/assets/"* "$frontend_target_dir/assets/" 2>/dev/null || true
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 5. Rebuild container
|
# 4. Rebuild container
|
||||||
log "Rebuilding container..."
|
log "Rebuilding container..."
|
||||||
cd "$api_source_dir"
|
cd "$api_source_dir"
|
||||||
|
|
||||||
local build_ok=false
|
local build_ok=false
|
||||||
if docker compose build --quiet 2>&1; then
|
local image_tag="dashcaddy-dashcaddy-api:latest"
|
||||||
build_ok=true
|
|
||||||
elif docker-compose build --quiet 2>&1; then
|
if docker build -t "$image_tag" . 2>&1; then
|
||||||
build_ok=true
|
build_ok=true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$build_ok" != "true" ]]; then
|
if [[ "$build_ok" != "true" ]]; then
|
||||||
log "ERROR: Docker build failed — rolling back"
|
log "ERROR: Docker build failed — rolling back code + data"
|
||||||
|
code_restore "$backup_dir"
|
||||||
# Restore backup
|
docker build -t "$image_tag" . 2>&1 | tail -3 || true
|
||||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
restart_container "$image_tag"
|
||||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
wait_for_health || true
|
||||||
done
|
write_result "false" "$to_version" "$(( $(date +%s) - start_time ))" "Docker build failed"
|
||||||
if [[ -d "$backup_dir/routes" ]]; then
|
|
||||||
rm -rf "$api_source_dir/routes"
|
|
||||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
|
||||||
fi
|
|
||||||
if [[ -d "$backup_dir/src" ]]; then
|
|
||||||
rm -rf "$api_source_dir/src"
|
|
||||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
|
||||||
fi
|
|
||||||
|
|
||||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "Docker build failed"
|
|
||||||
rm -f "${TRIGGER_FILE}.processing"
|
rm -f "${TRIGGER_FILE}.processing"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 6. Restart container
|
# 5. Restart container (rm + run so new env vars take effect)
|
||||||
log "Restarting container..."
|
restart_container "$image_tag"
|
||||||
if docker compose up -d 2>&1 || docker-compose up -d 2>&1; then
|
|
||||||
log "Container restarted"
|
|
||||||
else
|
|
||||||
log "ERROR: Container restart failed — rolling back"
|
|
||||||
|
|
||||||
# Restore backup
|
# 6. Health check
|
||||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
|
||||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
|
||||||
done
|
|
||||||
if [[ -d "$backup_dir/routes" ]]; then
|
|
||||||
rm -rf "$api_source_dir/routes"
|
|
||||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
|
||||||
fi
|
|
||||||
if [[ -d "$backup_dir/src" ]]; then
|
|
||||||
rm -rf "$api_source_dir/src"
|
|
||||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
|
||||||
fi
|
|
||||||
|
|
||||||
docker compose build --quiet 2>&1 || docker-compose build --quiet 2>&1 || true
|
|
||||||
docker compose up -d 2>&1 || docker-compose up -d 2>&1 || true
|
|
||||||
|
|
||||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "Container restart failed"
|
|
||||||
rm -f "${TRIGGER_FILE}.processing"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 7. Health check
|
|
||||||
if wait_for_health; then
|
if wait_for_health; then
|
||||||
local duration=$(( $(date +%s) - start_time ))
|
local duration=$(( $(date +%s) - start_time ))
|
||||||
log "=== Update successful: v${version} in ${duration}s ==="
|
log "=== Update successful: v${to_version} in ${duration}s ==="
|
||||||
write_result "true" "$version" "$duration"
|
write_result "true" "$to_version" "$duration"
|
||||||
else
|
else
|
||||||
local duration=$(( $(date +%s) - start_time ))
|
local duration=$(( $(date +%s) - start_time ))
|
||||||
log "ERROR: Health check failed after update — rolling back"
|
log "ERROR: Health check failed after update — rolling back code + data"
|
||||||
|
rollback_restore "$backup_dir"
|
||||||
# Restore backup
|
docker build -t "$image_tag" . 2>&1 | tail -3 || true
|
||||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
restart_container "$image_tag"
|
||||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
|
||||||
done
|
|
||||||
if [[ -d "$backup_dir/routes" ]]; then
|
|
||||||
rm -rf "$api_source_dir/routes"
|
|
||||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
|
||||||
fi
|
|
||||||
if [[ -d "$backup_dir/src" ]]; then
|
|
||||||
rm -rf "$api_source_dir/src"
|
|
||||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
|
||||||
fi
|
|
||||||
|
|
||||||
docker compose build --quiet 2>&1 || docker-compose build --quiet 2>&1 || true
|
|
||||||
docker compose up -d 2>&1 || docker-compose up -d 2>&1 || true
|
|
||||||
wait_for_health || log "WARNING: Rollback health check also failed"
|
wait_for_health || log "WARNING: Rollback health check also failed"
|
||||||
|
write_result "false" "$to_version" "$duration" "Health check failed after update"
|
||||||
write_result "false" "$version" "$duration" "Health check failed after update"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 8. Cleanup
|
# 7. Cleanup
|
||||||
rm -f "${TRIGGER_FILE}.processing"
|
rm -f "${TRIGGER_FILE}.processing"
|
||||||
rm -rf "${UPDATES_DIR}/staging" 2>/dev/null || true
|
rm -rf "${UPDATES_DIR}/staging" 2>/dev/null || true
|
||||||
|
|
||||||
|
|||||||
@@ -77,6 +77,15 @@ const themesRoutes = require('../routes/themes');
|
|||||||
const dockerResourcesRoutes = require('../routes/docker-resources');
|
const dockerResourcesRoutes = require('../routes/docker-resources');
|
||||||
const eventsRoutes = require('../routes/events');
|
const eventsRoutes = require('../routes/events');
|
||||||
const workflowsRoutes = require('../routes/workflows');
|
const workflowsRoutes = require('../routes/workflows');
|
||||||
|
const dependenciesRoutes = require('../routes/dependencies');
|
||||||
|
const DependencyManager = require('../dependency-manager');
|
||||||
|
const autoRestartRoutes = require('../routes/auto-restart');
|
||||||
|
const configDriftRoutes = require('../routes/config-drift');
|
||||||
|
const sslMonitorRoutes = require('../routes/ssl-monitor');
|
||||||
|
const { AutoRestartManager } = require('../auto-restart-manager');
|
||||||
|
const { ConfigDriftDetector } = require('../config-drift-detector');
|
||||||
|
const { SSLMonitor } = require('../ssl-monitor');
|
||||||
|
const { DNSPropagationChecker } = require('../dns-propagation');
|
||||||
|
|
||||||
// Constants
|
// Constants
|
||||||
const { APP } = require('../constants');
|
const { APP } = require('../constants');
|
||||||
@@ -335,6 +344,39 @@ async function createApp() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Initialize dependency manager
|
||||||
|
const dependencyManager = new DependencyManager({
|
||||||
|
servicesStateManager,
|
||||||
|
docker: ctx.docker,
|
||||||
|
notification: ctx.notification,
|
||||||
|
log,
|
||||||
|
});
|
||||||
|
ctx.dependencyManager = dependencyManager;
|
||||||
|
log.info('app', 'Dependency manager initialized');
|
||||||
|
|
||||||
|
// Initialize auto-restart manager
|
||||||
|
const autoRestartManager = new AutoRestartManager(ctx);
|
||||||
|
ctx.autoRestartManager = autoRestartManager;
|
||||||
|
autoRestartManager.start();
|
||||||
|
log.info('app', 'Auto-restart manager initialized');
|
||||||
|
|
||||||
|
// Initialize config drift detector
|
||||||
|
const driftDetector = new ConfigDriftDetector(ctx);
|
||||||
|
ctx.driftDetector = driftDetector;
|
||||||
|
driftDetector.startPolling(300000); // 5 min
|
||||||
|
log.info('app', 'Config drift detector initialized');
|
||||||
|
|
||||||
|
// Initialize SSL monitor
|
||||||
|
const sslMonitor = new SSLMonitor(ctx);
|
||||||
|
ctx.sslMonitor = sslMonitor;
|
||||||
|
sslMonitor.start(3600000); // 1 hour
|
||||||
|
log.info('app', 'SSL monitor initialized');
|
||||||
|
|
||||||
|
// Initialize DNS propagation checker
|
||||||
|
const dnsPropagationChecker = new DNSPropagationChecker(ctx);
|
||||||
|
ctx.dnsPropagationChecker = dnsPropagationChecker;
|
||||||
|
log.info('app', 'DNS propagation checker initialized');
|
||||||
|
|
||||||
// Build versioned API router
|
// Build versioned API router
|
||||||
const apiRouter = express.Router();
|
const apiRouter = express.Router();
|
||||||
|
|
||||||
@@ -375,7 +417,8 @@ async function createApp() {
|
|||||||
log: ctx.log,
|
log: ctx.log,
|
||||||
safeErrorMessage: ctx.safeErrorMessage,
|
safeErrorMessage: ctx.safeErrorMessage,
|
||||||
fetchT: ctx.fetchT,
|
fetchT: ctx.fetchT,
|
||||||
credentialManager: ctx.credentialManager
|
credentialManager: ctx.credentialManager,
|
||||||
|
dnsPropagationChecker: ctx.dnsPropagationChecker
|
||||||
}));
|
}));
|
||||||
apiRouter.use('/notifications', notificationRoutes({
|
apiRouter.use('/notifications', notificationRoutes({
|
||||||
notification: ctx.notification,
|
notification: ctx.notification,
|
||||||
@@ -384,7 +427,8 @@ async function createApp() {
|
|||||||
apiRouter.use('/containers', containerRoutes({
|
apiRouter.use('/containers', containerRoutes({
|
||||||
docker: ctx.docker,
|
docker: ctx.docker,
|
||||||
log: ctx.log,
|
log: ctx.log,
|
||||||
asyncHandler: ctx.asyncHandler
|
asyncHandler: ctx.asyncHandler,
|
||||||
|
workflowEngine: ctx.workflowEngine
|
||||||
}));
|
}));
|
||||||
apiRouter.use(serviceRoutes({
|
apiRouter.use(serviceRoutes({
|
||||||
servicesStateManager: ctx.servicesStateManager,
|
servicesStateManager: ctx.servicesStateManager,
|
||||||
@@ -488,13 +532,42 @@ async function createApp() {
|
|||||||
resourceMonitor: ctx.resourceMonitor,
|
resourceMonitor: ctx.resourceMonitor,
|
||||||
healthChecker: ctx.healthChecker,
|
healthChecker: ctx.healthChecker,
|
||||||
updateManager: ctx.updateManager,
|
updateManager: ctx.updateManager,
|
||||||
logError: ctx.logError
|
logError: ctx.logError,
|
||||||
|
dependencyManager: ctx.dependencyManager,
|
||||||
|
autoRestartManager: ctx.autoRestartManager,
|
||||||
|
driftDetector: ctx.driftDetector,
|
||||||
|
sslMonitor: ctx.sslMonitor,
|
||||||
|
dnsPropagationChecker: ctx.dnsPropagationChecker
|
||||||
}));
|
}));
|
||||||
apiRouter.use(workflowsRoutes({
|
apiRouter.use(workflowsRoutes({
|
||||||
workflowEngine: ctx.workflowEngine,
|
workflowEngine: ctx.workflowEngine,
|
||||||
licenseManager: ctx.licenseManager,
|
licenseManager: ctx.licenseManager,
|
||||||
asyncHandler: ctx.asyncHandler
|
asyncHandler: ctx.asyncHandler
|
||||||
}));
|
}));
|
||||||
|
apiRouter.use('/dependencies', dependenciesRoutes({
|
||||||
|
dependencyManager: ctx.dependencyManager,
|
||||||
|
servicesStateManager: ctx.servicesStateManager,
|
||||||
|
docker: ctx.docker,
|
||||||
|
asyncHandler: ctx.asyncHandler,
|
||||||
|
logError: ctx.logError,
|
||||||
|
resyncHealthChecker: ctx.resyncHealthChecker,
|
||||||
|
log: ctx.log,
|
||||||
|
}));
|
||||||
|
apiRouter.use(autoRestartRoutes({
|
||||||
|
autoRestartManager: ctx.autoRestartManager,
|
||||||
|
asyncHandler: ctx.asyncHandler,
|
||||||
|
logError: ctx.logError,
|
||||||
|
}));
|
||||||
|
apiRouter.use(configDriftRoutes({
|
||||||
|
driftDetector: ctx.driftDetector,
|
||||||
|
asyncHandler: ctx.asyncHandler,
|
||||||
|
logError: ctx.logError,
|
||||||
|
}));
|
||||||
|
apiRouter.use(sslMonitorRoutes({
|
||||||
|
sslMonitor: ctx.sslMonitor,
|
||||||
|
asyncHandler: ctx.asyncHandler,
|
||||||
|
logError: ctx.logError,
|
||||||
|
}));
|
||||||
|
|
||||||
// Inline API routes
|
// Inline API routes
|
||||||
apiRouter.get('/health', (req, res) => {
|
apiRouter.get('/health', (req, res) => {
|
||||||
|
|||||||
@@ -0,0 +1,411 @@
|
|||||||
|
/**
|
||||||
|
* SSL Certificate Monitor
|
||||||
|
* Periodically checks SSL certificates on services with HTTPS URLs.
|
||||||
|
* Alerts at 30, 14, and 7 days before expiry.
|
||||||
|
*
|
||||||
|
* @module ssl-monitor
|
||||||
|
*/
|
||||||
|
|
||||||
|
const tls = require('tls');
|
||||||
|
const EventEmitter = require('events');
|
||||||
|
const path = require('path');
|
||||||
|
const { readJsonFile, writeJsonFile } = require('./fs-helpers');
|
||||||
|
const { resolveServiceUrl } = require('./url-resolver');
|
||||||
|
|
||||||
|
/** Default check interval: 1 hour */
|
||||||
|
const DEFAULT_INTERVAL_MS = 3600000;
|
||||||
|
|
||||||
|
/** Alert thresholds in days */
|
||||||
|
const THRESHOLDS = {
|
||||||
|
WARNING: 30,
|
||||||
|
URGENT: 14,
|
||||||
|
CRITICAL: 7
|
||||||
|
};
|
||||||
|
|
||||||
|
/** TLS connection timeout in milliseconds */
|
||||||
|
const TLS_TIMEOUT_MS = 10000;
|
||||||
|
|
||||||
|
class SSLMonitor extends EventEmitter {
|
||||||
|
/**
|
||||||
|
* Create an SSLMonitor instance.
|
||||||
|
* @param {Object} ctx - Shared application context
|
||||||
|
* @param {Object} ctx.servicesStateManager - State manager for reading services
|
||||||
|
* @param {Function} ctx.buildServiceUrl - URL builder helper
|
||||||
|
* @param {Object} ctx.siteConfig - Site configuration
|
||||||
|
* @param {Object} ctx.notification - NotificationManager instance
|
||||||
|
* @param {Object} ctx.log - Logger instance
|
||||||
|
* @param {string} [ctx.SSL_CACHE_FILE] - Path to persist SSL cache
|
||||||
|
*/
|
||||||
|
constructor(ctx) {
|
||||||
|
super();
|
||||||
|
this.ctx = ctx;
|
||||||
|
this.log = ctx.log || console;
|
||||||
|
|
||||||
|
/** @type {Map<string, Object>} hostname → last cert check result */
|
||||||
|
this.certStatus = new Map();
|
||||||
|
|
||||||
|
/** @type {Map<string, number>} hostname → last notified threshold level */
|
||||||
|
this.notifiedThresholds = new Map();
|
||||||
|
|
||||||
|
/** @type {Map<string, string>} hostname → service ID mapping */
|
||||||
|
this.hostnameToServiceId = new Map();
|
||||||
|
|
||||||
|
/** @type {NodeJS.Timeout|null} */
|
||||||
|
this.intervalHandle = null;
|
||||||
|
|
||||||
|
/** Current config */
|
||||||
|
this.config = {
|
||||||
|
enabled: true,
|
||||||
|
intervalMs: DEFAULT_INTERVAL_MS
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Cache file path */
|
||||||
|
this.cacheFile = ctx.SSL_CACHE_FILE ||
|
||||||
|
path.join(path.dirname(ctx.SERVICES_FILE || './data'), 'ssl-cache.json');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check the SSL certificate for a given hostname and port.
|
||||||
|
* Connects via TLS with rejectUnauthorized: false to retrieve certificate info.
|
||||||
|
*
|
||||||
|
* @param {string} hostname - The hostname to check
|
||||||
|
* @param {number} [port=443] - The port to connect to
|
||||||
|
* @returns {Promise<Object>} Certificate information
|
||||||
|
*/
|
||||||
|
async checkCert(hostname, port = 443) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const socket = tls.connect({
|
||||||
|
host: hostname,
|
||||||
|
port,
|
||||||
|
rejectUnauthorized: false,
|
||||||
|
servername: hostname,
|
||||||
|
timeout: TLS_TIMEOUT_MS
|
||||||
|
}, () => {
|
||||||
|
try {
|
||||||
|
const cert = socket.getPeerCertificate();
|
||||||
|
|
||||||
|
if (!cert || Object.keys(cert).length === 0) {
|
||||||
|
socket.destroy();
|
||||||
|
return reject(new Error(`No certificate returned for ${hostname}:${port}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
const validFrom = new Date(cert.valid_from);
|
||||||
|
const validTo = new Date(cert.valid_to);
|
||||||
|
const now = new Date();
|
||||||
|
const msRemaining = validTo.getTime() - now.getTime();
|
||||||
|
const daysRemaining = Math.ceil(msRemaining / (1000 * 60 * 60 * 24));
|
||||||
|
|
||||||
|
const result = {
|
||||||
|
hostname,
|
||||||
|
port,
|
||||||
|
subject: cert.subject?.CN || cert.subject?.O || 'Unknown',
|
||||||
|
issuer: cert.issuer?.CN || cert.issuer?.O || 'Unknown',
|
||||||
|
validFrom: cert.valid_from,
|
||||||
|
validTo: cert.valid_to,
|
||||||
|
daysRemaining,
|
||||||
|
fingerprint: cert.fingerprint || null,
|
||||||
|
isExpiring: daysRemaining <= THRESHOLDS.WARNING,
|
||||||
|
checkedAt: new Date().toISOString()
|
||||||
|
};
|
||||||
|
|
||||||
|
socket.destroy();
|
||||||
|
resolve(result);
|
||||||
|
} catch (err) {
|
||||||
|
socket.destroy();
|
||||||
|
reject(err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('error', (err) => {
|
||||||
|
reject(new Error(`TLS connect error for ${hostname}:${port}: ${err.message}`));
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.setTimeout(TLS_TIMEOUT_MS, () => {
|
||||||
|
socket.destroy(new Error(`TLS connection timeout for ${hostname}:${port}`));
|
||||||
|
reject(new Error(`TLS connection timeout for ${hostname}:${port}`));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check SSL certificates for all services that have HTTPS URLs.
|
||||||
|
* Reads services from ctx.servicesStateManager, resolves URLs, and checks each HTTPS cert.
|
||||||
|
*
|
||||||
|
* @returns {Promise<Object>} Map of hostname → cert status
|
||||||
|
*/
|
||||||
|
async checkAll() {
|
||||||
|
if (!this.config.enabled) {
|
||||||
|
this.log.info('ssl-monitor', 'SSL monitoring is disabled, skipping check');
|
||||||
|
return this.getStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
let servicesData;
|
||||||
|
try {
|
||||||
|
servicesData = await this.ctx.servicesStateManager.read();
|
||||||
|
} catch (err) {
|
||||||
|
this.log.error('ssl-monitor', 'Failed to read services', { error: err.message });
|
||||||
|
return this.getStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
const services = Array.isArray(servicesData) ? servicesData : (servicesData.services || []);
|
||||||
|
|
||||||
|
for (const service of services) {
|
||||||
|
const serviceId = service.id || service.name?.toLowerCase();
|
||||||
|
if (!serviceId) continue;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const url = resolveServiceUrl(serviceId, service, this.ctx.siteConfig, this.ctx.buildServiceUrl);
|
||||||
|
if (!url) continue;
|
||||||
|
|
||||||
|
const parsed = new URL(url);
|
||||||
|
if (parsed.protocol !== 'https:') continue;
|
||||||
|
|
||||||
|
const hostname = parsed.hostname;
|
||||||
|
const port = parseInt(parsed.port) || 443;
|
||||||
|
|
||||||
|
// Map hostname back to service ID
|
||||||
|
this.hostnameToServiceId.set(hostname, serviceId);
|
||||||
|
|
||||||
|
const result = await this.checkCert(hostname, port);
|
||||||
|
|
||||||
|
// Store result
|
||||||
|
this.certStatus.set(hostname, result);
|
||||||
|
|
||||||
|
// Emit check event
|
||||||
|
this.emit('cert-check', { serviceId, hostname, result });
|
||||||
|
|
||||||
|
// Check alert thresholds
|
||||||
|
await this._checkAndNotify(hostname, result, serviceId);
|
||||||
|
} catch (err) {
|
||||||
|
this.log.warn('ssl-monitor', `Failed to check cert for service ${serviceId}`, {
|
||||||
|
error: err.message
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Persist results
|
||||||
|
await this._saveCache();
|
||||||
|
|
||||||
|
return this.getStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start periodic SSL certificate checking.
|
||||||
|
*
|
||||||
|
* @param {number} [intervalMs=3600000] - Check interval in milliseconds
|
||||||
|
*/
|
||||||
|
start(intervalMs) {
|
||||||
|
if (intervalMs !== undefined) {
|
||||||
|
this.config.intervalMs = intervalMs;
|
||||||
|
}
|
||||||
|
if (this.intervalHandle) {
|
||||||
|
this.log.warn('ssl-monitor', 'SSL monitor is already running');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.config.enabled = true;
|
||||||
|
|
||||||
|
// Load cached data
|
||||||
|
this._loadCache().catch(err => {
|
||||||
|
this.log.warn('ssl-monitor', 'Failed to load SSL cache', { error: err.message });
|
||||||
|
});
|
||||||
|
|
||||||
|
// Initial check (non-blocking)
|
||||||
|
this.checkAll().catch(err => {
|
||||||
|
this.log.error('ssl-monitor', 'Initial SSL check failed', { error: err.message });
|
||||||
|
});
|
||||||
|
|
||||||
|
// Schedule periodic checks
|
||||||
|
this.intervalHandle = setInterval(() => {
|
||||||
|
this.checkAll().catch(err => {
|
||||||
|
this.log.error('ssl-monitor', 'Periodic SSL check failed', { error: err.message });
|
||||||
|
});
|
||||||
|
}, this.config.intervalMs);
|
||||||
|
|
||||||
|
this.log.info('ssl-monitor', 'SSL monitoring started', {
|
||||||
|
intervalMs: this.config.intervalMs
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop periodic SSL certificate checking.
|
||||||
|
*/
|
||||||
|
stop() {
|
||||||
|
if (this.intervalHandle) {
|
||||||
|
clearInterval(this.intervalHandle);
|
||||||
|
this.intervalHandle = null;
|
||||||
|
}
|
||||||
|
this.config.enabled = false;
|
||||||
|
this.log.info('ssl-monitor', 'SSL monitoring stopped');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the current SSL certificate status for all checked hostnames.
|
||||||
|
*
|
||||||
|
* @returns {Object} Map of hostname → cert status
|
||||||
|
*/
|
||||||
|
getStatus() {
|
||||||
|
const status = {};
|
||||||
|
for (const [hostname, cert] of this.certStatus.entries()) {
|
||||||
|
status[hostname] = { ...cert };
|
||||||
|
}
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the SSL certificate status for a specific service.
|
||||||
|
*
|
||||||
|
* @param {string} serviceId - The service ID to look up
|
||||||
|
* @returns {Object|null} Certificate status or null if not found
|
||||||
|
*/
|
||||||
|
getServiceCertStatus(serviceId) {
|
||||||
|
// Find hostname mapped to this service
|
||||||
|
for (const [hostname, id] of this.hostnameToServiceId.entries()) {
|
||||||
|
if (id === serviceId) {
|
||||||
|
const cert = this.certStatus.get(hostname);
|
||||||
|
return cert ? { ...cert, serviceId } : null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current monitoring configuration.
|
||||||
|
*
|
||||||
|
* @returns {Object} Config with interval and enabled state
|
||||||
|
*/
|
||||||
|
getConfig() {
|
||||||
|
return { ...this.config };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update monitoring configuration.
|
||||||
|
*
|
||||||
|
* @param {Object} updates - Config updates
|
||||||
|
* @param {boolean} [updates.enabled] - Enable/disable monitoring
|
||||||
|
* @param {number} [updates.intervalMs] - Check interval in milliseconds
|
||||||
|
*/
|
||||||
|
updateConfig(updates) {
|
||||||
|
if (typeof updates.enabled === 'boolean') {
|
||||||
|
this.config.enabled = updates.enabled;
|
||||||
|
if (!updates.enabled && this.intervalHandle) {
|
||||||
|
this.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (typeof updates.intervalMs === 'number' && updates.intervalMs >= 60000) {
|
||||||
|
this.config.intervalMs = updates.intervalMs;
|
||||||
|
// Restart interval if running
|
||||||
|
if (this.intervalHandle) {
|
||||||
|
clearInterval(this.intervalHandle);
|
||||||
|
this.intervalHandle = setInterval(() => {
|
||||||
|
this.checkAll().catch(err => {
|
||||||
|
this.log.error('ssl-monitor', 'Periodic SSL check failed', { error: err.message });
|
||||||
|
});
|
||||||
|
}, this.config.intervalMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== Private Methods =====
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check alert thresholds and send notifications if thresholds are crossed.
|
||||||
|
* Only sends one notification per threshold per hostname.
|
||||||
|
*
|
||||||
|
* @param {string} hostname
|
||||||
|
* @param {Object} certResult
|
||||||
|
* @param {string} serviceId
|
||||||
|
*/
|
||||||
|
async _checkAndNotify(hostname, certResult, serviceId) {
|
||||||
|
const { daysRemaining } = certResult;
|
||||||
|
const key = hostname;
|
||||||
|
const lastNotified = this.notifiedThresholds.get(key) || Infinity;
|
||||||
|
|
||||||
|
let level = null;
|
||||||
|
let eventType = null;
|
||||||
|
let message = null;
|
||||||
|
|
||||||
|
if (daysRemaining <= THRESHOLDS.CRITICAL) {
|
||||||
|
level = THRESHOLDS.CRITICAL;
|
||||||
|
eventType = 'cert-critical';
|
||||||
|
message = `🔒 CRITICAL: SSL certificate for ${hostname} expires in ${daysRemaining} days!`;
|
||||||
|
} else if (daysRemaining <= THRESHOLDS.URGENT) {
|
||||||
|
level = THRESHOLDS.URGENT;
|
||||||
|
eventType = 'cert-expiring';
|
||||||
|
message = `⚠️ URGENT: SSL certificate for ${hostname} expires in ${daysRemaining} days`;
|
||||||
|
} else if (daysRemaining <= THRESHOLDS.WARNING) {
|
||||||
|
level = THRESHOLDS.WARNING;
|
||||||
|
eventType = 'cert-expiring';
|
||||||
|
message = `⚠️ SSL certificate for ${hostname} expires in ${daysRemaining} days`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (level !== null && level < lastNotified) {
|
||||||
|
// New threshold crossed — send notification
|
||||||
|
this.notifiedThresholds.set(key, level);
|
||||||
|
this.emit(eventType, { hostname, serviceId, daysRemaining, level });
|
||||||
|
|
||||||
|
if (this.ctx.notification) {
|
||||||
|
try {
|
||||||
|
await this.ctx.notification.send('ssl-cert-expiry', {
|
||||||
|
text: message,
|
||||||
|
hostname,
|
||||||
|
serviceId,
|
||||||
|
daysRemaining,
|
||||||
|
level,
|
||||||
|
validTo: certResult.validTo
|
||||||
|
}, level <= THRESHOLDS.CRITICAL ? 'error' : 'warning');
|
||||||
|
} catch (err) {
|
||||||
|
this.log.error('ssl-monitor', 'Failed to send SSL notification', { error: err.message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (level === null) {
|
||||||
|
// Cert is healthy — reset notification tracking
|
||||||
|
this.notifiedThresholds.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Persist cert status cache to disk.
|
||||||
|
*/
|
||||||
|
async _saveCache() {
|
||||||
|
try {
|
||||||
|
const data = {
|
||||||
|
lastChecked: new Date().toISOString(),
|
||||||
|
certs: {},
|
||||||
|
hostnameToServiceId: Object.fromEntries(this.hostnameToServiceId)
|
||||||
|
};
|
||||||
|
for (const [hostname, cert] of this.certStatus.entries()) {
|
||||||
|
data.certs[hostname] = cert;
|
||||||
|
}
|
||||||
|
await writeJsonFile(this.cacheFile, data);
|
||||||
|
} catch (err) {
|
||||||
|
this.log.warn('ssl-monitor', 'Failed to save SSL cache', { error: err.message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load cert status cache from disk.
|
||||||
|
*/
|
||||||
|
async _loadCache() {
|
||||||
|
try {
|
||||||
|
const data = await readJsonFile(this.cacheFile, null);
|
||||||
|
if (data && data.certs) {
|
||||||
|
for (const [hostname, cert] of Object.entries(data.certs)) {
|
||||||
|
this.certStatus.set(hostname, cert);
|
||||||
|
}
|
||||||
|
if (data.hostnameToServiceId) {
|
||||||
|
for (const [hostname, serviceId] of Object.entries(data.hostnameToServiceId)) {
|
||||||
|
this.hostnameToServiceId.set(hostname, serviceId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.log.info('ssl-monitor', 'Loaded SSL cache', {
|
||||||
|
certCount: this.certStatus.size
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
this.log.warn('ssl-monitor', 'Failed to load SSL cache', { error: err.message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = SSLMonitor;
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# =============================================================================
|
||||||
|
# DashCaddy Gitea — Off-host backup to Dropbox
|
||||||
|
# =============================================================================
|
||||||
|
# - Stops gitea container briefly to ensure SQLite DB consistency
|
||||||
|
# - Syncs /var/lib/docker/volumes/gitea-data to dropbox:/Apps/dashcaddy-gitea-backups/<date>/
|
||||||
|
# - Date-stamped snapshots (one per day), kept for 7 days locally
|
||||||
|
# - Restarts gitea even if sync fails
|
||||||
|
# - Logs to /var/log/gitea-backup.log
|
||||||
|
# =============================================================================
|
||||||
|
set -u # don't use -e: we want to always restart gitea
|
||||||
|
|
||||||
|
LOG=/var/log/gitea-backup.log
|
||||||
|
DATA_SRC=/var/lib/docker/volumes/gitea-data/_data
|
||||||
|
DEST="dropbox:/Apps/dashcaddy-gitea-backups"
|
||||||
|
TODAY=$(date -u +%Y-%m-%d)
|
||||||
|
BACKUP_PATH="${DEST}/${TODAY}"
|
||||||
|
RETENTION_DAYS=7
|
||||||
|
|
||||||
|
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG"; }
|
||||||
|
|
||||||
|
log "=== Backup start ==="
|
||||||
|
|
||||||
|
# 0. Sanity checks
|
||||||
|
if [ ! -d "$DATA_SRC" ]; then
|
||||||
|
log "ERROR: data dir $DATA_SRC missing"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 1. Stop gitea to flush SQLite
|
||||||
|
log "Stopping gitea container..."
|
||||||
|
docker stop gitea >> "$LOG" 2>&1
|
||||||
|
STOP_RC=$?
|
||||||
|
if [ $STOP_RC -ne 0 ]; then
|
||||||
|
log "WARNING: docker stop returned $STOP_RC — container may not be running"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 2. Sync (use copy so source files are preserved as-is, no --delete)
|
||||||
|
log "Syncing $DATA_SRC -> $BACKUP_PATH"
|
||||||
|
rclone copy "$DATA_SRC" "$BACKUP_PATH" \
|
||||||
|
--transfers 4 \
|
||||||
|
--checkers 8 \
|
||||||
|
--retries 3 \
|
||||||
|
--low-level-retries 10 \
|
||||||
|
--stats 30s \
|
||||||
|
--log-file "$LOG" \
|
||||||
|
--log-level INFO
|
||||||
|
SYNC_RC=$?
|
||||||
|
|
||||||
|
# 3. Always restart gitea
|
||||||
|
log "Starting gitea container..."
|
||||||
|
docker start gitea >> "$LOG" 2>&1
|
||||||
|
START_RC=$?
|
||||||
|
|
||||||
|
# Wait for gitea to be ready
|
||||||
|
for i in {1..30}; do
|
||||||
|
if curl -sf http://localhost:3000/api/v1/version > /dev/null 2>&1; then
|
||||||
|
log "Gitea is up after ${i}s"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
# 4. Cleanup old backups (older than RETENTION_DAYS)
|
||||||
|
log "Pruning local + remote snapshots older than ${RETENTION_DAYS} days..."
|
||||||
|
CUTOFF=$(date -u -d "${RETENTION_DAYS} days ago" +%Y-%m-%d)
|
||||||
|
rclone lsf "$DEST/" --dirs-only 2>/dev/null | while read -r d; do
|
||||||
|
# rclone returns names with trailing /
|
||||||
|
name="${d%/}"
|
||||||
|
if [[ "$name" < "$CUTOFF" ]]; then
|
||||||
|
log " removing old: $name"
|
||||||
|
rclone purge "${DEST}/${name}" >> "$LOG" 2>&1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# 5. Report
|
||||||
|
if [ $SYNC_RC -eq 0 ] && [ $START_RC -eq 0 ]; then
|
||||||
|
log "=== Backup OK ==="
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
log "=== Backup completed with errors (sync=$SYNC_RC, start=$START_RC) ==="
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
Executable
+76
@@ -0,0 +1,76 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Samihost fail2ban watchdog — auto-unban whitelisted IPs and keep ignoreip list in sync.
|
||||||
|
# Deployed to /usr/local/bin/samihost-fail2ban-watchdog.sh on 194.163.161.162
|
||||||
|
# Cron: every 30 min (0,30 * * * *)
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
JAIL_LOCAL=/etc/fail2ban/jail.local
|
||||||
|
BACKUP=/etc/fail2ban/jail.local.watchdog.bak
|
||||||
|
EXPECTED_IGNOREIP="127.0.0.1/8 ::1 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7 fe80::/10 100.64.0.0/10 100.121.150.22 100.85.236.10 100.71.97.12 100.81.59.99 100.98.123.59 194.233.88.206 173.212.201.200 194.163.161.162"
|
||||||
|
LOG=/var/log/samihost-fail2ban-watchdog.log
|
||||||
|
TELEGRAM_LOG=/tmp/fail2ban-watchdog-last-action
|
||||||
|
|
||||||
|
ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
|
||||||
|
log() { echo "$(ts) $*" | tee -a "$LOG"; }
|
||||||
|
|
||||||
|
mkdir -p "$(dirname "$LOG")"
|
||||||
|
touch "$LOG"
|
||||||
|
|
||||||
|
# --- 1. Verify ignoreip line is intact and matches expected ---
|
||||||
|
CURRENT=$(grep '^ignoreip' "$JAIL_LOCAL" | sed 's/^ignoreip[[:space:]]*=[[:space:]]*//' || true)
|
||||||
|
EXPECTED_NORMALIZED=$(echo "$EXPECTED_IGNOREIP" | tr ' ' '\n' | sort -u | tr '\n' ' ' | sed 's/ $//')
|
||||||
|
CURRENT_NORMALIZED=$(echo "$CURRENT" | tr ' ' '\n' | sort -u | tr '\n' ' ' | sed 's/ $//')
|
||||||
|
|
||||||
|
if [ "$CURRENT_NORMALIZED" != "$EXPECTED_NORMALIZED" ]; then
|
||||||
|
log "ALERT: ignoreip line drifted. Restoring."
|
||||||
|
cp "$JAIL_LOCAL" "$BACKUP"
|
||||||
|
sed -i "s|^ignoreip = .*|ignoreip = $EXPECTED_IGNOREIP|" "$JAIL_LOCAL"
|
||||||
|
fail2ban-client reload
|
||||||
|
echo "ignoreip restored at $(ts)" > "$TELEGRAM_LOG"
|
||||||
|
log "ignoreip restored, fail2ban reloaded"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 2. Unban any currently-banned IPs that match our trusted set ---
|
||||||
|
BANNED=$(fail2ban-client status sshd 2>/dev/null | awk -F: '/Banned IP list/{print $2}' | tr ' ' '\n' | grep -v '^$' || true)
|
||||||
|
UNBANNED=0
|
||||||
|
for ip in $BANNED; do
|
||||||
|
# Match against any trusted network
|
||||||
|
is_trusted=0
|
||||||
|
for net in 127.0.0.0/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 100.64.0.0/10 ::1 fc00::/7 fe80::/10 100.121.150.22 100.85.236.10 100.71.97.12 100.81.59.99 100.98.123.59 194.233.88.206 173.212.201.200 194.163.161.162; do
|
||||||
|
if [[ "$net" == *"/"* ]]; then
|
||||||
|
# CIDR match (simple IPv4 only — IPv6 needs python or ipcalc, skip for now)
|
||||||
|
base="${net%/*}"
|
||||||
|
mask="${net#*/}"
|
||||||
|
if [[ "$ip" == "$base"* ]] || python3 -c "import ipaddress,sys; sys.exit(0 if ipaddress.ip_address('$ip') in ipaddress.ip_network('$net', strict=False) else 1)" 2>/dev/null; then
|
||||||
|
is_trusted=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if [ "$ip" = "$net" ]; then
|
||||||
|
is_trusted=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$is_trusted" = "1" ]; then
|
||||||
|
if fail2ban-client set sshd unbanip "$ip" >/dev/null 2>&1; then
|
||||||
|
log "auto-unbanned trusted IP: $ip"
|
||||||
|
UNBANNED=$((UNBANNED+1))
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
[ "$UNBANNED" -gt 0 ] && echo "auto-unbanned $UNBANNED trusted IPs at $(ts)" > "$TELEGRAM_LOG"
|
||||||
|
|
||||||
|
# --- 3. Cap the ban count — if more than 200 are banned, mass-unban stale ones ---
|
||||||
|
TOTAL_BANNED=$(fail2ban-client status sshd 2>/dev/null | awk '/Currently banned/{print $NF}' || echo 0)
|
||||||
|
if [ "$TOTAL_BANNED" -gt 200 ]; then
|
||||||
|
log "ALERT: $TOTAL_BANNED IPs banned. Mass-unbanning all."
|
||||||
|
for ip in $BANNED; do
|
||||||
|
fail2ban-client set sshd unbanip "$ip" >/dev/null 2>&1 || true
|
||||||
|
done
|
||||||
|
echo "mass-unbanned $TOTAL_BANNED stale bans at $(ts)" > "$TELEGRAM_LOG"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "watchdog run complete (unbanned=$UNBANNED, total_banned=$TOTAL_BANNED)"
|
||||||
@@ -72,6 +72,7 @@ const bundles = {
|
|||||||
],
|
],
|
||||||
'init.js': [
|
'init.js': [
|
||||||
JS('core', 'init.js'),
|
JS('core', 'init.js'),
|
||||||
|
JS('monitoring-widgets.js'),
|
||||||
JS('keyboard-shortcuts.js'),
|
JS('keyboard-shortcuts.js'),
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|||||||
Vendored
+114
-87
File diff suppressed because one or more lines are too long
Vendored
+307
-232
File diff suppressed because one or more lines are too long
Vendored
+129
-18
File diff suppressed because one or more lines are too long
@@ -256,6 +256,9 @@
|
|||||||
<option value="on">🟢 Online</option>
|
<option value="on">🟢 Online</option>
|
||||||
<option value="off">🔴 Offline</option>
|
<option value="off">🔴 Offline</option>
|
||||||
</select>
|
</select>
|
||||||
|
<select id="service-filter-category" style="padding: 8px 12px; background: var(--bg); border: 1px solid var(--border); border-radius: 6px; color: var(--fg); font-size: 0.9rem;">
|
||||||
|
<option value="all">All Categories</option>
|
||||||
|
</select>
|
||||||
<button id="batch-operations-btn" class="btn-sm" style="padding: 8px 12px;">☰ Batch Operations</button>
|
<button id="batch-operations-btn" class="btn-sm" style="padding: 8px 12px;">☰ Batch Operations</button>
|
||||||
<span id="service-filter-count" style="color: var(--muted); font-size: 0.85rem; white-space: nowrap;"></span>
|
<span id="service-filter-count" style="color: var(--muted); font-size: 0.85rem; white-space: nowrap;"></span>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -41,8 +41,11 @@
|
|||||||
dismissedUpdates = new Set();
|
dismissedUpdates = new Set();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Track global update state for cross-component access
|
||||||
|
let knownUpdates = [];
|
||||||
|
|
||||||
// Fetch update data and show badges
|
// Fetch update data and show badges
|
||||||
async function refreshCardUpdates() {
|
async function refreshCardUpdates(notifyNew) {
|
||||||
try {
|
try {
|
||||||
const res = await fetch('/api/v1/updates/available');
|
const res = await fetch('/api/v1/updates/available');
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
@@ -51,9 +54,21 @@
|
|||||||
// Clear all update badges first
|
// Clear all update badges first
|
||||||
document.querySelectorAll('.update-available-badge').forEach(el => el.classList.remove('visible'));
|
document.querySelectorAll('.update-available-badge').forEach(el => el.classList.remove('visible'));
|
||||||
|
|
||||||
if (!data.updates?.length) return;
|
const updates = data.updates || [];
|
||||||
|
knownUpdates = updates; // store globally
|
||||||
|
|
||||||
for (const upd of data.updates) {
|
// Notify if new updates appeared (periodic check with notification)
|
||||||
|
if (notifyNew && updates.length > 0) {
|
||||||
|
const prev = window._lastKnownUpdateCount || 0;
|
||||||
|
if (prev > 0 && updates.length > prev) {
|
||||||
|
showNotification(`${updates.length} container update(s) available — click Update Management to review.`, 'info');
|
||||||
|
}
|
||||||
|
window._lastKnownUpdateCount = updates.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!updates.length) return;
|
||||||
|
|
||||||
|
for (const upd of updates) {
|
||||||
// Try to match by container name to service id
|
// Try to match by container name to service id
|
||||||
const apps = window.APPS || [];
|
const apps = window.APPS || [];
|
||||||
for (const app of apps) {
|
for (const app of apps) {
|
||||||
@@ -61,17 +76,24 @@
|
|||||||
// Skip dismissed updates
|
// Skip dismissed updates
|
||||||
if (dismissedUpdates.has(app.id)) break;
|
if (dismissedUpdates.has(app.id)) break;
|
||||||
const badge = document.getElementById('update-badge-' + app.id);
|
const badge = document.getElementById('update-badge-' + app.id);
|
||||||
|
const updateBtn = document.getElementById('update-btn-' + app.id);
|
||||||
if (badge) {
|
if (badge) {
|
||||||
badge.classList.add('visible');
|
badge.classList.add('visible');
|
||||||
badge.title = `Image digest changed. Click to dismiss if already up to date.\n${upd.imageName || ''}`;
|
badge.title = `Update available — click to open Update Management.`;
|
||||||
badge.style.cursor = 'pointer';
|
badge.style.cursor = 'pointer';
|
||||||
badge.onclick = (e) => {
|
badge.onclick = (e) => {
|
||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
badge.classList.remove('visible');
|
// Open Update Management modal focused on this app
|
||||||
dismissedUpdates.add(app.id);
|
if (window.openUpdateModal) window.openUpdateModal(app.id);
|
||||||
safeSessionSet('dismissed-updates', JSON.stringify([...dismissedUpdates]));
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
// Highlight update button if update is available
|
||||||
|
if (updateBtn) {
|
||||||
|
updateBtn.style.background = '#f97316';
|
||||||
|
updateBtn.style.borderColor = '#f97316';
|
||||||
|
updateBtn.style.boxShadow = '0 0 6px #f9731688';
|
||||||
|
updateBtn.title = `Update available — click to open Update Management.`;
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -90,10 +112,10 @@
|
|||||||
refreshCardUpdates();
|
refreshCardUpdates();
|
||||||
}, 5000);
|
}, 5000);
|
||||||
|
|
||||||
// Periodic refresh every 60 seconds
|
// Periodic refresh every 60 seconds — notify on new updates detected
|
||||||
setInterval(() => {
|
setInterval(() => {
|
||||||
refreshCardHealth();
|
refreshCardHealth();
|
||||||
refreshCardUpdates();
|
refreshCardUpdates(true); // true = notify if new updates found
|
||||||
}, 60000);
|
}, 60000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -95,6 +95,8 @@
|
|||||||
const card = el('div', 'card');
|
const card = el('div', 'card');
|
||||||
card.setAttribute('data-app', s.id);
|
card.setAttribute('data-app', s.id);
|
||||||
card.setAttribute('data-status', 'off'); // Initial status
|
card.setAttribute('data-status', 'off'); // Initial status
|
||||||
|
if (s.containerId) card.setAttribute('data-container-id', s.containerId);
|
||||||
|
if (s.category) card.setAttribute('data-category', s.category);
|
||||||
if (s.recipeId) card.setAttribute('data-recipe-id', s.recipeId);
|
if (s.recipeId) card.setAttribute('data-recipe-id', s.recipeId);
|
||||||
|
|
||||||
const dot = el('span', 'dot bad at-bl'); dot.id = 'dot-' + s.id + '-grid'; card.appendChild(dot);
|
const dot = el('span', 'dot bad at-bl'); dot.id = 'dot-' + s.id + '-grid'; card.appendChild(dot);
|
||||||
@@ -156,6 +158,16 @@
|
|||||||
nameSpan.appendChild(tsBadge);
|
nameSpan.appendChild(tsBadge);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add Category badge if service has one (colored pill with icon)
|
||||||
|
if (s.category) {
|
||||||
|
const cats = (typeof DC !== 'undefined' && DC.CATEGORIES) || window.DC_CATEGORIES || {};
|
||||||
|
const catInfo = cats[s.category] || {};
|
||||||
|
const catBadge = el('span', 'cat-badge', `${catInfo.icon || ''} ${s.category}`.trim());
|
||||||
|
catBadge.title = `Category: ${s.category}`;
|
||||||
|
catBadge.style.cssText = `margin-left: 6px; font-size: 0.65rem; padding: 1px 6px; border-radius: 999px; background: color-mix(in srgb, ${catInfo.color || '#7f8c8d'} 25%, transparent); color: ${catInfo.color || '#7f8c8d'}; border: 1px solid color-mix(in srgb, ${catInfo.color || '#7f8c8d'} 50%, transparent); white-space: nowrap; font-weight: 500;`;
|
||||||
|
nameSpan.appendChild(catBadge);
|
||||||
|
}
|
||||||
|
|
||||||
row.appendChild(el('span', 'spacer'));
|
row.appendChild(el('span', 'spacer'));
|
||||||
|
|
||||||
const pill = el('span', 'badge off', 'OFF'); pill.id = 'badge-' + s.id; row.appendChild(pill);
|
const pill = el('span', 'badge off', 'OFF'); pill.id = 'badge-' + s.id; row.appendChild(pill);
|
||||||
@@ -282,6 +294,9 @@
|
|||||||
|
|
||||||
// Group recipe cards visually after grid is built
|
// Group recipe cards visually after grid is built
|
||||||
if (window.groupRecipeCards) requestAnimationFrame(() => window.groupRecipeCards());
|
if (window.groupRecipeCards) requestAnimationFrame(() => window.groupRecipeCards());
|
||||||
|
|
||||||
|
// Refresh the service filter so the category dropdown reflects new services
|
||||||
|
if (window.refreshServiceFilter) window.refreshServiceFilter();
|
||||||
}
|
}
|
||||||
|
|
||||||
function setBadge(id, up, responseTime = null) {
|
function setBadge(id, up, responseTime = null) {
|
||||||
|
|||||||
@@ -59,11 +59,13 @@
|
|||||||
}
|
}
|
||||||
_dashboardInitialized = true;
|
_dashboardInitialized = true;
|
||||||
await window.loadServices();
|
await window.loadServices();
|
||||||
|
await loadTemplateCategories();
|
||||||
window.buildGrid();
|
window.buildGrid();
|
||||||
animateTopCards();
|
animateTopCards();
|
||||||
window.refreshAll();
|
window.refreshAll();
|
||||||
setInterval(window.refreshAll, DC.POLL.DASHBOARD);
|
setInterval(window.refreshAll, DC.POLL.DASHBOARD);
|
||||||
if (typeof window.refreshCredsButtons === 'function') window.refreshCredsButtons();
|
if (typeof window.refreshCredsButtons === 'function') window.refreshCredsButtons();
|
||||||
|
if (typeof window.refreshMonitoringWidgets === 'function') window.refreshMonitoringWidgets();
|
||||||
// Update auth card (may have already been updated by the auto-load IIFE but ensure it's correct)
|
// Update auth card (may have already been updated by the auto-load IIFE but ensure it's correct)
|
||||||
if (typeof window._updateAuthCard === 'function') {
|
if (typeof window._updateAuthCard === 'function') {
|
||||||
try {
|
try {
|
||||||
@@ -200,6 +202,55 @@
|
|||||||
window.loadCustomServices = loadCustomServices;
|
window.loadCustomServices = loadCustomServices;
|
||||||
registerServiceWorker();
|
registerServiceWorker();
|
||||||
|
|
||||||
|
// ===== TEMPLATE CATEGORIES =====
|
||||||
|
// Cached template categories from /api/v1/templates for use across the UI
|
||||||
|
// (service create/edit, filter dropdown, category badges, etc.)
|
||||||
|
async function loadTemplateCategories() {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/v1/templates', { cache: 'no-store' });
|
||||||
|
if (!r.ok) return;
|
||||||
|
const data = await r.json();
|
||||||
|
if (data && data.categories) {
|
||||||
|
window.DC_CATEGORIES = data.categories;
|
||||||
|
// Also expose via globals.js constant for convenience
|
||||||
|
if (typeof DC !== 'undefined') DC.CATEGORIES = data.categories;
|
||||||
|
// Populate any category <select> that's already in the DOM
|
||||||
|
populateCategorySelects();
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.warn('[init] Failed to load template categories:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function populateCategorySelects() {
|
||||||
|
const cats = window.DC_CATEGORIES || (typeof DC !== 'undefined' && DC.CATEGORIES);
|
||||||
|
if (!cats) return;
|
||||||
|
document.querySelectorAll('select[data-role="service-category"]').forEach(select => {
|
||||||
|
const current = select.dataset.current || '';
|
||||||
|
// Clear options but keep the first (placeholder)
|
||||||
|
const placeholder = select.querySelector('option[value=""]');
|
||||||
|
select.innerHTML = '';
|
||||||
|
if (placeholder) select.appendChild(placeholder);
|
||||||
|
else {
|
||||||
|
const ph = document.createElement('option');
|
||||||
|
ph.value = '';
|
||||||
|
ph.textContent = '— Select category —';
|
||||||
|
select.appendChild(ph);
|
||||||
|
}
|
||||||
|
Object.entries(cats).forEach(([name, info]) => {
|
||||||
|
const opt = document.createElement('option');
|
||||||
|
opt.value = name;
|
||||||
|
opt.textContent = `${info.icon || ''} ${name}`.trim();
|
||||||
|
if (name === current) opt.selected = true;
|
||||||
|
select.appendChild(opt);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Allow other modules to re-run population after they (re)inject selects
|
||||||
|
window.populateCategorySelects = populateCategorySelects;
|
||||||
|
window.loadTemplateCategories = loadTemplateCategories;
|
||||||
|
|
||||||
// TOTP-gated initialization
|
// TOTP-gated initialization
|
||||||
(async () => {
|
(async () => {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -262,6 +262,7 @@
|
|||||||
const proxyIp = document.getElementById('external-proxy-ip').value.trim() || SITE.dnsIp || 'localhost';
|
const proxyIp = document.getElementById('external-proxy-ip').value.trim() || SITE.dnsIp || 'localhost';
|
||||||
const preserveHost = document.getElementById('external-preserve-host').checked;
|
const preserveHost = document.getElementById('external-preserve-host').checked;
|
||||||
const followRedirects = document.getElementById('external-follow-redirects').checked;
|
const followRedirects = document.getElementById('external-follow-redirects').checked;
|
||||||
|
const category = document.getElementById('external-service-category')?.value || '';
|
||||||
|
|
||||||
if (!name || !externalUrl) {
|
if (!name || !externalUrl) {
|
||||||
showNotification('Please fill in Name and External URL', 'warning');
|
showNotification('Please fill in Name and External URL', 'warning');
|
||||||
@@ -341,6 +342,8 @@
|
|||||||
isExternal: true,
|
isExternal: true,
|
||||||
isCustom: true
|
isCustom: true
|
||||||
};
|
};
|
||||||
|
// Only attach category if user actually picked one
|
||||||
|
if (category) newService.category = category;
|
||||||
|
|
||||||
window.APPS.push(newService);
|
window.APPS.push(newService);
|
||||||
results.dashboard = true;
|
results.dashboard = true;
|
||||||
@@ -457,6 +460,13 @@
|
|||||||
const healthCheck = document.getElementById('health-check-input')?.value || '';
|
const healthCheck = document.getElementById('health-check-input')?.value || '';
|
||||||
const timeout = document.getElementById('timeout-input')?.value || 30;
|
const timeout = document.getElementById('timeout-input')?.value || 30;
|
||||||
|
|
||||||
|
// Category is optional — pulled from either local or external select by the
|
||||||
|
// openAddServiceModal reset. If user doesn't choose one, it stays undefined
|
||||||
|
// and we don't send it (so the backend keeps the existing behavior).
|
||||||
|
const categoryEl = document.getElementById('service-category-input')
|
||||||
|
|| document.getElementById('external-service-category');
|
||||||
|
const category = categoryEl?.value || '';
|
||||||
|
|
||||||
const dnsToken = window.getToken(getPrimaryDnsId(), 'admin');
|
const dnsToken = window.getToken(getPrimaryDnsId(), 'admin');
|
||||||
|
|
||||||
if (!name || !port || !ip) {
|
if (!name || !port || !ip) {
|
||||||
@@ -525,6 +535,8 @@
|
|||||||
logo: logo || `/assets/${subdomain}.png`,
|
logo: logo || `/assets/${subdomain}.png`,
|
||||||
tailscaleOnly: tailscaleOnly || false
|
tailscaleOnly: tailscaleOnly || false
|
||||||
};
|
};
|
||||||
|
// Only include category if user actually picked one
|
||||||
|
if (category) serviceConfig.category = category;
|
||||||
|
|
||||||
await window.addServiceToConfig(serviceConfig);
|
await window.addServiceToConfig(serviceConfig);
|
||||||
results.dashboard = true;
|
results.dashboard = true;
|
||||||
|
|||||||
@@ -19,6 +19,16 @@
|
|||||||
document.getElementById('edit-tailscale-only').checked = service.tailscaleOnly || false;
|
document.getElementById('edit-tailscale-only').checked = service.tailscaleOnly || false;
|
||||||
document.getElementById('edit-logo-url').value = service.logo || '';
|
document.getElementById('edit-logo-url').value = service.logo || '';
|
||||||
|
|
||||||
|
// Populate the category select for this service, then set the current value.
|
||||||
|
// populateCategorySelects() uses data-current so we set it first, then call.
|
||||||
|
const categorySelect = document.getElementById('edit-service-category');
|
||||||
|
if (categorySelect) {
|
||||||
|
categorySelect.dataset.current = service.category || '';
|
||||||
|
if (typeof window.populateCategorySelects === 'function') {
|
||||||
|
window.populateCategorySelects();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
modal.classList.add('show');
|
modal.classList.add('show');
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,6 +46,7 @@
|
|||||||
const newIp = document.getElementById('edit-ip').value.trim() || 'localhost';
|
const newIp = document.getElementById('edit-ip').value.trim() || 'localhost';
|
||||||
const tailscaleOnly = document.getElementById('edit-tailscale-only').checked;
|
const tailscaleOnly = document.getElementById('edit-tailscale-only').checked;
|
||||||
const newLogo = document.getElementById('edit-logo-url').value.trim();
|
const newLogo = document.getElementById('edit-logo-url').value.trim();
|
||||||
|
const newCategory = document.getElementById('edit-service-category')?.value || '';
|
||||||
|
|
||||||
if (!newSubdomain) {
|
if (!newSubdomain) {
|
||||||
showNotification('Subdomain is required', 'warning');
|
showNotification('Subdomain is required', 'warning');
|
||||||
@@ -51,6 +62,7 @@
|
|||||||
if (newIp !== currentEditService.ip) changes.push('ip');
|
if (newIp !== currentEditService.ip) changes.push('ip');
|
||||||
if (tailscaleOnly !== (currentEditService.tailscaleOnly || false)) changes.push('tailscale');
|
if (tailscaleOnly !== (currentEditService.tailscaleOnly || false)) changes.push('tailscale');
|
||||||
if (newLogo && newLogo !== currentEditService.logo) changes.push('logo');
|
if (newLogo && newLogo !== currentEditService.logo) changes.push('logo');
|
||||||
|
if (newCategory !== (currentEditService.category || '')) changes.push('category');
|
||||||
|
|
||||||
if (changes.length === 0) {
|
if (changes.length === 0) {
|
||||||
closeServiceEditModal();
|
closeServiceEditModal();
|
||||||
@@ -72,7 +84,8 @@
|
|||||||
port: newPort || currentEditService.port,
|
port: newPort || currentEditService.port,
|
||||||
ip: newIp,
|
ip: newIp,
|
||||||
tailscaleOnly,
|
tailscaleOnly,
|
||||||
logo: newLogo || undefined
|
logo: newLogo || undefined,
|
||||||
|
category: newCategory
|
||||||
})
|
})
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -91,7 +104,8 @@
|
|||||||
port: newPort || window.APPS[appIndex].port,
|
port: newPort || window.APPS[appIndex].port,
|
||||||
ip: newIp,
|
ip: newIp,
|
||||||
tailscaleOnly,
|
tailscaleOnly,
|
||||||
logo: newLogo || window.APPS[appIndex].logo
|
logo: newLogo || window.APPS[appIndex].logo,
|
||||||
|
category: newCategory || undefined
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -187,6 +187,9 @@
|
|||||||
name: serviceConfig.name,
|
name: serviceConfig.name,
|
||||||
logo: serviceConfig.logo || `/assets/${serviceConfig.subdomain}.png`
|
logo: serviceConfig.logo || `/assets/${serviceConfig.subdomain}.png`
|
||||||
};
|
};
|
||||||
|
// Forward optional metadata fields if provided
|
||||||
|
if (serviceConfig.category) newService.category = serviceConfig.category;
|
||||||
|
if (serviceConfig.containerId) newService.containerId = serviceConfig.containerId;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await secureFetch('/api/v1/services', {
|
const response = await secureFetch('/api/v1/services', {
|
||||||
|
|||||||
@@ -82,6 +82,16 @@
|
|||||||
Enter a URL or upload an image file (PNG, JPG, SVG)
|
Enter a URL or upload an image file (PNG, JPG, SVG)
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Category -->
|
||||||
|
<div>
|
||||||
|
<label for="edit-service-category" class="form-label-accent-sm">
|
||||||
|
Category
|
||||||
|
</label>
|
||||||
|
<select id="edit-service-category" data-role="service-category" class="form-input-md">
|
||||||
|
<option value="">— No category —</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="weather-modal-buttons" style="margin-top: 24px;">
|
<div class="weather-modal-buttons" style="margin-top: 24px;">
|
||||||
@@ -239,6 +249,15 @@
|
|||||||
Reload Caddy after adding
|
Reload Caddy after adding
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<!-- Category -->
|
||||||
|
<div>
|
||||||
|
<label for="service-category-input" style="font-size: 0.8rem; color: var(--muted); margin-bottom: 4px; display: block;">Category</label>
|
||||||
|
<select id="service-category-input" data-role="service-category" style="width: 100%;">
|
||||||
|
<option value="">— No category —</option>
|
||||||
|
</select>
|
||||||
|
<div style="font-size: 0.7rem; color: var(--muted); margin-top: 3px;">Group services on the dashboard by purpose (Media, Productivity, etc.)</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<hr style="border: none; border-top: 1px solid var(--border); margin: 4px 0;" />
|
<hr style="border: none; border-top: 1px solid var(--border); margin: 4px 0;" />
|
||||||
|
|
||||||
<div class="grid-2col">
|
<div class="grid-2col">
|
||||||
@@ -326,6 +345,14 @@
|
|||||||
Follow Redirects
|
Follow Redirects
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<!-- Category (external) -->
|
||||||
|
<div>
|
||||||
|
<label for="external-service-category" style="font-size: 0.8rem; color: var(--muted); margin-bottom: 4px; display: block;">Category</label>
|
||||||
|
<select id="external-service-category" data-role="service-category" style="width: 100%;">
|
||||||
|
<option value="">— No category —</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -0,0 +1,304 @@
|
|||||||
|
// ========== MONITORING WIDGETS ==========
|
||||||
|
// Embeds a compact system-resource + health summary panel directly on the
|
||||||
|
// main dashboard. Replaces the need for a separate monitoring-dashboard.html
|
||||||
|
// page — quick at-a-glance stats where you already are.
|
||||||
|
(function () {
|
||||||
|
|
||||||
|
// ----- Style injection (scoped to .dc-monitor so it doesn't leak) -----
|
||||||
|
const styleEl = document.createElement('style');
|
||||||
|
styleEl.textContent = `
|
||||||
|
.dc-monitor {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
|
||||||
|
gap: 12px;
|
||||||
|
margin-bottom: 16px;
|
||||||
|
padding: 12px 16px;
|
||||||
|
background: var(--card-base);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
}
|
||||||
|
.dc-monitor-card {
|
||||||
|
padding: 10px 12px;
|
||||||
|
background: var(--card-bg, rgba(255,255,255,0.04));
|
||||||
|
border-radius: 8px;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
.dc-monitor-label {
|
||||||
|
font-size: 0.7rem;
|
||||||
|
color: var(--muted);
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: 0.5px;
|
||||||
|
margin-bottom: 4px;
|
||||||
|
}
|
||||||
|
.dc-monitor-value {
|
||||||
|
font-size: 1.4rem;
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--fg);
|
||||||
|
}
|
||||||
|
.dc-monitor-sub {
|
||||||
|
font-size: 0.7rem;
|
||||||
|
color: var(--muted);
|
||||||
|
margin-top: 4px;
|
||||||
|
}
|
||||||
|
.dc-monitor-bar {
|
||||||
|
margin-top: 6px;
|
||||||
|
width: 100%;
|
||||||
|
height: 4px;
|
||||||
|
background: color-mix(in srgb, var(--muted) 20%, transparent);
|
||||||
|
border-radius: 2px;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.dc-monitor-bar-fill {
|
||||||
|
height: 100%;
|
||||||
|
width: 0%;
|
||||||
|
background: var(--ok-fg, #27ae60);
|
||||||
|
transition: width 0.3s ease, background 0.3s ease;
|
||||||
|
}
|
||||||
|
.dc-monitor-bar-fill.warn { background: #f39c12; }
|
||||||
|
.dc-monitor-bar-fill.bad { background: #e74c3c; }
|
||||||
|
.dc-monitor-header {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
margin-bottom: 8px;
|
||||||
|
}
|
||||||
|
.dc-monitor-title {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
font-weight: 500;
|
||||||
|
color: var(--muted);
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
.dc-monitor-pill {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 4px;
|
||||||
|
padding: 2px 8px;
|
||||||
|
border-radius: 999px;
|
||||||
|
font-size: 0.7rem;
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
.dc-monitor-pill.ok { background: color-mix(in srgb, #27ae60 20%, transparent); color: #27ae60; }
|
||||||
|
.dc-monitor-pill.warn { background: color-mix(in srgb, #f39c12 20%, transparent); color: #f39c12; }
|
||||||
|
.dc-monitor-pill.bad { background: color-mix(in srgb, #e74c3c 20%, transparent); color: #e74c3c; }
|
||||||
|
.dc-monitor-refresh {
|
||||||
|
font-size: 0.7rem;
|
||||||
|
color: var(--muted);
|
||||||
|
opacity: 0.7;
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
document.head.appendChild(styleEl);
|
||||||
|
|
||||||
|
// ----- Container element (inserted above service-filter-bar) -----
|
||||||
|
const filterBar = document.getElementById('service-filter-bar');
|
||||||
|
if (!filterBar) return;
|
||||||
|
|
||||||
|
const panel = document.createElement('div');
|
||||||
|
panel.className = 'dc-monitor';
|
||||||
|
panel.id = 'dc-monitor-panel';
|
||||||
|
panel.innerHTML = `
|
||||||
|
<div class="dc-monitor-header" style="grid-column: 1 / -1;">
|
||||||
|
<div class="dc-monitor-title">📊 System Overview</div>
|
||||||
|
<span class="dc-monitor-refresh" id="dc-monitor-refresh-stamp">—</span>
|
||||||
|
</div>
|
||||||
|
<div class="dc-monitor-card">
|
||||||
|
<div class="dc-monitor-label">Services</div>
|
||||||
|
<div class="dc-monitor-value" id="dc-monitor-services">—</div>
|
||||||
|
<div class="dc-monitor-sub" id="dc-monitor-services-sub">loading…</div>
|
||||||
|
</div>
|
||||||
|
<div class="dc-monitor-card">
|
||||||
|
<div class="dc-monitor-label">Containers Up</div>
|
||||||
|
<div class="dc-monitor-value" id="dc-monitor-containers">—</div>
|
||||||
|
<div class="dc-monitor-sub" id="dc-monitor-containers-sub">loading…</div>
|
||||||
|
</div>
|
||||||
|
<div class="dc-monitor-card">
|
||||||
|
<div class="dc-monitor-label">Avg CPU</div>
|
||||||
|
<div class="dc-monitor-value" id="dc-monitor-cpu">—</div>
|
||||||
|
<div class="dc-monitor-bar"><div class="dc-monitor-bar-fill" id="dc-monitor-cpu-bar"></div></div>
|
||||||
|
</div>
|
||||||
|
<div class="dc-monitor-card">
|
||||||
|
<div class="dc-monitor-label">Avg Memory</div>
|
||||||
|
<div class="dc-monitor-value" id="dc-monitor-mem">—</div>
|
||||||
|
<div class="dc-monitor-bar"><div class="dc-monitor-bar-fill" id="dc-monitor-mem-bar"></div></div>
|
||||||
|
</div>
|
||||||
|
<div class="dc-monitor-card">
|
||||||
|
<div class="dc-monitor-label">Health</div>
|
||||||
|
<div class="dc-monitor-value" id="dc-monitor-health">—</div>
|
||||||
|
<div class="dc-monitor-sub" id="dc-monitor-health-sub">—</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
// Insert ABOVE the filter bar
|
||||||
|
filterBar.parentNode.insertBefore(panel, filterBar);
|
||||||
|
|
||||||
|
// ----- Helpers -----
|
||||||
|
function setBar(id, pct) {
|
||||||
|
const el = document.getElementById(id);
|
||||||
|
if (!el) return;
|
||||||
|
const p = Math.max(0, Math.min(100, Number(pct) || 0));
|
||||||
|
el.style.width = p + '%';
|
||||||
|
el.classList.remove('warn', 'bad');
|
||||||
|
if (p >= 85) el.classList.add('bad');
|
||||||
|
else if (p >= 65) el.classList.add('warn');
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtPct(v) {
|
||||||
|
if (v == null || isNaN(v)) return '—';
|
||||||
|
return (Math.round(v * 10) / 10) + '%';
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtBytes(b) {
|
||||||
|
if (b == null || isNaN(b)) return '—';
|
||||||
|
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||||
|
let i = 0;
|
||||||
|
while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
|
||||||
|
return b.toFixed(1) + ' ' + units[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
function setServicesCard() {
|
||||||
|
const total = (window.APPS || []).length;
|
||||||
|
let up = 0;
|
||||||
|
document.querySelectorAll('#cards .card').forEach(c => {
|
||||||
|
if (c.dataset.status === 'on') up++;
|
||||||
|
});
|
||||||
|
const el = document.getElementById('dc-monitor-services');
|
||||||
|
const sub = document.getElementById('dc-monitor-services-sub');
|
||||||
|
if (el) el.textContent = `${up} / ${total}`;
|
||||||
|
if (sub) sub.textContent = total === 0 ? 'no services yet' : `${up} online · ${total - up} offline`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyHealthSummary(data) {
|
||||||
|
const el = document.getElementById('dc-monitor-health');
|
||||||
|
const sub = document.getElementById('dc-monitor-health-sub');
|
||||||
|
if (!el) return;
|
||||||
|
if (!data || data.summary == null) {
|
||||||
|
el.textContent = '—';
|
||||||
|
if (sub) sub.textContent = 'no data';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const s = data.summary;
|
||||||
|
const healthy = s.healthy ?? s.up ?? 0;
|
||||||
|
const unhealthy = s.unhealthy ?? s.down ?? 0;
|
||||||
|
const total = s.total ?? (healthy + unhealthy);
|
||||||
|
el.textContent = `${healthy}/${total}`;
|
||||||
|
if (sub) {
|
||||||
|
if (unhealthy === 0) {
|
||||||
|
sub.innerHTML = '<span class="dc-monitor-pill ok">● all healthy</span>';
|
||||||
|
} else if (unhealthy <= 2) {
|
||||||
|
sub.innerHTML = `<span class="dc-monitor-pill warn">● ${unhealthy} degraded</span>`;
|
||||||
|
} else {
|
||||||
|
sub.innerHTML = `<span class="dc-monitor-pill bad">● ${unhealthy} down</span>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----- Data fetches -----
|
||||||
|
async function fetchStats() {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/v1/monitoring/stats', { cache: 'no-store' });
|
||||||
|
if (!r.ok) return null;
|
||||||
|
const data = await r.json();
|
||||||
|
return (data && data.stats) ? data.stats : null;
|
||||||
|
} catch (_) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchHealth() {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/v1/health-checks/status', { cache: 'no-store' });
|
||||||
|
if (!r.ok) return null;
|
||||||
|
return await r.json();
|
||||||
|
} catch (_) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyStats(stats) {
|
||||||
|
const containers = document.getElementById('dc-monitor-containers');
|
||||||
|
const containersSub = document.getElementById('dc-monitor-containers-sub');
|
||||||
|
const cpuEl = document.getElementById('dc-monitor-cpu');
|
||||||
|
const memEl = document.getElementById('dc-monitor-mem');
|
||||||
|
|
||||||
|
if (!stats) {
|
||||||
|
if (containers) containers.textContent = '—';
|
||||||
|
if (cpuEl) cpuEl.textContent = '—';
|
||||||
|
if (memEl) memEl.textContent = '—';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const entries = Object.values(stats);
|
||||||
|
if (entries.length === 0) {
|
||||||
|
if (containers) containers.textContent = '0';
|
||||||
|
if (containersSub) containersSub.textContent = 'no containers reporting';
|
||||||
|
if (cpuEl) cpuEl.textContent = '0%';
|
||||||
|
if (memEl) memEl.textContent = '0%';
|
||||||
|
setBar('dc-monitor-cpu-bar', 0);
|
||||||
|
setBar('dc-monitor-mem-bar', 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cpuSum = 0, memSum = 0, memBytes = 0, cpuCount = 0, memCount = 0;
|
||||||
|
entries.forEach(s => {
|
||||||
|
// CPU may be percentage (0-100) or fraction (0-1) — handle both
|
||||||
|
if (s.cpu != null) {
|
||||||
|
const cpu = Number(s.cpu);
|
||||||
|
if (!isNaN(cpu)) {
|
||||||
|
cpuSum += cpu > 1 ? cpu : cpu * 100;
|
||||||
|
cpuCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (s.memory != null) {
|
||||||
|
const mem = Number(s.memory);
|
||||||
|
if (!isNaN(mem)) {
|
||||||
|
memSum += mem;
|
||||||
|
memBytes += Number(s.memoryUsage || 0);
|
||||||
|
memCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const avgCpu = cpuCount ? cpuSum / cpuCount : 0;
|
||||||
|
const avgMem = memCount ? memSum / memCount : 0;
|
||||||
|
|
||||||
|
if (containers) containers.textContent = String(entries.length);
|
||||||
|
if (containersSub) {
|
||||||
|
const memTxt = memBytes ? ` · ${fmtBytes(memBytes)} RAM` : '';
|
||||||
|
containersSub.textContent = `running${memTxt}`;
|
||||||
|
}
|
||||||
|
if (cpuEl) cpuEl.textContent = fmtPct(avgCpu);
|
||||||
|
if (memEl) memEl.textContent = fmtPct(avgMem);
|
||||||
|
setBar('dc-monitor-cpu-bar', avgCpu);
|
||||||
|
setBar('dc-monitor-mem-bar', avgMem);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----- Public refresh function -----
|
||||||
|
let inFlight = false;
|
||||||
|
async function refresh() {
|
||||||
|
if (inFlight) return;
|
||||||
|
inFlight = true;
|
||||||
|
try {
|
||||||
|
setServicesCard();
|
||||||
|
const [stats, health] = await Promise.all([fetchStats(), fetchHealth()]);
|
||||||
|
applyStats(stats);
|
||||||
|
applyHealthSummary(health);
|
||||||
|
const stamp = document.getElementById('dc-monitor-refresh-stamp');
|
||||||
|
if (stamp) {
|
||||||
|
const now = new Date();
|
||||||
|
stamp.textContent = `updated ${now.toLocaleTimeString()}`;
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
inFlight = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expose for init.js to call once and re-call after each refreshAll cycle
|
||||||
|
window.refreshMonitoringWidgets = refresh;
|
||||||
|
|
||||||
|
// Auto-refresh on the STATS interval (separate from full DASHBOARD refresh)
|
||||||
|
setInterval(refresh, (typeof DC !== 'undefined' && DC.POLL && DC.POLL.STATS) || 5000);
|
||||||
|
|
||||||
|
// Refresh once on first script load (init.js also calls this; double-call is harmless)
|
||||||
|
setTimeout(refresh, 200);
|
||||||
|
|
||||||
|
})();
|
||||||
@@ -2,11 +2,50 @@
|
|||||||
(function() {
|
(function() {
|
||||||
const searchInput = document.getElementById('service-filter-search');
|
const searchInput = document.getElementById('service-filter-search');
|
||||||
const statusSelect = document.getElementById('service-filter-status');
|
const statusSelect = document.getElementById('service-filter-status');
|
||||||
|
const categorySelect = document.getElementById('service-filter-category');
|
||||||
const countSpan = document.getElementById('service-filter-count');
|
const countSpan = document.getElementById('service-filter-count');
|
||||||
|
|
||||||
|
// Build a single category list from both the API categories and any
|
||||||
|
// categories present on the actual rendered cards (covers custom services
|
||||||
|
// whose category isn't in TEMPLATE_CATEGORIES).
|
||||||
|
function getCategoryList() {
|
||||||
|
const seen = new Set();
|
||||||
|
const fromCards = new Set();
|
||||||
|
document.querySelectorAll('#cards .card[data-category]').forEach(c => {
|
||||||
|
const cat = c.dataset.category.trim();
|
||||||
|
if (cat) fromCards.add(cat);
|
||||||
|
});
|
||||||
|
const apiCats = (window.DC_CATEGORIES || (typeof DC !== 'undefined' && DC.CATEGORIES)) || {};
|
||||||
|
const all = Object.keys(apiCats).concat([...fromCards].filter(c => !apiCats[c]));
|
||||||
|
all.forEach(c => seen.add(c));
|
||||||
|
return { list: [...seen], apiCats };
|
||||||
|
}
|
||||||
|
|
||||||
|
function refreshCategoryDropdown() {
|
||||||
|
if (!categorySelect) return;
|
||||||
|
const { list, apiCats } = getCategoryList();
|
||||||
|
const current = categorySelect.value;
|
||||||
|
categorySelect.innerHTML = '<option value="all">All Categories</option>';
|
||||||
|
list.sort().forEach(name => {
|
||||||
|
const info = apiCats[name];
|
||||||
|
const opt = document.createElement('option');
|
||||||
|
opt.value = name;
|
||||||
|
opt.textContent = info ? `${info.icon || ''} ${name}`.trim() : name;
|
||||||
|
categorySelect.appendChild(opt);
|
||||||
|
});
|
||||||
|
// Restore selection if it still exists
|
||||||
|
if (current && [...categorySelect.options].some(o => o.value === current)) {
|
||||||
|
categorySelect.value = current;
|
||||||
|
} else {
|
||||||
|
categorySelect.value = 'all';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function updateFilter() {
|
function updateFilter() {
|
||||||
|
refreshCategoryDropdown();
|
||||||
const query = searchInput.value.toLowerCase().trim();
|
const query = searchInput.value.toLowerCase().trim();
|
||||||
const statusFilter = statusSelect.value; // 'all', 'on', or 'off'
|
const statusFilter = statusSelect.value; // 'all', 'on', or 'off'
|
||||||
|
const categoryFilter = categorySelect ? categorySelect.value : 'all';
|
||||||
|
|
||||||
const cards = document.querySelectorAll('#cards .card');
|
const cards = document.querySelectorAll('#cards .card');
|
||||||
let visibleCount = 0;
|
let visibleCount = 0;
|
||||||
@@ -15,11 +54,13 @@
|
|||||||
const name = card.querySelector('.name')?.textContent?.toLowerCase() || '';
|
const name = card.querySelector('.name')?.textContent?.toLowerCase() || '';
|
||||||
const app = card.dataset.app?.toLowerCase() || '';
|
const app = card.dataset.app?.toLowerCase() || '';
|
||||||
const status = card.dataset.status || 'off'; // 'on' or 'off'
|
const status = card.dataset.status || 'off'; // 'on' or 'off'
|
||||||
|
const category = card.dataset.category || '';
|
||||||
|
|
||||||
const matchesSearch = !query || name.includes(query) || app.includes(query);
|
const matchesSearch = !query || name.includes(query) || app.includes(query);
|
||||||
const matchesStatus = statusFilter === 'all' || status === statusFilter;
|
const matchesStatus = statusFilter === 'all' || status === statusFilter;
|
||||||
|
const matchesCategory = categoryFilter === 'all' || category === categoryFilter;
|
||||||
|
|
||||||
if (matchesSearch && matchesStatus) {
|
if (matchesSearch && matchesStatus && matchesCategory) {
|
||||||
card.style.display = '';
|
card.style.display = '';
|
||||||
visibleCount++;
|
visibleCount++;
|
||||||
} else {
|
} else {
|
||||||
@@ -44,6 +85,7 @@
|
|||||||
|
|
||||||
searchInput?.addEventListener('input', debounce(updateFilter, 200));
|
searchInput?.addEventListener('input', debounce(updateFilter, 200));
|
||||||
statusSelect?.addEventListener('change', updateFilter);
|
statusSelect?.addEventListener('change', updateFilter);
|
||||||
|
categorySelect?.addEventListener('change', updateFilter);
|
||||||
|
|
||||||
// Initial count on page load
|
// Initial count on page load
|
||||||
if (document.readyState === 'loading') {
|
if (document.readyState === 'loading') {
|
||||||
@@ -52,6 +94,7 @@
|
|||||||
setTimeout(updateFilter, 500);
|
setTimeout(updateFilter, 500);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Expose for external triggers
|
// Expose for external triggers (called after buildGrid to repopulate categories)
|
||||||
window.refreshServiceFilter = updateFilter;
|
window.refreshServiceFilter = updateFilter;
|
||||||
|
window.refreshCategoryDropdown = refreshCategoryDropdown;
|
||||||
})();
|
})();
|
||||||
|
|||||||
@@ -17,8 +17,10 @@
|
|||||||
|
|
||||||
<!-- Tab: Available Updates -->
|
<!-- Tab: Available Updates -->
|
||||||
<div id="updates-available" class="panel-section active">
|
<div id="updates-available" class="panel-section active">
|
||||||
<div style="margin-bottom: 12px;">
|
<div style="margin-bottom: 12px; display: flex; gap: 8px; align-items: center;">
|
||||||
<button id="updates-check-btn" class="btn-accent-solid">🔍 Check for Updates</button>
|
<button id="updates-check-btn" class="btn-accent-solid">🔍 Check for Updates</button>
|
||||||
|
<button id="updates-update-all-btn" style="display: none; padding: 6px 14px; font-size: 0.82rem; background: #f97316; color: #fff; border: 1px solid #f97316; border-radius: 6px; cursor: pointer;">⬆️ Update All</button>
|
||||||
|
<span id="updates-count-badge" style="display: none; padding: 4px 10px; border-radius: 12px; font-size: 0.78rem; font-weight: 600; background: var(--accent); color: var(--bg);"></span>
|
||||||
</div>
|
</div>
|
||||||
<div id="updates-available-container" style="max-height: 450px; overflow-y: auto;">
|
<div id="updates-available-container" style="max-height: 450px; overflow-y: auto;">
|
||||||
<div class="panel-empty"><span class="empty-icon">📦</span> Click "Check for Updates" to scan containers.</div>
|
<div class="panel-empty"><span class="empty-icon">📦</span> Click "Check for Updates" to scan containers.</div>
|
||||||
@@ -94,13 +96,24 @@
|
|||||||
if (updates.length === 0) {
|
if (updates.length === 0) {
|
||||||
availableContainer.innerHTML = '<div class="panel-empty"><span class="empty-icon">✅</span>All containers are up to date.</div>';
|
availableContainer.innerHTML = '<div class="panel-empty"><span class="empty-icon">✅</span>All containers are up to date.</div>';
|
||||||
lastCheckSpan.textContent = '';
|
lastCheckSpan.textContent = '';
|
||||||
|
document.getElementById('updates-update-all-btn').style.display = 'none';
|
||||||
|
document.getElementById('updates-count-badge').style.display = 'none';
|
||||||
|
window._pendingUpdates = [];
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let html = '<table style="width: 100%; border-collapse: collapse; font-size: 0.85rem;">';
|
let html = '<table style="width: 100%; border-collapse: collapse; font-size: 0.85rem;">';
|
||||||
html += '<tr style="border-bottom: 1px solid var(--border); color: var(--muted);"><th style="padding: 8px; text-align: left;">Container</th><th style="padding: 8px; text-align: left;">Image</th><th style="padding: 8px; text-align: left;">Current</th><th style="padding: 8px; text-align: left;">Latest</th><th style="padding: 8px; text-align: right;">Actions</th></tr>';
|
html += '<tr style="border-bottom: 1px solid var(--border); color: var(--muted);"><th style="padding: 8px; text-align: left;">Container</th><th style="padding: 8px; text-align: left;">Image</th><th style="padding: 8px; text-align: left;">Current</th><th style="padding: 8px; text-align: left;">Latest</th><th style="padding: 8px; text-align: right;">Actions</th></tr>';
|
||||||
for (const u of updates) {
|
for (const u of updates) {
|
||||||
html += `<tr style="border-bottom: 1px solid var(--border);">`;
|
// Match app by containerId first, then name
|
||||||
|
const appId = (() => {
|
||||||
|
const apps = window.APPS || [];
|
||||||
|
for (const a of apps) {
|
||||||
|
if (a.containerId === u.containerId || a.name === u.containerName || a.id === u.containerName) return a.id;
|
||||||
|
}
|
||||||
|
return u.containerName;
|
||||||
|
})();
|
||||||
|
html += `<tr data-app-id="${escapeHtml(appId)}" style="border-bottom: 1px solid var(--border);">`;
|
||||||
html += `<td style="padding: 8px; font-weight: 500;">${escapeHtml(u.containerName)}</td>`;
|
html += `<td style="padding: 8px; font-weight: 500;">${escapeHtml(u.containerName)}</td>`;
|
||||||
html += `<td style="padding: 8px; color: var(--muted);">${escapeHtml(u.imageName)}</td>`;
|
html += `<td style="padding: 8px; color: var(--muted);">${escapeHtml(u.imageName)}</td>`;
|
||||||
html += `<td style="padding: 8px;"><code style="font-size: 0.78rem; background: var(--bg); padding: 2px 6px; border-radius: 4px;">${escapeHtml(u.currentDigest)}</code></td>`;
|
html += `<td style="padding: 8px;"><code style="font-size: 0.78rem; background: var(--bg); padding: 2px 6px; border-radius: 4px;">${escapeHtml(u.currentDigest)}</code></td>`;
|
||||||
@@ -114,6 +127,20 @@
|
|||||||
availableContainer.innerHTML = html;
|
availableContainer.innerHTML = html;
|
||||||
lastCheckSpan.textContent = updates.length + ' update(s) available';
|
lastCheckSpan.textContent = updates.length + ' update(s) available';
|
||||||
|
|
||||||
|
// Show count badge and Update All button
|
||||||
|
const countBadge = document.getElementById('updates-count-badge');
|
||||||
|
const updateAllBtn = document.getElementById('updates-update-all-btn');
|
||||||
|
if (countBadge) {
|
||||||
|
countBadge.textContent = updates.length + ' pending';
|
||||||
|
countBadge.style.display = '';
|
||||||
|
}
|
||||||
|
if (updateAllBtn && updates.length > 0) {
|
||||||
|
updateAllBtn.style.display = '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store updates for Update All button
|
||||||
|
window._pendingUpdates = updates;
|
||||||
|
|
||||||
// Wire update buttons
|
// Wire update buttons
|
||||||
availableContainer.querySelectorAll('.update-now-btn').forEach(btn => {
|
availableContainer.querySelectorAll('.update-now-btn').forEach(btn => {
|
||||||
btn.addEventListener('click', async () => {
|
btn.addEventListener('click', async () => {
|
||||||
@@ -174,6 +201,38 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update All — sequentially, skip failures
|
||||||
|
async function updateAllContainers() {
|
||||||
|
const updates = window._pendingUpdates || [];
|
||||||
|
if (!updates.length) return;
|
||||||
|
const btn = document.getElementById('updates-update-all-btn');
|
||||||
|
if (!confirm(`Update all ${updates.length} containers? Each will restart.`)) return;
|
||||||
|
btn.textContent = '⏳ Updating...';
|
||||||
|
btn.disabled = true;
|
||||||
|
let success = 0, failed = 0;
|
||||||
|
for (const u of updates) {
|
||||||
|
try {
|
||||||
|
const r = await secureFetch(`/api/v1/updates/update/${encodeURIComponent(u.containerId)}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ autoRollback: true })
|
||||||
|
});
|
||||||
|
const d = await r.json();
|
||||||
|
if (d.success) success++;
|
||||||
|
else failed++;
|
||||||
|
} catch (_) { failed++; }
|
||||||
|
}
|
||||||
|
btn.textContent = `✅ Done`;
|
||||||
|
showNotification(`Update all: ${success} succeeded, ${failed} failed.`, success > 0 && failed === 0 ? 'success' : 'error');
|
||||||
|
setTimeout(() => {
|
||||||
|
btn.textContent = '⬆️ Update All';
|
||||||
|
btn.disabled = false;
|
||||||
|
loadAvailable();
|
||||||
|
}, 3000);
|
||||||
|
}
|
||||||
|
|
||||||
|
document.getElementById('updates-update-all-btn')?.addEventListener('click', updateAllContainers);
|
||||||
|
|
||||||
async function checkForUpdates() {
|
async function checkForUpdates() {
|
||||||
checkBtn.textContent = '🔍 Checking...';
|
checkBtn.textContent = '🔍 Checking...';
|
||||||
checkBtn.disabled = true;
|
checkBtn.disabled = true;
|
||||||
@@ -499,6 +558,21 @@
|
|||||||
});
|
});
|
||||||
wireModal(modal, cancelBtn);
|
wireModal(modal, cancelBtn);
|
||||||
|
|
||||||
|
// Open Update Management modal, optionally scrolled to a specific app
|
||||||
|
window.openUpdateModal = function(appId) {
|
||||||
|
modal?.classList.add('show');
|
||||||
|
loadAvailable().then(() => {
|
||||||
|
if (!appId) return;
|
||||||
|
// Scroll to and highlight the matching row
|
||||||
|
const row = availableContainer.querySelector(`[data-app-id="${appId}"]`);
|
||||||
|
if (row) {
|
||||||
|
row.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||||
|
row.style.background = 'rgba(249,115,22,0.15)';
|
||||||
|
setTimeout(() => { row.style.background = ''; }, 3000);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
// Lazy-load tabs
|
// Lazy-load tabs
|
||||||
document.querySelector('[data-panel="updates-history"]')?.addEventListener('click', loadHistory);
|
document.querySelector('[data-panel="updates-history"]')?.addEventListener('click', loadHistory);
|
||||||
document.querySelector('[data-panel="updates-auto"]')?.addEventListener('click', loadAutoConfig);
|
document.querySelector('[data-panel="updates-auto"]')?.addEventListener('click', loadAutoConfig);
|
||||||
|
|||||||
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
const CACHE = 'dashcaddy-shell-8ef9c82616';
|
const CACHE = 'dashcaddy-shell-43a872cc40';
|
||||||
const PRECACHE = [
|
const PRECACHE = [
|
||||||
'/',
|
'/',
|
||||||
'/index.html',
|
'/index.html',
|
||||||
|
|||||||
Reference in New Issue
Block a user