Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
954be9e868 | ||
|
|
afcccf811e | ||
|
|
0aa7244cf4 | ||
|
|
1d8919532b | ||
|
|
ea9bdf9598 | ||
|
|
c52016d727 | ||
|
|
588188edb5 |
@@ -1 +1 @@
|
||||
dev
|
||||
1.9.0
|
||||
|
||||
@@ -0,0 +1,503 @@
|
||||
/**
|
||||
* Auto-Restart Manager - Per-container restart policies with retry tracking
|
||||
*
|
||||
* When a container goes down, attempts automatic restart up to N times
|
||||
* (configurable per-service). Sends notifications on each attempt and
|
||||
* when max retries are exceeded. Integrates with HealthChecker events.
|
||||
*
|
||||
* @module auto-restart-manager
|
||||
*/
|
||||
|
||||
const EventEmitter = require('events');
|
||||
const path = require('path');
|
||||
const { readJsonFile, writeJsonFile } = require('./fs-helpers');
|
||||
|
||||
/**
|
||||
* Default policy values applied when a new policy is created.
|
||||
* @readonly
|
||||
*/
|
||||
const DEFAULT_POLICY = {
|
||||
enabled: true,
|
||||
maxRetries: 3,
|
||||
retryIntervalMs: 5000,
|
||||
windowMinutes: 10,
|
||||
currentRetries: 0,
|
||||
lastRestartAt: null,
|
||||
cooldownUntil: null,
|
||||
};
|
||||
|
||||
/**
|
||||
* Manages automatic container restart policies and execution.
|
||||
*
|
||||
* @extends EventEmitter
|
||||
*
|
||||
* @fires AutoRestartManager#auto-restart-attempt
|
||||
* @fires AutoRestartManager#auto-restart-success
|
||||
* @fires AutoRestartManager#auto-restart-failed
|
||||
* @fires AutoRestartManager#auto-restart-max-reached
|
||||
*/
|
||||
class AutoRestartManager extends EventEmitter {
|
||||
/**
|
||||
* @param {Object} ctx - Shared application context
|
||||
* @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode })
|
||||
* @param {Object} ctx.healthChecker - HealthChecker singleton
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
* @param {Function} ctx.logError - Error logging function
|
||||
* @param {string} ctx.SERVICES_FILE - Path to services.json (used to derive data dir)
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.ctx = ctx;
|
||||
this.log = ctx.log || console;
|
||||
this.logError = ctx.logError || ((_ctx, err) => console.error(err));
|
||||
this.docker = ctx.docker;
|
||||
this.healthChecker = ctx.healthChecker;
|
||||
this.notification = ctx.notification;
|
||||
|
||||
/** @type {Map<string, Object>} serviceId -> policy */
|
||||
this.policies = new Map();
|
||||
|
||||
/** Path to the JSON file that persists policies */
|
||||
this.policiesFile = path.join(path.dirname(ctx.SERVICES_FILE), 'auto-restart-policies.json');
|
||||
|
||||
/** Track previous health status per service for transition detection */
|
||||
this._previousHealth = new Map();
|
||||
|
||||
/** Bound handlers so we can remove them on stop() */
|
||||
this._onStatusCheck = this._handleStatusCheck.bind(this);
|
||||
this._started = false;
|
||||
}
|
||||
|
||||
// ─── Lifecycle ────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Load persisted policies, then wire into HealthChecker events.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async start() {
|
||||
if (this._started) return;
|
||||
|
||||
// Load persisted policies from disk
|
||||
try {
|
||||
const data = await readJsonFile(this.policiesFile, {});
|
||||
for (const [serviceId, policy] of Object.entries(data)) {
|
||||
this.policies.set(serviceId, { ...DEFAULT_POLICY, ...policy });
|
||||
}
|
||||
this.log.info('auto-restart', 'Policies loaded', { count: this.policies.size });
|
||||
} catch (err) {
|
||||
this.log.error('auto-restart', 'Failed to load policies', { error: err.message });
|
||||
}
|
||||
|
||||
// Listen to health checker status transitions
|
||||
if (this.healthChecker) {
|
||||
this.healthChecker.on('status-check', this._onStatusCheck);
|
||||
}
|
||||
|
||||
this._started = true;
|
||||
this.log.info('auto-restart', 'Manager started');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove event listeners and stop processing health events.
|
||||
*/
|
||||
stop() {
|
||||
if (!this._started) return;
|
||||
|
||||
if (this.healthChecker) {
|
||||
this.healthChecker.removeListener('status-check', this._onStatusCheck);
|
||||
}
|
||||
|
||||
this._started = false;
|
||||
this.log.info('auto-restart', 'Manager stopped');
|
||||
}
|
||||
|
||||
// ─── Policy CRUD ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Create or update a restart policy for a service.
|
||||
*
|
||||
* @param {string} serviceId - Unique service identifier
|
||||
* @param {Object} policy - Partial policy fields to merge
|
||||
* @param {boolean} [policy.enabled=true]
|
||||
* @param {number} [policy.maxRetries=3]
|
||||
* @param {number} [policy.retryIntervalMs=5000]
|
||||
* @param {number} [policy.windowMinutes=10]
|
||||
* @returns {Promise<Object>} The resulting policy
|
||||
* @throws {Error} If serviceId is invalid
|
||||
*/
|
||||
async setPolicy(serviceId, policy) {
|
||||
if (!serviceId || typeof serviceId !== 'string') {
|
||||
throw new Error('serviceId is required');
|
||||
}
|
||||
|
||||
const existing = this.policies.get(serviceId) || { ...DEFAULT_POLICY, serviceId };
|
||||
|
||||
const merged = {
|
||||
...existing,
|
||||
...policy,
|
||||
serviceId,
|
||||
// Never allow caller to override runtime counters directly
|
||||
currentRetries: existing.currentRetries || 0,
|
||||
lastRestartAt: existing.lastRestartAt,
|
||||
cooldownUntil: existing.cooldownUntil,
|
||||
};
|
||||
|
||||
this.policies.set(serviceId, merged);
|
||||
await this._savePolicies();
|
||||
|
||||
this.log.info('auto-restart', 'Policy set', { serviceId, enabled: merged.enabled });
|
||||
return { ...merged };
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the policy for a service.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Object|null} Policy object or null if none exists
|
||||
*/
|
||||
getPolicy(serviceId) {
|
||||
const policy = this.policies.get(serviceId);
|
||||
return policy ? { ...policy } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all policies as an array.
|
||||
* @returns {Object[]}
|
||||
*/
|
||||
listPolicies() {
|
||||
return Array.from(this.policies.values()).map(p => ({ ...p }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a service's restart policy.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<boolean>} true if a policy was removed
|
||||
*/
|
||||
async removePolicy(serviceId) {
|
||||
if (!this.policies.has(serviceId)) return false;
|
||||
|
||||
this.policies.delete(serviceId);
|
||||
await this._savePolicies();
|
||||
|
||||
this.log.info('auto-restart', 'Policy removed', { serviceId });
|
||||
return true;
|
||||
}
|
||||
|
||||
// ─── Core Restart Logic ──────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Called when a container is detected as down.
|
||||
*
|
||||
* Checks policy, cooldown, and retry count, then either attempts a
|
||||
* Docker restart or notifies that max retries were exceeded.
|
||||
*
|
||||
* @param {string} serviceId - Service identifier
|
||||
* @param {string} containerId - Docker container ID to restart
|
||||
* @returns {Promise<Object>} Result of the operation
|
||||
*/
|
||||
async handleContainerDown(serviceId, containerId) {
|
||||
const policy = this.policies.get(serviceId);
|
||||
if (!policy) {
|
||||
return { action: 'ignored', reason: 'no-policy' };
|
||||
}
|
||||
|
||||
if (!policy.enabled) {
|
||||
return { action: 'ignored', reason: 'disabled' };
|
||||
}
|
||||
|
||||
// Check cooldown window
|
||||
const now = Date.now();
|
||||
if (policy.cooldownUntil && now < policy.cooldownUntil) {
|
||||
this.log.info('auto-restart', 'Skipping — cooldown active', {
|
||||
serviceId,
|
||||
cooldownUntil: new Date(policy.cooldownUntil).toISOString(),
|
||||
});
|
||||
return { action: 'skipped', reason: 'cooldown' };
|
||||
}
|
||||
|
||||
// Max retries exceeded — notify and enter cooldown
|
||||
if (policy.currentRetries >= policy.maxRetries) {
|
||||
const cooldownMs = policy.windowMinutes * 60 * 1000;
|
||||
policy.cooldownUntil = now + cooldownMs;
|
||||
policy.currentRetries = 0; // Reset so next window can try again
|
||||
await this._savePolicies();
|
||||
|
||||
const eventData = {
|
||||
serviceId,
|
||||
containerId,
|
||||
maxRetries: policy.maxRetries,
|
||||
cooldownUntil: policy.cooldownUntil,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-max-reached
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-max-reached', eventData);
|
||||
|
||||
// Send notification
|
||||
try {
|
||||
await this._notify('auto-restart', {
|
||||
containerName: serviceId,
|
||||
message: `⛔ Max auto-restart retries (${policy.maxRetries}) exceeded for "${serviceId}". Cooldown until ${new Date(policy.cooldownUntil).toISOString()}.`,
|
||||
...eventData,
|
||||
});
|
||||
} catch (notifErr) {
|
||||
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||
}
|
||||
|
||||
return { action: 'max-reached', ...eventData };
|
||||
}
|
||||
|
||||
// Wait for the configured retry interval before attempting
|
||||
if (policy.retryIntervalMs > 0 && policy.lastRestartAt) {
|
||||
const elapsed = now - new Date(policy.lastRestartAt).getTime();
|
||||
if (elapsed < policy.retryIntervalMs) {
|
||||
const waitMs = policy.retryIntervalMs - elapsed;
|
||||
this.log.info('auto-restart', 'Waiting for retry interval', { serviceId, waitMs });
|
||||
await new Promise(resolve => setTimeout(resolve, waitMs));
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt restart
|
||||
policy.currentRetries += 1;
|
||||
const attemptNum = policy.currentRetries;
|
||||
const maxRetries = policy.maxRetries;
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-attempt
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-attempt', {
|
||||
serviceId,
|
||||
containerId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
try {
|
||||
if (!this.docker?.client) {
|
||||
throw new Error('Docker client not available');
|
||||
}
|
||||
|
||||
const container = this.docker.client.getContainer(containerId);
|
||||
await container.start();
|
||||
|
||||
policy.lastRestartAt = new Date().toISOString();
|
||||
await this._savePolicies();
|
||||
|
||||
const successData = {
|
||||
serviceId,
|
||||
containerId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-success
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-success', successData);
|
||||
|
||||
// Notify
|
||||
try {
|
||||
await this._notify('auto-restart', {
|
||||
containerName: serviceId,
|
||||
message: `🔄 Auto-restart attempt ${attemptNum}/${maxRetries} succeeded for "${serviceId}".`,
|
||||
...successData,
|
||||
});
|
||||
} catch (notifErr) {
|
||||
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||
}
|
||||
|
||||
this.log.info('auto-restart', 'Container restarted', {
|
||||
serviceId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
});
|
||||
|
||||
return { action: 'restarted', ...successData };
|
||||
} catch (restartErr) {
|
||||
policy.lastRestartAt = new Date().toISOString();
|
||||
await this._savePolicies();
|
||||
|
||||
const failData = {
|
||||
serviceId,
|
||||
containerId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
error: restartErr.message,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-failed
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-failed', failData);
|
||||
|
||||
// Notify
|
||||
try {
|
||||
await this._notify('auto-restart', {
|
||||
containerName: serviceId,
|
||||
message: `❌ Auto-restart attempt ${attemptNum}/${maxRetries} failed for "${serviceId}": ${restartErr.message}`,
|
||||
...failData,
|
||||
});
|
||||
} catch (notifErr) {
|
||||
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||
}
|
||||
|
||||
this.log.error('auto-restart', 'Restart failed', {
|
||||
serviceId,
|
||||
attempt: attemptNum,
|
||||
error: restartErr.message,
|
||||
});
|
||||
|
||||
return { action: 'failed', ...failData };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a container recovers to healthy state.
|
||||
* Resets the retry counter for the associated service.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async handleContainerUp(serviceId) {
|
||||
const policy = this.policies.get(serviceId);
|
||||
if (!policy) return;
|
||||
|
||||
if (policy.currentRetries > 0) {
|
||||
policy.currentRetries = 0;
|
||||
policy.cooldownUntil = null;
|
||||
await this._savePolicies();
|
||||
|
||||
this.log.info('auto-restart', 'Retries reset after recovery', { serviceId });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Health Event Bridge ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Internal handler for HealthChecker `status-check` events.
|
||||
* Detects healthy→unhealthy and unhealthy→healthy transitions for tracked services.
|
||||
*
|
||||
* @param {Object} status - HealthChecker status object
|
||||
* @param {string} status.serviceId
|
||||
* @param {string} status.status - "up" or "down"
|
||||
* @private
|
||||
*/
|
||||
async _handleStatusCheck(status) {
|
||||
const { serviceId, status: currentStatus } = status;
|
||||
if (!serviceId) return;
|
||||
|
||||
// Only process services that have a restart policy
|
||||
if (!this.policies.has(serviceId)) return;
|
||||
|
||||
const previousStatus = this._previousHealth.get(serviceId);
|
||||
this._previousHealth.set(serviceId, currentStatus);
|
||||
|
||||
// Transition: healthy → unhealthy
|
||||
if (previousStatus === 'up' && currentStatus === 'down') {
|
||||
// Find the containerId from the health checker config or status details
|
||||
const containerId = this._resolveContainerId(serviceId, status);
|
||||
if (containerId) {
|
||||
try {
|
||||
await this.handleContainerDown(serviceId, containerId);
|
||||
} catch (err) {
|
||||
this.logError('auto-restart-health-bridge', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Transition: unhealthy → healthy (recovery)
|
||||
if (previousStatus === 'down' && currentStatus === 'up') {
|
||||
try {
|
||||
await this.handleContainerUp(serviceId);
|
||||
} catch (err) {
|
||||
this.logError('auto-restart-health-bridge', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to find the containerId for a service from various sources.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @param {Object} status - The status-check event data
|
||||
* @returns {string|null}
|
||||
* @private
|
||||
*/
|
||||
_resolveContainerId(serviceId, status) {
|
||||
// Check if it's in the status details (some health checks embed it)
|
||||
if (status.details?.containerId) return status.details.containerId;
|
||||
|
||||
// Look in the health checker config
|
||||
const hcService = this.healthChecker?.config?.services?.[serviceId];
|
||||
if (hcService?.containerId) return hcService.containerId;
|
||||
|
||||
// Try to look it up from the services state manager
|
||||
try {
|
||||
const servicesStateManager = this.ctx.servicesStateManager;
|
||||
if (servicesStateManager) {
|
||||
const readResult = servicesStateManager.read();
|
||||
if (readResult && typeof readResult.then === 'function') {
|
||||
// It returns a promise — fire-and-forget lookup
|
||||
readResult.then(list => {
|
||||
const found = (list || []).find(s => s.id === serviceId);
|
||||
return found?.containerId || null;
|
||||
}).catch(() => null);
|
||||
} else {
|
||||
const found = (readResult || []).find(s => s.id === serviceId);
|
||||
if (found?.containerId) return found.containerId;
|
||||
}
|
||||
}
|
||||
} catch (_) { /* best effort */ }
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Persistence ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Persist current policies to disk.
|
||||
* @returns {Promise<void>}
|
||||
* @private
|
||||
*/
|
||||
async _savePolicies() {
|
||||
try {
|
||||
const obj = {};
|
||||
for (const [serviceId, policy] of this.policies.entries()) {
|
||||
obj[serviceId] = { ...policy };
|
||||
}
|
||||
await writeJsonFile(this.policiesFile, obj);
|
||||
} catch (err) {
|
||||
this.log.error('auto-restart', 'Failed to save policies', { error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Send a notification via the notification manager.
|
||||
*
|
||||
* @param {string} event - Event type (e.g. 'auto-restart')
|
||||
* @param {Object} data - Notification payload
|
||||
* @returns {Promise<Object>}
|
||||
* @private
|
||||
*/
|
||||
async _notify(event, data) {
|
||||
if (this.notification?.send) {
|
||||
return this.notification.send(event, data);
|
||||
}
|
||||
return { success: false, reason: 'no-notification-manager' };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { AutoRestartManager, DEFAULT_POLICY };
|
||||
@@ -0,0 +1,376 @@
|
||||
/**
|
||||
* Config Drift Detector - Compares services.json with live Docker state
|
||||
*
|
||||
* Detects discrepancies between the configured service list and what is
|
||||
* actually running in Docker, including missing containers, unknown
|
||||
* containers, port mismatches, state mismatches, and stale records.
|
||||
*
|
||||
* @module config-drift-detector
|
||||
*/
|
||||
|
||||
const EventEmitter = require('events');
|
||||
|
||||
/**
|
||||
* @typedef {Object} DriftReport
|
||||
* @property {string} checkedAt - ISO timestamp of the check
|
||||
* @property {Object[]} missingContainers - Services with containerId but container absent in Docker
|
||||
* @property {Object[]} unknownContainers - Running Docker containers with sami.managed label but not in services.json
|
||||
* @property {Object[]} portMismatch - Service port != container mapped port
|
||||
* @property {Object[]} stateMismatch - Service expected up but container stopped/absent
|
||||
* @property {Object[]} staleRecords - Services with containerId pointing to removed containers
|
||||
* @property {boolean} hasDrift - Whether any drift category is non-empty
|
||||
*/
|
||||
|
||||
/**
|
||||
* Detects and reports configuration drift between services.json and Docker.
|
||||
*
|
||||
* @extends EventEmitter
|
||||
*
|
||||
* @fires ConfigDriftDetector#drift-detected
|
||||
*/
|
||||
class ConfigDriftDetector extends EventEmitter {
|
||||
/**
|
||||
* @param {Object} ctx - Shared application context
|
||||
* @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode })
|
||||
* @param {Object} ctx.servicesStateManager - StateManager for services.json
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
* @param {Function} ctx.logError - Error logging function
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.ctx = ctx;
|
||||
this.log = ctx.log || console;
|
||||
this.logError = ctx.logError || ((_c, err) => console.error(err));
|
||||
this.docker = ctx.docker;
|
||||
this.servicesStateManager = ctx.servicesStateManager;
|
||||
this.notification = ctx.notification;
|
||||
|
||||
/** @type {DriftReport|null} Cached report from last detection */
|
||||
this.lastReport = null;
|
||||
|
||||
/** @type {NodeJS.Timeout|null} Polling timer reference */
|
||||
this._pollTimer = null;
|
||||
|
||||
/** Whether polling is currently active */
|
||||
this._polling = false;
|
||||
}
|
||||
|
||||
// ─── Detection ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Run a full drift detection and return the report.
|
||||
*
|
||||
* Reads services from servicesStateManager and live containers from Docker,
|
||||
* then compares them across five drift categories.
|
||||
*
|
||||
* @returns {Promise<DriftReport>}
|
||||
*/
|
||||
async detect() {
|
||||
const checkedAt = new Date().toISOString();
|
||||
|
||||
// Gather configured services
|
||||
let services = [];
|
||||
try {
|
||||
const data = await this.servicesStateManager.read();
|
||||
services = Array.isArray(data) ? data : (data.services || []);
|
||||
} catch (err) {
|
||||
this.log.error('drift', 'Failed to read services', { error: err.message });
|
||||
}
|
||||
|
||||
// Gather live Docker containers
|
||||
let containers = [];
|
||||
try {
|
||||
containers = await this.docker.client.listContainers({ all: true });
|
||||
} catch (err) {
|
||||
this.log.error('drift', 'Failed to list containers', { error: err.message });
|
||||
}
|
||||
|
||||
// Build lookup maps
|
||||
const containerById = new Map(); // containerId (short or long) → container info
|
||||
const containerByName = new Map(); // container name → container info
|
||||
|
||||
for (const c of containers) {
|
||||
// Store by full ID
|
||||
containerById.set(c.Id, c);
|
||||
// Store by short ID (first 12 chars)
|
||||
if (c.Id && c.Id.length >= 12) {
|
||||
containerById.set(c.Id.substring(0, 12), c);
|
||||
}
|
||||
// Store by name (strip leading /)
|
||||
for (const name of (c.Names || [])) {
|
||||
containerByName.set(name.replace(/^\//, ''), c);
|
||||
}
|
||||
}
|
||||
|
||||
// Build set of service containerIds for reverse lookup
|
||||
const serviceContainerIds = new Set();
|
||||
const serviceByContainerId = new Map();
|
||||
|
||||
for (const svc of services) {
|
||||
if (svc.containerId) {
|
||||
serviceContainerIds.add(svc.containerId);
|
||||
// Index by both full and short ID
|
||||
serviceByContainerId.set(svc.containerId, svc);
|
||||
if (svc.containerId.length >= 12) {
|
||||
serviceByContainerId.set(svc.containerId.substring(0, 12), svc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const missingContainers = [];
|
||||
const portMismatch = [];
|
||||
const stateMismatch = [];
|
||||
const staleRecords = [];
|
||||
|
||||
for (const svc of services) {
|
||||
if (!svc.containerId) continue;
|
||||
|
||||
// Look up the container
|
||||
const container = containerById.get(svc.containerId)
|
||||
|| containerById.get(svc.containerId.substring(0, 12));
|
||||
|
||||
if (!container) {
|
||||
// Container ID referenced but not found in Docker at all
|
||||
staleRecords.push({
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
containerId: svc.containerId,
|
||||
reason: 'Container not found in Docker',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Missing container — service expects it but it's not running
|
||||
if (container.State !== 'running') {
|
||||
missingContainers.push({
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
containerId: svc.containerId,
|
||||
containerState: container.State,
|
||||
containerStatus: container.Status,
|
||||
});
|
||||
|
||||
// Also a state mismatch if the service is expected to be up
|
||||
stateMismatch.push({
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
expectedState: 'running',
|
||||
actualState: container.State,
|
||||
containerId: svc.containerId,
|
||||
});
|
||||
}
|
||||
|
||||
// Port mismatch detection
|
||||
if (svc.port && container.State === 'running') {
|
||||
const actualPorts = this._extractContainerPorts(container);
|
||||
if (actualPorts.length > 0 && !actualPorts.includes(svc.port)) {
|
||||
portMismatch.push({
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
configuredPort: svc.port,
|
||||
actualPorts,
|
||||
containerId: svc.containerId,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Unknown managed containers: Docker containers with sami.managed label
|
||||
// that are NOT in services.json
|
||||
const unknownContainers = [];
|
||||
for (const c of containers) {
|
||||
const isManaged = c.Labels && c.Labels['sami.managed'] === 'true';
|
||||
if (!isManaged) continue;
|
||||
|
||||
const isInServices = serviceByContainerId.has(c.Id)
|
||||
|| serviceByContainerId.has(c.Id.substring(0, 12));
|
||||
|
||||
if (!isInServices) {
|
||||
unknownContainers.push({
|
||||
containerId: c.Id,
|
||||
name: (c.Names && c.Names[0] || '').replace(/^\//, ''),
|
||||
image: c.Image,
|
||||
state: c.State,
|
||||
status: c.Status,
|
||||
app: c.Labels?.['sami.app'] || null,
|
||||
subdomain: c.Labels?.['sami.subdomain'] || null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const report = {
|
||||
checkedAt,
|
||||
missingContainers,
|
||||
unknownContainers,
|
||||
portMismatch,
|
||||
stateMismatch,
|
||||
staleRecords,
|
||||
hasDrift: missingContainers.length > 0
|
||||
|| unknownContainers.length > 0
|
||||
|| portMismatch.length > 0
|
||||
|| stateMismatch.length > 0
|
||||
|| staleRecords.length > 0,
|
||||
};
|
||||
|
||||
// Cache for quick API access
|
||||
this.lastReport = report;
|
||||
|
||||
// Emit and notify if drift detected
|
||||
if (report.hasDrift) {
|
||||
/**
|
||||
* @event ConfigDriftDetector#drift-detected
|
||||
* @type {DriftReport}
|
||||
*/
|
||||
this.emit('drift-detected', report);
|
||||
|
||||
try {
|
||||
await this._sendDriftNotification(report);
|
||||
} catch (notifErr) {
|
||||
this.log.error('drift', 'Failed to send drift notification', {
|
||||
error: notifErr.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
this.log.info('drift', 'Detection complete', {
|
||||
hasDrift: report.hasDrift,
|
||||
missing: report.missingContainers.length,
|
||||
unknown: report.unknownContainers.length,
|
||||
portMismatch: report.portMismatch.length,
|
||||
stateMismatch: report.stateMismatch.length,
|
||||
stale: report.staleRecords.length,
|
||||
});
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
// ─── Auto-fix ────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Attempt to auto-fix drift:
|
||||
* - Remove stale records (services referencing removed containers)
|
||||
* - Flag unknown containers for review
|
||||
*
|
||||
* @returns {Promise<{ staleRemoved: number, unknownFlagged: number }>}
|
||||
*/
|
||||
async autoFix() {
|
||||
const report = await this.detect();
|
||||
let staleRemoved = 0;
|
||||
|
||||
// Remove stale records from services.json
|
||||
if (report.staleRecords.length > 0) {
|
||||
const staleIds = new Set(report.staleRecords.map(r => r.serviceId));
|
||||
await this.servicesStateManager.update(services => {
|
||||
const before = services.length;
|
||||
const cleaned = services.filter(s => !staleIds.has(s.id));
|
||||
staleRemoved = before - cleaned.length;
|
||||
return cleaned;
|
||||
});
|
||||
}
|
||||
|
||||
const unknownFlagged = report.unknownContainers.length;
|
||||
|
||||
this.log.info('drift', 'Auto-fix applied', { staleRemoved, unknownFlagged });
|
||||
|
||||
return { staleRemoved, unknownFlagged };
|
||||
}
|
||||
|
||||
// ─── Polling ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Start periodic drift detection.
|
||||
*
|
||||
* @param {number} [intervalMs=300000] - Polling interval in milliseconds (default 5 min)
|
||||
*/
|
||||
startPolling(intervalMs = 300000) {
|
||||
this.stopPolling();
|
||||
|
||||
this._polling = true;
|
||||
this._pollTimer = setInterval(async () => {
|
||||
try {
|
||||
await this.detect();
|
||||
} catch (err) {
|
||||
this.logError('drift-poll', err);
|
||||
}
|
||||
}, intervalMs);
|
||||
|
||||
this.log.info('drift', 'Polling started', { intervalMs });
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop periodic drift detection.
|
||||
*/
|
||||
stopPolling() {
|
||||
if (this._pollTimer) {
|
||||
clearInterval(this._pollTimer);
|
||||
this._pollTimer = null;
|
||||
}
|
||||
this._polling = false;
|
||||
this.log.info('drift', 'Polling stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether polling is currently active.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isPolling() {
|
||||
return this._polling;
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Extract mapped host ports from a Docker container info object.
|
||||
*
|
||||
* @param {Object} container - Dockerode container info
|
||||
* @returns {number[]} Array of host port numbers
|
||||
* @private
|
||||
*/
|
||||
_extractContainerPorts(container) {
|
||||
const ports = [];
|
||||
if (!container.Ports) return ports;
|
||||
|
||||
for (const p of container.Ports) {
|
||||
if (p.PublicPort) {
|
||||
ports.push(p.PublicPort);
|
||||
}
|
||||
}
|
||||
|
||||
return ports;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a notification about detected drift.
|
||||
*
|
||||
* @param {DriftReport} report
|
||||
* @returns {Promise<Object>}
|
||||
* @private
|
||||
*/
|
||||
async _sendDriftNotification(report) {
|
||||
if (!this.notification?.send) {
|
||||
return { success: false, reason: 'no-notification-manager' };
|
||||
}
|
||||
|
||||
const parts = [];
|
||||
if (report.missingContainers.length > 0) {
|
||||
parts.push(`Missing containers: ${report.missingContainers.map(c => c.name).join(', ')}`);
|
||||
}
|
||||
if (report.unknownContainers.length > 0) {
|
||||
parts.push(`Unknown managed containers: ${report.unknownContainers.map(c => c.name).join(', ')}`);
|
||||
}
|
||||
if (report.portMismatch.length > 0) {
|
||||
parts.push(`Port mismatches: ${report.portMismatch.map(c => c.name).join(', ')}`);
|
||||
}
|
||||
if (report.staleRecords.length > 0) {
|
||||
parts.push(`Stale records: ${report.staleRecords.map(c => c.name).join(', ')}`);
|
||||
}
|
||||
|
||||
return this.notification.send('drift-detected', {
|
||||
text: `⚠️ Configuration drift detected:\n${parts.join('\n')}`,
|
||||
report,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { ConfigDriftDetector };
|
||||
@@ -0,0 +1,605 @@
|
||||
/**
|
||||
* Dependency Manager - Service dependency tracking with ordered restart chains
|
||||
*
|
||||
* Manages directed acyclic graph (DAG) of service dependencies. Services can
|
||||
* declare which other services they depend on, and this manager provides:
|
||||
* - Full dependency graph inspection
|
||||
* - Topological ordering for safe restart chains
|
||||
* - Circular dependency detection
|
||||
* - Health-aware restart with per-service polling
|
||||
*
|
||||
* Dependencies are stored directly on service objects in services.json:
|
||||
* { id, name, ..., dependsOn: ['service-id-1', 'service-id-2'] }
|
||||
*
|
||||
* @module dependency-manager
|
||||
*/
|
||||
|
||||
const EventEmitter = require('events');
|
||||
|
||||
/** Maximum seconds to wait for a single container to become healthy after restart */
|
||||
const HEALTH_CHECK_TIMEOUT_MS = 30_000;
|
||||
|
||||
/** Interval between container health polls */
|
||||
const HEALTH_CHECK_INTERVAL_MS = 1_000;
|
||||
|
||||
/**
|
||||
* @typedef {Object} ServiceNode
|
||||
* @property {string} serviceId
|
||||
* @property {string} name
|
||||
* @property {string|null} containerId
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} DependencyEdge
|
||||
* @property {string} from - The service that depends
|
||||
* @property {string} to - The service being depended upon
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} DependencyGraph
|
||||
* @property {ServiceNode[]} nodes
|
||||
* @property {DependencyEdge[]} edges
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} DependencyStatusEntry
|
||||
* @property {string} serviceId
|
||||
* @property {string} name
|
||||
* @property {boolean} isUp
|
||||
* @property {string} [error]
|
||||
*/
|
||||
|
||||
/**
|
||||
* DependencyManager — tracks service dependencies and orchestrates ordered restarts.
|
||||
*
|
||||
* Events emitted:
|
||||
* - `dependency-restart-start` ({ serviceId, chain: string[] })
|
||||
* - `dependency-restart-progress` ({ serviceId, currentServiceId, index, total })
|
||||
* - `dependency-restart-complete` ({ serviceId, chain: string[], results: Array })
|
||||
* - `dependency-restart-failed` ({ serviceId, failedServiceId, error, chain: string[] })
|
||||
*
|
||||
* @extends EventEmitter
|
||||
*/
|
||||
class DependencyManager extends EventEmitter {
|
||||
/**
|
||||
* @param {Object} ctx - Application context
|
||||
* @param {Object} ctx.servicesStateManager - StateManager for services.json
|
||||
* @param {Object} ctx.docker - Docker context ({ client: Dockerode })
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
/** @private */
|
||||
this.ctx = ctx;
|
||||
/** @private */
|
||||
this._servicesStateManager = ctx.servicesStateManager;
|
||||
/** @private */
|
||||
this._docker = ctx.docker;
|
||||
/** @private */
|
||||
this._notification = ctx.notification;
|
||||
/** @private */
|
||||
this._log = ctx.log || console;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Core helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Load all services from the state manager.
|
||||
* @private
|
||||
* @returns {Promise<Object[]>}
|
||||
*/
|
||||
async _loadServices() {
|
||||
const data = await this._servicesStateManager.read();
|
||||
return Array.isArray(data) ? data : (data.services || []);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a single service by ID.
|
||||
* @private
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<Object|null>}
|
||||
*/
|
||||
async _findService(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
return services.find(s => s.id === serviceId) || null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Graph queries
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Return the full dependency graph for visualisation.
|
||||
*
|
||||
* @returns {Promise<DependencyGraph>}
|
||||
*/
|
||||
async getDependencyGraph() {
|
||||
const services = await this._loadServices();
|
||||
|
||||
const nodes = services.map(s => ({
|
||||
serviceId: s.id,
|
||||
name: s.name,
|
||||
containerId: s.containerId || null,
|
||||
}));
|
||||
|
||||
const edges = [];
|
||||
for (const service of services) {
|
||||
const deps = service.dependsOn || [];
|
||||
for (const depId of deps) {
|
||||
edges.push({ from: service.id, to: depId });
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes, edges };
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the services that depend on the given service (reverse deps).
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<Object[]>} Services whose `dependsOn` includes `serviceId`.
|
||||
*/
|
||||
async getDependents(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
return services.filter(s => (s.dependsOn || []).includes(serviceId));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the direct dependencies for a service.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<Object[]>} Services that `serviceId` depends on.
|
||||
*/
|
||||
async getDependencies(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
const service = services.find(s => s.id === serviceId);
|
||||
if (!service) return [];
|
||||
const depIds = service.dependsOn || [];
|
||||
return services.filter(s => depIds.includes(s.id));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Topological sort
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build an adjacency list for the current dependency graph.
|
||||
* Edge direction: service → its dependencies (i.e. what it depends on).
|
||||
*
|
||||
* @private
|
||||
* @param {Object[]} services
|
||||
* @returns {Map<string, string[]>}
|
||||
*/
|
||||
_buildAdjacencyList(services) {
|
||||
const adj = new Map();
|
||||
for (const service of services) {
|
||||
adj.set(service.id, (service.dependsOn || []).slice());
|
||||
}
|
||||
return adj;
|
||||
}
|
||||
|
||||
/**
|
||||
* DFS-based topological sort with cycle detection (white/gray/black coloring).
|
||||
*
|
||||
* Returns services in restart order: dependencies first, dependents last.
|
||||
* The target service is included at the end.
|
||||
*
|
||||
* @private
|
||||
* @param {string} serviceId - Target service (will be last in the result).
|
||||
* @param {Object[]} services - All services.
|
||||
* @param {Map<string, string[]>} adj - Adjacency list (service → deps).
|
||||
* @returns {string[]} Ordered service IDs for restart.
|
||||
* @throws {Error} If a circular dependency is detected.
|
||||
*/
|
||||
_topologicalSort(serviceId, services, adj) {
|
||||
// Collect only the reachable sub-graph from serviceId
|
||||
const visited = new Set();
|
||||
const reachable = new Set();
|
||||
|
||||
const collectReachable = (id) => {
|
||||
if (reachable.has(id)) return;
|
||||
reachable.add(id);
|
||||
for (const dep of (adj.get(id) || [])) {
|
||||
collectReachable(dep);
|
||||
}
|
||||
};
|
||||
collectReachable(serviceId);
|
||||
|
||||
// DFS topological sort on the reachable sub-graph
|
||||
const WHITE = 0, GRAY = 1, BLACK = 2;
|
||||
const color = new Map();
|
||||
for (const id of reachable) color.set(id, WHITE);
|
||||
|
||||
const result = [];
|
||||
|
||||
const dfs = (id) => {
|
||||
if (color.get(id) === BLACK) return;
|
||||
if (color.get(id) === GRAY) {
|
||||
throw new Error(`Circular dependency detected involving service "${id}"`);
|
||||
}
|
||||
color.set(id, GRAY);
|
||||
for (const dep of (adj.get(id) || [])) {
|
||||
dfs(dep);
|
||||
}
|
||||
color.set(id, BLACK);
|
||||
result.push(id);
|
||||
};
|
||||
|
||||
// Visit the target last so it ends up at the end of the result
|
||||
// Actually, we want deps *first* then the target.
|
||||
// The DFS naturally puts deps before dependents, so starting from
|
||||
// serviceId will place it last (which is correct for restart order).
|
||||
dfs(serviceId);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the topologically ordered restart chain for a service.
|
||||
*
|
||||
* The returned array lists all services that must be restarted,
|
||||
* starting with leaf dependencies and ending with the target service.
|
||||
*
|
||||
* @param {string} serviceId - The service to build the chain for.
|
||||
* @returns {Promise<string[]>} Ordered service IDs.
|
||||
* @throws {Error} If `serviceId` doesn't exist or a circular dependency is found.
|
||||
*/
|
||||
async getOrderedRestartChain(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
const service = services.find(s => s.id === serviceId);
|
||||
if (!service) {
|
||||
throw new Error(`Service "${serviceId}" not found`);
|
||||
}
|
||||
|
||||
const adj = this._buildAdjacencyList(services);
|
||||
return this._topologicalSort(serviceId, services, adj);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Validation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Validate a proposed set of dependencies for a service.
|
||||
*
|
||||
* Checks:
|
||||
* - All referenced service IDs exist.
|
||||
* - Adding these dependencies would not create a circular dependency.
|
||||
* - A service cannot depend on itself.
|
||||
*
|
||||
* @param {string} serviceId - The service to set dependencies on.
|
||||
* @param {string[]} dependsOn - Proposed dependency IDs.
|
||||
* @returns {Promise<{ valid: boolean, errors: string[] }>}
|
||||
*/
|
||||
async validateDependencies(serviceId, dependsOn) {
|
||||
const errors = [];
|
||||
|
||||
if (!Array.isArray(dependsOn)) {
|
||||
return { valid: false, errors: ['dependsOn must be an array'] };
|
||||
}
|
||||
|
||||
const services = await this._loadServices();
|
||||
const allIds = new Set(services.map(s => s.id));
|
||||
|
||||
// Service must exist
|
||||
if (!allIds.has(serviceId)) {
|
||||
return { valid: false, errors: [`Service "${serviceId}" not found`] };
|
||||
}
|
||||
|
||||
// Self-dependency
|
||||
if (dependsOn.includes(serviceId)) {
|
||||
errors.push(`Service "${serviceId}" cannot depend on itself`);
|
||||
}
|
||||
|
||||
// Existence check
|
||||
for (const depId of dependsOn) {
|
||||
if (!allIds.has(depId)) {
|
||||
errors.push(`Dependency service "${depId}" does not exist`);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
return { valid: false, errors };
|
||||
}
|
||||
|
||||
// Circular dependency check: temporarily set the proposed dependsOn
|
||||
// and attempt a topological sort.
|
||||
const tempServices = services.map(s => {
|
||||
if (s.id === serviceId) {
|
||||
return { ...s, dependsOn: dependsOn.slice() };
|
||||
}
|
||||
return { ...s };
|
||||
});
|
||||
|
||||
const adj = this._buildAdjacencyList(tempServices);
|
||||
|
||||
// Check every node for cycles with the new edges
|
||||
try {
|
||||
const WHITE = 0, GRAY = 1, BLACK = 2;
|
||||
const color = new Map();
|
||||
for (const s of tempServices) color.set(s.id, WHITE);
|
||||
|
||||
const dfs = (id) => {
|
||||
if (color.get(id) === BLACK) return;
|
||||
if (color.get(id) === GRAY) {
|
||||
throw new Error(`Circular dependency detected involving service "${id}"`);
|
||||
}
|
||||
color.set(id, GRAY);
|
||||
for (const dep of (adj.get(id) || [])) {
|
||||
dfs(dep);
|
||||
}
|
||||
color.set(id, BLACK);
|
||||
};
|
||||
|
||||
for (const s of tempServices) {
|
||||
if (color.get(s.id) === WHITE) {
|
||||
dfs(s.id);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(err.message);
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Health status
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Get the current container status for a service and all its transitive dependencies.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<DependencyStatusEntry[]>}
|
||||
* @throws {Error} If `serviceId` doesn't exist.
|
||||
*/
|
||||
async getDependencyStatus(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
const service = services.find(s => s.id === serviceId);
|
||||
if (!service) {
|
||||
throw new Error(`Service "${serviceId}" not found`);
|
||||
}
|
||||
|
||||
// Collect all transitive dependencies via BFS
|
||||
const serviceMap = new Map(services.map(s => [s.id, s]));
|
||||
const visited = new Set();
|
||||
const queue = [serviceId];
|
||||
const allRelated = [];
|
||||
|
||||
while (queue.length > 0) {
|
||||
const currentId = queue.shift();
|
||||
if (visited.has(currentId)) continue;
|
||||
visited.add(currentId);
|
||||
|
||||
const svc = serviceMap.get(currentId);
|
||||
if (!svc) continue;
|
||||
|
||||
allRelated.push(svc);
|
||||
|
||||
for (const depId of (svc.dependsOn || [])) {
|
||||
if (!visited.has(depId)) {
|
||||
queue.push(depId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Query container status for each
|
||||
const results = [];
|
||||
for (const svc of allRelated) {
|
||||
const entry = {
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
isUp: false,
|
||||
};
|
||||
|
||||
if (!svc.containerId) {
|
||||
entry.error = 'No container associated with this service';
|
||||
results.push(entry);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const container = this._docker.client.getContainer(svc.containerId);
|
||||
const info = await container.inspect();
|
||||
entry.isUp = info.State?.Running === true;
|
||||
} catch (err) {
|
||||
entry.error = err.message || 'Unable to inspect container';
|
||||
}
|
||||
|
||||
results.push(entry);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Restart with dependencies
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Wait for a container to report as running after a restart.
|
||||
*
|
||||
* @private
|
||||
* @param {string} containerId
|
||||
* @param {number} [timeoutMs=30000]
|
||||
* @returns {Promise<boolean>} `true` if healthy, `false` if timed out.
|
||||
*/
|
||||
async _waitForContainerHealthy(containerId, timeoutMs = HEALTH_CHECK_TIMEOUT_MS) {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
try {
|
||||
const container = this._docker.client.getContainer(containerId);
|
||||
const info = await container.inspect();
|
||||
if (info.State?.Running === true) {
|
||||
return true;
|
||||
}
|
||||
} catch {
|
||||
// Container might not be inspectable during restart — keep polling
|
||||
}
|
||||
await new Promise(r => setTimeout(r, HEALTH_CHECK_INTERVAL_MS));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restart a service and all its dependencies in topological order.
|
||||
*
|
||||
* Emits progress events and sends a notification on completion/failure.
|
||||
* This method is designed to be called from the route handler and
|
||||
* **does not throw** — errors are reported via events and notifications.
|
||||
*
|
||||
* @param {string} serviceId - Target service to restart (with deps).
|
||||
* @returns {Promise<{ success: boolean, chain: string[], results: Array }>}
|
||||
*/
|
||||
async restartWithDependencies(serviceId) {
|
||||
const service = await this._findService(serviceId);
|
||||
if (!service) {
|
||||
const err = new Error(`Service "${serviceId}" not found`);
|
||||
this.emit('dependency-restart-failed', {
|
||||
serviceId,
|
||||
failedServiceId: serviceId,
|
||||
error: err.message,
|
||||
chain: [],
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
|
||||
let chain;
|
||||
try {
|
||||
chain = await this.getOrderedRestartChain(serviceId);
|
||||
} catch (err) {
|
||||
this.emit('dependency-restart-failed', {
|
||||
serviceId,
|
||||
failedServiceId: serviceId,
|
||||
error: err.message,
|
||||
chain: [],
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
|
||||
const services = await this._loadServices();
|
||||
const serviceMap = new Map(services.map(s => [s.id, s]));
|
||||
|
||||
this._log.info('dependency', 'Starting dependency restart chain', {
|
||||
serviceId,
|
||||
chain,
|
||||
});
|
||||
|
||||
this.emit('dependency-restart-start', { serviceId, chain });
|
||||
|
||||
const results = [];
|
||||
const total = chain.length;
|
||||
|
||||
for (let i = 0; i < total; i++) {
|
||||
const currentId = chain[i];
|
||||
const svc = serviceMap.get(currentId);
|
||||
|
||||
this.emit('dependency-restart-progress', {
|
||||
serviceId,
|
||||
currentServiceId: currentId,
|
||||
index: i,
|
||||
total,
|
||||
});
|
||||
|
||||
if (!svc || !svc.containerId) {
|
||||
const msg = !svc
|
||||
? `Service "${currentId}" not found in state`
|
||||
: `Service "${currentId}" has no container — skipping restart`;
|
||||
this._log.warn('dependency', msg);
|
||||
results.push({ serviceId: currentId, restarted: false, skipped: true, reason: msg });
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const container = this._docker.client.getContainer(svc.containerId);
|
||||
this._log.info('dependency', `Restarting container for service "${currentId}"`, {
|
||||
containerId: svc.containerId,
|
||||
});
|
||||
await container.restart();
|
||||
|
||||
// Wait for it to come back up
|
||||
const healthy = await this._waitForContainerHealthy(svc.containerId);
|
||||
if (!healthy) {
|
||||
const msg = `Container for service "${currentId}" did not become healthy within ${HEALTH_CHECK_TIMEOUT_MS / 1000}s`;
|
||||
this._log.warn('dependency', msg);
|
||||
results.push({ serviceId: currentId, restarted: true, healthy: false, error: msg });
|
||||
|
||||
// Abort chain — dependency didn't come back
|
||||
this.emit('dependency-restart-failed', {
|
||||
serviceId,
|
||||
failedServiceId: currentId,
|
||||
error: msg,
|
||||
chain,
|
||||
});
|
||||
await this._notifyRestartResult(serviceId, false, chain, results, currentId);
|
||||
return { success: false, chain, results };
|
||||
}
|
||||
|
||||
this._log.info('dependency', `Service "${currentId}" is healthy after restart`);
|
||||
results.push({ serviceId: currentId, restarted: true, healthy: true });
|
||||
} catch (err) {
|
||||
const msg = err.message || 'Unknown error during restart';
|
||||
this._log.error('dependency', `Failed to restart service "${currentId}"`, {
|
||||
error: msg,
|
||||
});
|
||||
results.push({ serviceId: currentId, restarted: false, error: msg });
|
||||
|
||||
this.emit('dependency-restart-failed', {
|
||||
serviceId,
|
||||
failedServiceId: currentId,
|
||||
error: msg,
|
||||
chain,
|
||||
});
|
||||
await this._notifyRestartResult(serviceId, false, chain, results, currentId);
|
||||
return { success: false, chain, results };
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('dependency-restart-complete', { serviceId, chain, results });
|
||||
await this._notifyRestartResult(serviceId, true, chain, results);
|
||||
return { success: true, chain, results };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a notification about the restart result.
|
||||
*
|
||||
* @private
|
||||
* @param {string} serviceId
|
||||
* @param {boolean} success
|
||||
* @param {string[]} chain
|
||||
* @param {Array} results
|
||||
* @param {string} [failedServiceId]
|
||||
*/
|
||||
async _notifyRestartResult(serviceId, success, chain, results, failedServiceId) {
|
||||
if (!this._notification) return;
|
||||
|
||||
try {
|
||||
if (success) {
|
||||
await this._notification.send('dependency-restart-complete', {
|
||||
text: `✅ Dependency restart chain completed for "${serviceId}". Restarted: ${chain.join(' → ')}`,
|
||||
serviceId,
|
||||
chain,
|
||||
results,
|
||||
});
|
||||
} else {
|
||||
await this._notification.send('dependency-restart-failed', {
|
||||
text: `❌ Dependency restart chain failed for "${serviceId}" at "${failedServiceId}". Chain: ${chain.join(' → ')}`,
|
||||
serviceId,
|
||||
failedServiceId,
|
||||
chain,
|
||||
results,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
this._log.error('dependency', 'Failed to send restart notification', {
|
||||
error: err.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = DependencyManager;
|
||||
@@ -0,0 +1,273 @@
|
||||
/**
|
||||
* DNS Propagation Checker
|
||||
* Verifies DNS record propagation by querying multiple resolvers.
|
||||
* Runs as background jobs with configurable timeout and interval.
|
||||
*
|
||||
* @module dns-propagation
|
||||
*/
|
||||
|
||||
const dns = require('dns').promises;
|
||||
const EventEmitter = require('events');
|
||||
|
||||
/** Default verification options */
|
||||
const DEFAULT_OPTIONS = {
|
||||
timeout: 300000, // 5 minutes
|
||||
interval: 10000, // 10 seconds
|
||||
resolvers: ['1.1.1.1', '8.8.8.8', '9.9.9.9']
|
||||
};
|
||||
|
||||
/** Maximum age for stored verification results (1 hour) */
|
||||
const MAX_RESULT_AGE_MS = 3600000;
|
||||
|
||||
class DNSPropagationChecker extends EventEmitter {
|
||||
/**
|
||||
* Create a DNSPropagationChecker instance.
|
||||
* @param {Object} ctx - Shared application context
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.ctx = ctx;
|
||||
this.log = ctx.log || console;
|
||||
|
||||
/** @type {Map<string, Object>} domain → verification status */
|
||||
this.verifications = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that a DNS record has propagated by querying multiple resolvers.
|
||||
* Retries every `interval` ms until `timeout` is reached.
|
||||
*
|
||||
* @param {string} domain - The domain to check (e.g., 'test.sami')
|
||||
* @param {string} expectedIp - The expected IP address
|
||||
* @param {Object} [options={}] - Verification options
|
||||
* @param {number} [options.timeout=300000] - Maximum time to wait (ms)
|
||||
* @param {number} [options.interval=10000] - Time between retries (ms)
|
||||
* @param {string[]} [options.resolvers] - DNS resolvers to query
|
||||
* @returns {Promise<Object>} Verification result
|
||||
*/
|
||||
async verifyRecord(domain, expectedIp, options = {}) {
|
||||
const startTime = Date.now();
|
||||
const {
|
||||
timeout = DEFAULT_OPTIONS.timeout,
|
||||
interval = DEFAULT_OPTIONS.interval,
|
||||
resolvers = DEFAULT_OPTIONS.resolvers
|
||||
} = options;
|
||||
|
||||
const allResults = [];
|
||||
let propagated = false;
|
||||
|
||||
while (Date.now() - startTime < timeout) {
|
||||
const roundResults = [];
|
||||
|
||||
for (const resolver of resolvers) {
|
||||
const checkStart = Date.now();
|
||||
try {
|
||||
// Use dns.resolve4 with a custom resolver
|
||||
const resolverInstance = new dns.Resolver();
|
||||
resolverInstance.setServers([resolver]);
|
||||
resolverInstance.setTimeout(5000);
|
||||
|
||||
const addresses = await resolverInstance.resolve4(domain);
|
||||
const matched = addresses.includes(expectedIp);
|
||||
|
||||
const result = {
|
||||
resolver,
|
||||
ips: addresses,
|
||||
matched,
|
||||
checkedAt: new Date().toISOString(),
|
||||
responseTime: Date.now() - checkStart
|
||||
};
|
||||
|
||||
roundResults.push(result);
|
||||
|
||||
if (matched) {
|
||||
propagated = true;
|
||||
}
|
||||
} catch (err) {
|
||||
roundResults.push({
|
||||
resolver,
|
||||
ips: [],
|
||||
matched: false,
|
||||
checkedAt: new Date().toISOString(),
|
||||
error: err.code || err.message,
|
||||
responseTime: Date.now() - checkStart
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
allResults.push(...roundResults);
|
||||
|
||||
// Emit progress event
|
||||
this.emit('propagation-check', {
|
||||
domain,
|
||||
expectedIp,
|
||||
roundResults,
|
||||
elapsed: Date.now() - startTime,
|
||||
propagated
|
||||
});
|
||||
|
||||
if (propagated) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Wait before next attempt
|
||||
await new Promise(resolve => setTimeout(resolve, interval));
|
||||
}
|
||||
|
||||
const totalTime = Date.now() - startTime;
|
||||
|
||||
return {
|
||||
domain,
|
||||
expectedIp,
|
||||
propagated,
|
||||
results: allResults,
|
||||
totalTime,
|
||||
checkedAt: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a background DNS propagation verification.
|
||||
* Does not block — returns immediately with the job reference.
|
||||
*
|
||||
* @param {string} domain - The domain to verify
|
||||
* @param {string} expectedIp - The expected IP address
|
||||
* @param {Object} [options={}] - Verification options
|
||||
* @returns {Object} Job status object
|
||||
*/
|
||||
startVerification(domain, expectedIp, options = {}) {
|
||||
// If there's already a running verification for this domain, return it
|
||||
const existing = this.verifications.get(domain);
|
||||
if (existing && existing.status === 'running') {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const job = {
|
||||
domain,
|
||||
expectedIp,
|
||||
status: 'running',
|
||||
startedAt: new Date().toISOString(),
|
||||
progress: [],
|
||||
result: null
|
||||
};
|
||||
|
||||
this.verifications.set(domain, job);
|
||||
|
||||
// Run verification in background (non-blocking)
|
||||
this.verifyRecord(domain, expectedIp, options)
|
||||
.then(result => {
|
||||
job.status = 'completed';
|
||||
job.result = result;
|
||||
job.completedAt = new Date().toISOString();
|
||||
|
||||
if (result.propagated) {
|
||||
this.emit('propagation-complete', result);
|
||||
|
||||
if (this.ctx.notification) {
|
||||
this.ctx.notification.send('dns-propagation', {
|
||||
text: `✅ DNS record for ${domain} propagated successfully to ${expectedIp}`,
|
||||
domain,
|
||||
expectedIp,
|
||||
totalTime: result.totalTime
|
||||
}, 'success').catch(err => {
|
||||
this.log.error('dns-propagation', 'Failed to send propagation notification', {
|
||||
error: err.message
|
||||
});
|
||||
});
|
||||
}
|
||||
} else {
|
||||
this.emit('propagation-timeout', result);
|
||||
|
||||
if (this.ctx.notification) {
|
||||
this.ctx.notification.send('dns-propagation', {
|
||||
text: `⏱️ DNS propagation timeout for ${domain} — expected ${expectedIp} not found after ${Math.round(result.totalTime / 1000)}s`,
|
||||
domain,
|
||||
expectedIp,
|
||||
totalTime: result.totalTime
|
||||
}, 'warning').catch(err => {
|
||||
this.log.error('dns-propagation', 'Failed to send timeout notification', {
|
||||
error: err.message
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
job.status = 'error';
|
||||
job.error = err.message;
|
||||
job.completedAt = new Date().toISOString();
|
||||
|
||||
this.log.error('dns-propagation', `Verification failed for ${domain}`, {
|
||||
error: err.message
|
||||
});
|
||||
});
|
||||
|
||||
return job;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current verification status for a domain.
|
||||
*
|
||||
* @param {string} domain - The domain to look up
|
||||
* @returns {Object|null} Verification status or null if not found
|
||||
*/
|
||||
getVerificationStatus(domain) {
|
||||
const job = this.verifications.get(domain);
|
||||
if (!job) return null;
|
||||
return {
|
||||
domain: job.domain,
|
||||
expectedIp: job.expectedIp,
|
||||
status: job.status,
|
||||
startedAt: job.startedAt,
|
||||
completedAt: job.completedAt || null,
|
||||
result: job.result || null,
|
||||
error: job.error || null
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all recent verifications.
|
||||
*
|
||||
* @returns {Object[]} Array of verification statuses
|
||||
*/
|
||||
getAllVerifications() {
|
||||
const results = [];
|
||||
for (const [domain, job] of this.verifications.entries()) {
|
||||
results.push({
|
||||
domain,
|
||||
expectedIp: job.expectedIp,
|
||||
status: job.status,
|
||||
startedAt: job.startedAt,
|
||||
completedAt: job.completedAt || null,
|
||||
propagated: job.result?.propagated || null,
|
||||
totalTime: job.result?.totalTime || null,
|
||||
error: job.error || null
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove verifications older than 1 hour.
|
||||
*/
|
||||
cleanup() {
|
||||
const now = Date.now();
|
||||
for (const [domain, job] of this.verifications.entries()) {
|
||||
const completedAt = job.completedAt ? new Date(job.completedAt).getTime() : null;
|
||||
const startedAt = new Date(job.startedAt).getTime();
|
||||
|
||||
// Clean up completed/error jobs older than 1 hour
|
||||
// Also clean up stale running jobs that started over 2 hours ago
|
||||
const age = completedAt ? (now - completedAt) : (now - startedAt);
|
||||
const maxAge = job.status === 'running' ? MAX_RESULT_AGE_MS * 2 : MAX_RESULT_AGE_MS;
|
||||
|
||||
if (age > maxAge) {
|
||||
this.verifications.delete(domain);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = DNSPropagationChecker;
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "dashcaddy-api",
|
||||
"version": "1.6.0",
|
||||
"version": "1.9.0",
|
||||
"description": "DashCaddy API server - Dashboard backend for Docker, Caddy & DNS management",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* Auto-Restart Policy Routes
|
||||
*
|
||||
* CRUD endpoints for per-container auto-restart policies.
|
||||
* Also provides a dry-run test endpoint.
|
||||
*
|
||||
* @module routes/auto-restart
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
const { success } = require('../response-helpers');
|
||||
const { ValidationError, NotFoundError } = require('../errors');
|
||||
|
||||
/**
|
||||
* Auto-restart route factory
|
||||
*
|
||||
* @param {Object} deps - Explicit dependencies
|
||||
* @param {Object} deps.autoRestartManager - AutoRestartManager instance
|
||||
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||
* @param {Function} deps.logError - Error logging function
|
||||
* @returns {express.Router}
|
||||
*/
|
||||
module.exports = function ({ autoRestartManager, asyncHandler, logError }) {
|
||||
const router = express.Router();
|
||||
|
||||
/**
|
||||
* GET /auto-restart/policies
|
||||
* List all configured auto-restart policies.
|
||||
*/
|
||||
router.get('/policies', asyncHandler(async (_req, res) => {
|
||||
const policies = autoRestartManager.listPolicies();
|
||||
success(res, { policies });
|
||||
}, 'auto-restart-list'));
|
||||
|
||||
/**
|
||||
* GET /auto-restart/policies/:serviceId
|
||||
* Get the restart policy for a single service.
|
||||
*/
|
||||
router.get('/policies/:serviceId', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
|
||||
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
|
||||
throw new ValidationError('Invalid service ID format');
|
||||
}
|
||||
|
||||
const policy = autoRestartManager.getPolicy(serviceId);
|
||||
if (!policy) {
|
||||
throw new NotFoundError(`Auto-restart policy for "${serviceId}"`);
|
||||
}
|
||||
|
||||
success(res, { policy });
|
||||
}, 'auto-restart-get'));
|
||||
|
||||
/**
|
||||
* POST /auto-restart/policies/:serviceId
|
||||
* Create or update a restart policy.
|
||||
*
|
||||
* Body: { enabled, maxRetries, retryIntervalMs, windowMinutes }
|
||||
*/
|
||||
router.post('/policies/:serviceId', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
|
||||
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
|
||||
throw new ValidationError('Invalid service ID format');
|
||||
}
|
||||
|
||||
const { enabled, maxRetries, retryIntervalMs, windowMinutes } = req.body;
|
||||
|
||||
// Validate inputs
|
||||
if (enabled !== undefined && typeof enabled !== 'boolean') {
|
||||
throw new ValidationError('enabled must be a boolean');
|
||||
}
|
||||
if (maxRetries !== undefined) {
|
||||
if (!Number.isInteger(maxRetries) || maxRetries < 0 || maxRetries > 100) {
|
||||
throw new ValidationError('maxRetries must be an integer between 0 and 100');
|
||||
}
|
||||
}
|
||||
if (retryIntervalMs !== undefined) {
|
||||
if (!Number.isInteger(retryIntervalMs) || retryIntervalMs < 0 || retryIntervalMs > 3600000) {
|
||||
throw new ValidationError('retryIntervalMs must be an integer between 0 and 3600000');
|
||||
}
|
||||
}
|
||||
if (windowMinutes !== undefined) {
|
||||
if (!Number.isInteger(windowMinutes) || windowMinutes < 0 || windowMinutes > 1440) {
|
||||
throw new ValidationError('windowMinutes must be an integer between 0 and 1440');
|
||||
}
|
||||
}
|
||||
|
||||
const policy = await autoRestartManager.setPolicy(serviceId, {
|
||||
...(enabled !== undefined && { enabled }),
|
||||
...(maxRetries !== undefined && { maxRetries }),
|
||||
...(retryIntervalMs !== undefined && { retryIntervalMs }),
|
||||
...(windowMinutes !== undefined && { windowMinutes }),
|
||||
});
|
||||
|
||||
success(res, { policy, message: `Policy ${serviceId} saved` });
|
||||
}, 'auto-restart-set'));
|
||||
|
||||
/**
|
||||
* DELETE /auto-restart/policies/:serviceId
|
||||
* Remove a restart policy.
|
||||
*/
|
||||
router.delete('/policies/:serviceId', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
|
||||
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
|
||||
throw new ValidationError('Invalid service ID format');
|
||||
}
|
||||
|
||||
const removed = await autoRestartManager.removePolicy(serviceId);
|
||||
if (!removed) {
|
||||
throw new NotFoundError(`Auto-restart policy for "${serviceId}"`);
|
||||
}
|
||||
|
||||
success(res, { message: `Policy for "${serviceId}" removed` });
|
||||
}, 'auto-restart-delete'));
|
||||
|
||||
/**
|
||||
* POST /auto-restart/policies/:serviceId/test
|
||||
* Dry-run: simulate a restart attempt without actually restarting.
|
||||
* Returns what *would* happen given the current policy state.
|
||||
*/
|
||||
router.post('/policies/:serviceId/test', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
|
||||
if (!serviceId || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}$/.test(serviceId)) {
|
||||
throw new ValidationError('Invalid service ID format');
|
||||
}
|
||||
|
||||
const policy = autoRestartManager.getPolicy(serviceId);
|
||||
if (!policy) {
|
||||
throw new NotFoundError(`Auto-restart policy for "${serviceId}"`);
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const inCooldown = policy.cooldownUntil && now < policy.cooldownUntil;
|
||||
const wouldRetry = !inCooldown && policy.currentRetries < policy.maxRetries;
|
||||
const nextAttempt = policy.currentRetries + 1;
|
||||
|
||||
success(res, {
|
||||
dryRun: true,
|
||||
serviceId,
|
||||
policy: {
|
||||
enabled: policy.enabled,
|
||||
currentRetries: policy.currentRetries,
|
||||
maxRetries: policy.maxRetries,
|
||||
cooldownUntil: policy.cooldownUntil,
|
||||
inCooldown,
|
||||
},
|
||||
wouldRestart: policy.enabled && wouldRetry,
|
||||
wouldMaxOut: !wouldRetry && !inCooldown,
|
||||
nextAttempt: wouldRetry ? nextAttempt : null,
|
||||
message: !policy.enabled
|
||||
? 'Policy is disabled — no restart would occur'
|
||||
: inCooldown
|
||||
? `In cooldown until ${new Date(policy.cooldownUntil).toISOString()} — would skip`
|
||||
: wouldRetry
|
||||
? `Would attempt restart ${nextAttempt}/${policy.maxRetries}`
|
||||
: `Max retries (${policy.maxRetries}) already reached — would enter cooldown`,
|
||||
});
|
||||
}, 'auto-restart-test'));
|
||||
|
||||
return router;
|
||||
};
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* Config Drift Detection Routes
|
||||
*
|
||||
* API endpoints for running drift detection, reading cached reports,
|
||||
* auto-fixing drift, and controlling periodic polling.
|
||||
*
|
||||
* @module routes/config-drift
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
const { success } = require('../response-helpers');
|
||||
const { ValidationError, NotFoundError } = require('../errors');
|
||||
|
||||
/**
|
||||
* Config-drift route factory
|
||||
*
|
||||
* @param {Object} deps - Explicit dependencies
|
||||
* @param {Object} deps.driftDetector - ConfigDriftDetector instance
|
||||
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||
* @param {Function} deps.logError - Error logging function
|
||||
* @returns {express.Router}
|
||||
*/
|
||||
module.exports = function ({ driftDetector, asyncHandler, logError }) {
|
||||
const router = express.Router();
|
||||
|
||||
/**
|
||||
* GET /config-drift/report
|
||||
* Run a fresh drift detection and return the full report.
|
||||
*/
|
||||
router.get('/report', asyncHandler(async (_req, res) => {
|
||||
const report = await driftDetector.detect();
|
||||
success(res, { report });
|
||||
}, 'drift-report'));
|
||||
|
||||
/**
|
||||
* GET /config-drift/last
|
||||
* Return the last cached drift report (no re-detection).
|
||||
*/
|
||||
router.get('/last', asyncHandler(async (_req, res) => {
|
||||
if (!driftDetector.lastReport) {
|
||||
throw new NotFoundError('No cached drift report — run detection first');
|
||||
}
|
||||
|
||||
success(res, { report: driftDetector.lastReport });
|
||||
}, 'drift-last'));
|
||||
|
||||
/**
|
||||
* POST /config-drift/fix
|
||||
* Auto-fix detected drift: remove stale records, flag unknown containers.
|
||||
*/
|
||||
router.post('/fix', asyncHandler(async (_req, res) => {
|
||||
const result = await driftDetector.autoFix();
|
||||
success(res, {
|
||||
message: 'Auto-fix applied',
|
||||
staleRemoved: result.staleRemoved,
|
||||
unknownFlagged: result.unknownFlagged,
|
||||
});
|
||||
}, 'drift-fix'));
|
||||
|
||||
/**
|
||||
* POST /config-drift/polling
|
||||
* Enable or disable periodic drift detection polling.
|
||||
*
|
||||
* Body: { enabled: boolean, intervalMs?: number }
|
||||
*/
|
||||
router.post('/polling', asyncHandler(async (req, res) => {
|
||||
const { enabled, intervalMs } = req.body;
|
||||
|
||||
if (typeof enabled !== 'boolean') {
|
||||
throw new ValidationError('enabled must be a boolean');
|
||||
}
|
||||
|
||||
if (intervalMs !== undefined) {
|
||||
if (!Number.isInteger(intervalMs) || intervalMs < 10000 || intervalMs > 86400000) {
|
||||
throw new ValidationError('intervalMs must be an integer between 10000 and 86400000 (10s – 24h)');
|
||||
}
|
||||
}
|
||||
|
||||
if (enabled) {
|
||||
driftDetector.startPolling(intervalMs || 300000);
|
||||
success(res, {
|
||||
message: 'Drift polling enabled',
|
||||
intervalMs: intervalMs || 300000,
|
||||
});
|
||||
} else {
|
||||
driftDetector.stopPolling();
|
||||
success(res, { message: 'Drift polling disabled' });
|
||||
}
|
||||
}, 'drift-polling'));
|
||||
|
||||
return router;
|
||||
};
|
||||
@@ -10,9 +10,10 @@ const { success } = require('../response-helpers');
|
||||
* @param {Object} deps.docker - Docker client wrapper (client, pull methods)
|
||||
* @param {Object} deps.log - Logger instance
|
||||
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||
* @param {Object} deps.workflowEngine - WorkflowEngine instance (optional)
|
||||
* @returns {express.Router}
|
||||
*/
|
||||
module.exports = function({ docker, log, asyncHandler }) {
|
||||
module.exports = function({ docker, log, asyncHandler, workflowEngine }) {
|
||||
const router = express.Router();
|
||||
|
||||
// Helper: verify container exists before operating on it
|
||||
@@ -66,6 +67,11 @@ module.exports = function({ docker, log, asyncHandler }) {
|
||||
log.info('docker', `Pulling latest image: ${imageName}`);
|
||||
await docker.pull(imageName);
|
||||
|
||||
// Trigger pre-update workflow (backup before update)
|
||||
if (workflowEngine) {
|
||||
try { await workflowEngine.triggerEvent('pre-update', { containerId: containerId, containerName, imageName }); } catch (w) { log.warn('workflow', 'pre-update trigger failed: ' + w.message); }
|
||||
}
|
||||
|
||||
// Get current container config for recreation
|
||||
const hostConfig = containerInfo.HostConfig;
|
||||
const config = {
|
||||
@@ -135,10 +141,15 @@ module.exports = function({ docker, log, asyncHandler }) {
|
||||
}
|
||||
|
||||
success(res, {
|
||||
message: `Container ${containerName} updated successfully`,
|
||||
newContainerId: newContainerInfo.Id
|
||||
});
|
||||
}, 'container-update'));
|
||||
message: `Container ${containerName} updated successfully`,
|
||||
newContainerId: newContainerInfo.Id
|
||||
});
|
||||
|
||||
// Trigger post-update workflow
|
||||
if (workflowEngine) {
|
||||
try { await workflowEngine.triggerEvent('post-update', { containerId: containerId, containerName, imageName, newContainerId: newContainerInfo.Id }); } catch (w) { log.warn('workflow', 'post-update trigger failed: ' + w.message); }
|
||||
}
|
||||
}, 'container-update'));
|
||||
|
||||
// Check for available updates (compares local and remote image digests)
|
||||
router.get('/:id/check-update', asyncHandler(async (req, res) => {
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* Dependencies Route — REST API for service dependency tracking
|
||||
*
|
||||
* Endpoints:
|
||||
* GET /dependencies/graph Full dependency graph
|
||||
* GET /dependencies/validate Validate a proposed dep chain
|
||||
* GET /dependencies/:serviceId Direct deps for one service
|
||||
* GET /dependencies/:serviceId/chain Ordered restart chain
|
||||
* GET /dependencies/:serviceId/status Dependency health status
|
||||
* POST /dependencies/:serviceId Set dependencies
|
||||
* DELETE /dependencies/:serviceId Remove all dependencies
|
||||
* POST /dependencies/:serviceId/restart Restart with dependency chain
|
||||
*
|
||||
* @module routes/dependencies
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
const { success, error: errorResponse } = require('../response-helpers');
|
||||
const { NotFoundError, ValidationError } = require('../errors');
|
||||
|
||||
/**
|
||||
* Dependencies route factory
|
||||
*
|
||||
* @param {Object} deps - Explicit dependencies
|
||||
* @param {Object} deps.dependencyManager - DependencyManager instance
|
||||
* @param {Object} deps.servicesStateManager - State manager for services.json
|
||||
* @param {Object} deps.docker - Docker client wrapper
|
||||
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||
* @param {Function} deps.logError - Error logging function
|
||||
* @param {Function} deps.resyncHealthChecker - Health checker resync function
|
||||
* @param {Object} deps.log - Logger instance
|
||||
* @returns {express.Router}
|
||||
*/
|
||||
module.exports = function({
|
||||
dependencyManager,
|
||||
servicesStateManager,
|
||||
docker,
|
||||
asyncHandler,
|
||||
logError,
|
||||
resyncHealthChecker,
|
||||
log,
|
||||
}) {
|
||||
const router = express.Router();
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// GET /dependencies/graph — Full dependency graph
|
||||
// -------------------------------------------------------------------------
|
||||
router.get('/graph', asyncHandler(async (req, res) => {
|
||||
const graph = await dependencyManager.getDependencyGraph();
|
||||
success(res, { graph });
|
||||
}, 'dep-graph'));
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// GET /dependencies/validate — Validate a proposed dep chain (query params)
|
||||
// -------------------------------------------------------------------------
|
||||
router.get('/validate', asyncHandler(async (req, res) => {
|
||||
const { serviceId, dependsOn } = req.query;
|
||||
|
||||
if (!serviceId) {
|
||||
throw new ValidationError('serviceId query parameter is required');
|
||||
}
|
||||
|
||||
// dependsOn may be a comma-separated string or already an array
|
||||
let parsed;
|
||||
if (Array.isArray(dependsOn)) {
|
||||
parsed = dependsOn;
|
||||
} else if (typeof dependsOn === 'string' && dependsOn.length > 0) {
|
||||
parsed = dependsOn.split(',').map(s => s.trim()).filter(Boolean);
|
||||
} else {
|
||||
parsed = [];
|
||||
}
|
||||
|
||||
const result = await dependencyManager.validateDependencies(serviceId, parsed);
|
||||
success(res, result);
|
||||
}, 'dep-validate'));
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// GET /dependencies/:serviceId — Direct deps for one service
|
||||
// -------------------------------------------------------------------------
|
||||
router.get('/:serviceId', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
const dependencies = await dependencyManager.getDependencies(serviceId);
|
||||
const dependents = await dependencyManager.getDependents(serviceId);
|
||||
|
||||
// Read the service's current dependsOn array
|
||||
const services = await servicesStateManager.read();
|
||||
const allServices = Array.isArray(services) ? services : (services.services || []);
|
||||
const service = allServices.find(s => s.id === serviceId);
|
||||
|
||||
if (!service) {
|
||||
throw new NotFoundError(`Service "${serviceId}"`);
|
||||
}
|
||||
|
||||
success(res, {
|
||||
serviceId,
|
||||
dependsOn: service.dependsOn || [],
|
||||
dependencies,
|
||||
dependents: dependents.map(d => ({ id: d.id, name: d.name })),
|
||||
});
|
||||
}, 'dep-get'));
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// GET /dependencies/:serviceId/chain — Ordered restart chain
|
||||
// -------------------------------------------------------------------------
|
||||
router.get('/:serviceId/chain', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
const chain = await dependencyManager.getOrderedRestartChain(serviceId);
|
||||
success(res, { serviceId, chain });
|
||||
}, 'dep-chain'));
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// GET /dependencies/:serviceId/status — Dependency health status
|
||||
// -------------------------------------------------------------------------
|
||||
router.get('/:serviceId/status', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
const statuses = await dependencyManager.getDependencyStatus(serviceId);
|
||||
success(res, { serviceId, statuses });
|
||||
}, 'dep-status'));
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// POST /dependencies/:serviceId — Set dependencies
|
||||
// -------------------------------------------------------------------------
|
||||
router.post('/:serviceId', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
const { dependsOn } = req.body;
|
||||
|
||||
if (!Array.isArray(dependsOn)) {
|
||||
throw new ValidationError('Request body must include dependsOn as an array of service IDs');
|
||||
}
|
||||
|
||||
// Validate first
|
||||
const validation = await dependencyManager.validateDependencies(serviceId, dependsOn);
|
||||
if (!validation.valid) {
|
||||
return errorResponse(res, validation.errors.join('; '), 400);
|
||||
}
|
||||
|
||||
// Update the service
|
||||
let found = false;
|
||||
await servicesStateManager.update(services => {
|
||||
const arr = Array.isArray(services) ? services : [];
|
||||
return arr.map(s => {
|
||||
if (s.id === serviceId) {
|
||||
found = true;
|
||||
return { ...s, dependsOn: dependsOn.slice() };
|
||||
}
|
||||
return s;
|
||||
});
|
||||
});
|
||||
|
||||
if (!found) {
|
||||
throw new NotFoundError(`Service "${serviceId}"`);
|
||||
}
|
||||
|
||||
log.info('dependency', 'Dependencies updated', { serviceId, dependsOn });
|
||||
|
||||
success(res, {
|
||||
message: `Dependencies updated for "${serviceId}"`,
|
||||
serviceId,
|
||||
dependsOn,
|
||||
});
|
||||
}, 'dep-set'));
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// DELETE /dependencies/:serviceId — Remove all dependencies for a service
|
||||
// -------------------------------------------------------------------------
|
||||
router.delete('/:serviceId', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
|
||||
let found = false;
|
||||
await servicesStateManager.update(services => {
|
||||
const arr = Array.isArray(services) ? services : [];
|
||||
return arr.map(s => {
|
||||
if (s.id === serviceId) {
|
||||
found = true;
|
||||
const updated = { ...s };
|
||||
delete updated.dependsOn;
|
||||
return updated;
|
||||
}
|
||||
return s;
|
||||
});
|
||||
});
|
||||
|
||||
if (!found) {
|
||||
throw new NotFoundError(`Service "${serviceId}"`);
|
||||
}
|
||||
|
||||
log.info('dependency', 'Dependencies removed', { serviceId });
|
||||
|
||||
success(res, {
|
||||
message: `All dependencies removed for "${serviceId}"`,
|
||||
serviceId,
|
||||
});
|
||||
}, 'dep-delete'));
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// POST /dependencies/:serviceId/restart — Restart with dependency chain
|
||||
// -------------------------------------------------------------------------
|
||||
router.post('/:serviceId/restart', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
|
||||
// Verify the service exists
|
||||
const services = await servicesStateManager.read();
|
||||
const allServices = Array.isArray(services) ? services : (services.services || []);
|
||||
if (!allServices.find(s => s.id === serviceId)) {
|
||||
throw new NotFoundError(`Service "${serviceId}"`);
|
||||
}
|
||||
|
||||
// Get the chain first for the response (before async restart begins)
|
||||
let chain;
|
||||
try {
|
||||
chain = await dependencyManager.getOrderedRestartChain(serviceId);
|
||||
} catch (err) {
|
||||
return errorResponse(res, err.message, 400);
|
||||
}
|
||||
|
||||
// Respond immediately with the chain order
|
||||
success(res, {
|
||||
message: `Dependency restart initiated for "${serviceId}"`,
|
||||
serviceId,
|
||||
chain,
|
||||
});
|
||||
|
||||
// Run the restart chain asynchronously so the client doesn't block
|
||||
dependencyManager.restartWithDependencies(serviceId).catch(err => {
|
||||
if (log) {
|
||||
log.error('dependency', 'Async dependency restart failed', {
|
||||
serviceId,
|
||||
error: err.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
}, 'dep-restart'));
|
||||
|
||||
return router;
|
||||
};
|
||||
@@ -26,7 +26,8 @@ module.exports = function({
|
||||
log,
|
||||
safeErrorMessage,
|
||||
fetchT,
|
||||
credentialManager
|
||||
credentialManager,
|
||||
dnsPropagationChecker
|
||||
}) {
|
||||
const router = express.Router();
|
||||
|
||||
@@ -139,6 +140,14 @@ module.exports = function({
|
||||
});
|
||||
|
||||
if (result.status === 'ok') {
|
||||
// Start DNS propagation verification in background
|
||||
if (dnsPropagationChecker && ip) {
|
||||
const fullDomain = domain;
|
||||
dnsPropagationChecker.startVerification(fullDomain, ip).catch(err => {
|
||||
log('DNS propagation check start failed:', err.message);
|
||||
});
|
||||
}
|
||||
|
||||
success(res, { message: `DNS record ${domain} -> ${ip} created` });
|
||||
} else {
|
||||
// Error handled by middleware
|
||||
@@ -641,5 +650,68 @@ module.exports = function({
|
||||
}
|
||||
}, 'dns-update'));
|
||||
|
||||
// ===== DNS PROPAGATION =====
|
||||
|
||||
// GET /propagation — Get all recent DNS propagation checks
|
||||
router.get('/propagation', asyncHandler(async (req, res) => {
|
||||
if (!dnsPropagationChecker) {
|
||||
return success(res, { verifications: [], message: 'DNS propagation checker not available' });
|
||||
}
|
||||
|
||||
// Cleanup old entries
|
||||
dnsPropagationChecker.cleanup();
|
||||
|
||||
const verifications = dnsPropagationChecker.getAllVerifications();
|
||||
success(res, { verifications });
|
||||
}, 'dns-propagation-all'));
|
||||
|
||||
// POST /propagation/verify — Manually trigger DNS propagation verification
|
||||
router.post('/propagation/verify', asyncHandler(async (req, res) => {
|
||||
if (!dnsPropagationChecker) {
|
||||
return errorResponse(res, 'DNS propagation checker not available', 503);
|
||||
}
|
||||
|
||||
const { domain, expectedIp } = req.body;
|
||||
|
||||
if (!domain || !expectedIp) {
|
||||
throw new ValidationError('domain and expectedIp are required');
|
||||
}
|
||||
|
||||
// Validate domain format
|
||||
if (!REGEX.DOMAIN.test(domain)) {
|
||||
throw new ValidationError('[DC-301] Invalid domain format');
|
||||
}
|
||||
|
||||
// Validate IP address
|
||||
const validatorLib = require('validator');
|
||||
if (!validatorLib.isIP(expectedIp)) {
|
||||
throw new ValidationError('[DC-210] Invalid IP address');
|
||||
}
|
||||
|
||||
const job = dnsPropagationChecker.startVerification(domain, expectedIp);
|
||||
success(res, {
|
||||
message: 'DNS propagation verification started',
|
||||
domain,
|
||||
expectedIp,
|
||||
status: job.status
|
||||
});
|
||||
}, 'dns-propagation-verify'));
|
||||
|
||||
// GET /propagation/:domain — Get propagation status for a specific domain
|
||||
router.get('/propagation/:domain', asyncHandler(async (req, res) => {
|
||||
if (!dnsPropagationChecker) {
|
||||
return success(res, { verification: null, message: 'DNS propagation checker not available' });
|
||||
}
|
||||
|
||||
const { domain } = req.params;
|
||||
const status = dnsPropagationChecker.getVerificationStatus(domain);
|
||||
|
||||
if (!status) {
|
||||
throw new NotFoundError(`No propagation check found for domain: ${domain}`);
|
||||
}
|
||||
|
||||
success(res, { verification: status });
|
||||
}, 'dns-propagation-domain'));
|
||||
|
||||
return router;
|
||||
};
|
||||
|
||||
@@ -8,9 +8,10 @@ const express = require('express');
|
||||
* @param {Object} deps.healthChecker - Health checker
|
||||
* @param {Object} deps.updateManager - Update manager
|
||||
* @param {Function} deps.logError - Error logging function
|
||||
* @param {Object} deps.dependencyManager - Dependency manager for restart chain events
|
||||
* @returns {express.Router}
|
||||
*/
|
||||
module.exports = function({ resourceMonitor, healthChecker, updateManager, logError }) {
|
||||
module.exports = function({ resourceMonitor, healthChecker, updateManager, logError, dependencyManager, autoRestartManager, driftDetector, sslMonitor, dnsPropagationChecker }) {
|
||||
const router = express.Router();
|
||||
const clients = new Set();
|
||||
|
||||
@@ -74,6 +75,48 @@ module.exports = function({ resourceMonitor, healthChecker, updateManager, logEr
|
||||
});
|
||||
}
|
||||
|
||||
// Dependency manager events
|
||||
if (dependencyManager) {
|
||||
dependencyManager.on('dependency-restart-start', (data) => {
|
||||
broadcast('dependency-restart-start', data);
|
||||
});
|
||||
dependencyManager.on('dependency-restart-progress', (data) => {
|
||||
broadcast('dependency-restart-progress', data);
|
||||
});
|
||||
dependencyManager.on('dependency-restart-complete', (data) => {
|
||||
broadcast('dependency-restart-complete', data);
|
||||
});
|
||||
dependencyManager.on('dependency-restart-failed', (data) => {
|
||||
broadcast('dependency-restart-failed', data);
|
||||
});
|
||||
}
|
||||
|
||||
// Auto-restart manager events
|
||||
if (autoRestartManager) {
|
||||
autoRestartManager.on('auto-restart-attempt', (data) => broadcast('auto-restart-attempt', data));
|
||||
autoRestartManager.on('auto-restart-success', (data) => broadcast('auto-restart-success', data));
|
||||
autoRestartManager.on('auto-restart-failed', (data) => broadcast('auto-restart-failed', data));
|
||||
autoRestartManager.on('auto-restart-max-reached', (data) => broadcast('auto-restart-max-reached', data));
|
||||
}
|
||||
|
||||
// Config drift detector events
|
||||
if (driftDetector) {
|
||||
driftDetector.on('drift-detected', (data) => broadcast('drift-detected', data));
|
||||
}
|
||||
|
||||
// SSL monitor events
|
||||
if (sslMonitor) {
|
||||
sslMonitor.on('cert-expiring', (data) => broadcast('cert-expiring', data));
|
||||
sslMonitor.on('cert-critical', (data) => broadcast('cert-critical', data));
|
||||
}
|
||||
|
||||
// DNS propagation checker events
|
||||
if (dnsPropagationChecker) {
|
||||
dnsPropagationChecker.on('propagation-check', (data) => broadcast('dns-propagation-check', data));
|
||||
dnsPropagationChecker.on('propagation-complete', (data) => broadcast('dns-propagation-complete', data));
|
||||
dnsPropagationChecker.on('propagation-timeout', (data) => broadcast('dns-propagation-timeout', data));
|
||||
}
|
||||
|
||||
// SSE endpoint
|
||||
router.get('/stream', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
|
||||
@@ -372,7 +372,7 @@ module.exports = function({
|
||||
// Add a new service
|
||||
router.post('/services', asyncHandler(async (req, res) => {
|
||||
try {
|
||||
const { id, name, logo } = req.body;
|
||||
const { id, name, logo, category, containerId, port, ip, tailscaleOnly } = req.body;
|
||||
|
||||
if (!id || !name) {
|
||||
throw new ValidationError('id and name are required');
|
||||
@@ -391,7 +391,14 @@ module.exports = function({
|
||||
throw new ConflictError(`Service "${id}" already exists`, id);
|
||||
}
|
||||
|
||||
services.push({ id, name, logo: logo || `/assets/${id}.png` });
|
||||
const newService = { id, name, logo: logo || `/assets/${id}.png` };
|
||||
// Persist optional metadata fields if provided
|
||||
if (category) newService.category = category;
|
||||
if (containerId) newService.containerId = containerId;
|
||||
if (port) newService.port = port;
|
||||
if (ip) newService.ip = ip;
|
||||
if (typeof tailscaleOnly === 'boolean') newService.tailscaleOnly = tailscaleOnly;
|
||||
services.push(newService);
|
||||
return services;
|
||||
});
|
||||
|
||||
@@ -542,6 +549,8 @@ module.exports = function({
|
||||
};
|
||||
if (name) services[serviceIndex].name = name;
|
||||
if (logo) services[serviceIndex].logo = logo;
|
||||
// Allow category update via update endpoint too (optional body field)
|
||||
if (req.body.category !== undefined) services[serviceIndex].category = req.body.category || undefined;
|
||||
results.services = 'updated';
|
||||
} else {
|
||||
results.services = 'not found';
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* SSL Monitor Routes
|
||||
* REST API endpoints for SSL certificate monitoring.
|
||||
*
|
||||
* @module routes/ssl-monitor
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
const { success, error: errorResponse, notFound } = require('../response-helpers');
|
||||
|
||||
/**
|
||||
* SSL Monitor route factory
|
||||
* @param {Object} deps - Explicit dependencies
|
||||
* @param {Object} deps.sslMonitor - SSLMonitor instance
|
||||
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
||||
* @param {Function} deps.logError - Error logging function
|
||||
* @returns {express.Router}
|
||||
*/
|
||||
module.exports = function({ sslMonitor, asyncHandler, logError }) {
|
||||
const router = express.Router();
|
||||
|
||||
/**
|
||||
* GET /ssl/certificates
|
||||
* Get all SSL certificate statuses
|
||||
*/
|
||||
router.get('/certificates', asyncHandler(async (req, res) => {
|
||||
const status = sslMonitor.getStatus();
|
||||
success(res, { certificates: status });
|
||||
}, 'ssl-certificates'));
|
||||
|
||||
/**
|
||||
* GET /ssl/certificates/:serviceId
|
||||
* Get SSL certificate status for a specific service
|
||||
*/
|
||||
router.get('/certificates/:serviceId', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
const certStatus = sslMonitor.getServiceCertStatus(serviceId);
|
||||
|
||||
if (!certStatus) {
|
||||
return notFound(res, `No SSL certificate status found for service: ${serviceId}`);
|
||||
}
|
||||
|
||||
success(res, { certificate: certStatus });
|
||||
}, 'ssl-certificate-service'));
|
||||
|
||||
/**
|
||||
* POST /ssl/check
|
||||
* Trigger an on-demand check of all SSL certificates
|
||||
*/
|
||||
router.post('/check', asyncHandler(async (req, res) => {
|
||||
const results = await sslMonitor.checkAll();
|
||||
success(res, { certificates: results, message: 'SSL check completed' });
|
||||
}, 'ssl-check-all'));
|
||||
|
||||
/**
|
||||
* POST /ssl/check/:serviceId
|
||||
* Check the SSL certificate for a specific service
|
||||
*/
|
||||
router.post('/check/:serviceId', asyncHandler(async (req, res) => {
|
||||
const { serviceId } = req.params;
|
||||
|
||||
// Look up the existing cert status to find the hostname
|
||||
const existingCert = sslMonitor.getServiceCertStatus(serviceId);
|
||||
if (!existingCert) {
|
||||
return notFound(res, `No HTTPS URL found for service: ${serviceId}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await sslMonitor.checkCert(existingCert.hostname, existingCert.port);
|
||||
success(res, { certificate: { ...result, serviceId } });
|
||||
} catch (err) {
|
||||
errorResponse(res, `Failed to check SSL certificate: ${err.message}`, 500);
|
||||
}
|
||||
}, 'ssl-check-service'));
|
||||
|
||||
/**
|
||||
* GET /ssl/config
|
||||
* Get current SSL monitoring configuration
|
||||
*/
|
||||
router.get('/config', asyncHandler(async (req, res) => {
|
||||
const config = sslMonitor.getConfig();
|
||||
success(res, { config });
|
||||
}, 'ssl-config-get'));
|
||||
|
||||
/**
|
||||
* POST /ssl/config
|
||||
* Update SSL monitoring configuration
|
||||
* Body: { enabled: boolean, intervalMs: number }
|
||||
*/
|
||||
router.post('/config', asyncHandler(async (req, res) => {
|
||||
const { enabled, intervalMs } = req.body;
|
||||
|
||||
// Validate inputs
|
||||
if (enabled !== undefined && typeof enabled !== 'boolean') {
|
||||
return errorResponse(res, 'enabled must be a boolean', 400);
|
||||
}
|
||||
if (intervalMs !== undefined) {
|
||||
if (typeof intervalMs !== 'number' || intervalMs < 60000) {
|
||||
return errorResponse(res, 'intervalMs must be a number >= 60000 (1 minute)', 400);
|
||||
}
|
||||
}
|
||||
|
||||
const updates = {};
|
||||
if (enabled !== undefined) updates.enabled = enabled;
|
||||
if (intervalMs !== undefined) updates.intervalMs = intervalMs;
|
||||
|
||||
sslMonitor.updateConfig(updates);
|
||||
const config = sslMonitor.getConfig();
|
||||
success(res, { config, message: 'SSL monitoring config updated' });
|
||||
}, 'ssl-config-update'));
|
||||
|
||||
return router;
|
||||
};
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# DashCaddy Host-Side Updater
|
||||
# Triggered by systemd path unit when the container writes trigger.json.
|
||||
# Reads the trigger, backs up current API, copies new files, rebuilds container.
|
||||
# Reads the trigger, backs up current API + data/, copies new files, rebuilds container.
|
||||
# Writes result.json so the new container knows the outcome.
|
||||
#
|
||||
# This runs on the HOST, outside the container.
|
||||
@@ -16,6 +16,10 @@ readonly CONTAINER_NAME="dashcaddy-api"
|
||||
readonly MAX_BACKUPS=3
|
||||
readonly HEALTH_TIMEOUT=60
|
||||
|
||||
# Data directory backup — stored alongside code backups so everything rolls back together
|
||||
readonly DATA_SOURCE_DIR="/opt/dashcaddy/dashcaddy-api/data"
|
||||
readonly DATA_BACKUP_PREFIX="data-backup"
|
||||
|
||||
log() { echo "[dashcaddy-update] $(date '+%Y-%m-%d %H:%M:%S') $*"; }
|
||||
|
||||
write_result() {
|
||||
@@ -56,6 +60,34 @@ cleanup_old_backups() {
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Data backup (rsync for efficiency + permissions) ──────────────────────────
|
||||
backup_data_dir() {
|
||||
local backup_dir="$1"
|
||||
if [[ -d "$DATA_SOURCE_DIR" ]]; then
|
||||
log "Backing up data/ to ${backup_dir}/${DATA_BACKUP_PREFIX}/"
|
||||
mkdir -p "${backup_dir}/${DATA_BACKUP_PREFIX}"
|
||||
rsync -a --delete "$DATA_SOURCE_DIR/" "${backup_dir}/${DATA_BACKUP_PREFIX}/" 2>/dev/null \
|
||||
|| cp -a "$DATA_SOURCE_DIR" "${backup_dir}/${DATA_BACKUP_PREFIX}"
|
||||
log "Data backup complete ($(du -sh "${backup_dir}/${DATA_BACKUP_PREFIX}" 2>/dev/null | cut -f1))"
|
||||
else
|
||||
log "WARNING: Data source dir $DATA_SOURCE_DIR not found — skipping data backup"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Data restore ──────────────────────────────────────────────────────────────
|
||||
restore_data_dir() {
|
||||
local backup_dir="$1"
|
||||
local data_backup="${backup_dir}/${DATA_BACKUP_PREFIX}"
|
||||
if [[ -d "$data_backup" ]]; then
|
||||
log "Restoring data/ from backup..."
|
||||
rsync -a --delete "$data_backup/" "$DATA_SOURCE_DIR/" 2>/dev/null \
|
||||
|| cp -a "$data_backup" "$DATA_SOURCE_DIR"
|
||||
log "Data restored successfully"
|
||||
else
|
||||
log "WARNING: No data backup found at ${data_backup} — data/ not restored"
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
local port="${1:-3001}"
|
||||
local timeout="$HEALTH_TIMEOUT"
|
||||
@@ -75,6 +107,59 @@ wait_for_health() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Shared rollback: restore code + data ────────────────────────────────────
|
||||
rollback_restore() {
|
||||
local backup_dir="$1"
|
||||
log "Rolling back: restoring code files..."
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
restore_data_dir "$backup_dir"
|
||||
}
|
||||
|
||||
# ── Shared container restart (preserves SERVICES_FILE env var) ───────────────
|
||||
# Uses rm + run so new env vars (e.g. SERVICES_FILE) take effect.
|
||||
# If docker-compose is not configured, falls back to docker start.
|
||||
restart_container() {
|
||||
local image="$1"
|
||||
log "Restarting container (rm + run to pick up env vars)..."
|
||||
# Stop and remove existing container so new env var is applied
|
||||
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
||||
|
||||
# Re-create with same volumes and the SERVICES_FILE env var
|
||||
docker run -d --restart unless-stopped --name "$CONTAINER_NAME" \
|
||||
-p 127.0.0.1:3001:3001 \
|
||||
-v /opt/dashcaddy/dashcaddy-api/data:/app/data \
|
||||
-e SERVICES_FILE=/app/data/services.json \
|
||||
"$image"
|
||||
log "Container restarted with fresh env"
|
||||
}
|
||||
|
||||
# ── Code-only restore (used after failed build when data hasn't changed yet) ──
|
||||
code_restore() {
|
||||
local backup_dir="$1"
|
||||
log "Restoring code files..."
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
local start_time
|
||||
start_time=$(date +%s)
|
||||
@@ -94,45 +179,69 @@ main() {
|
||||
staging_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}'))['stagingDir'])")
|
||||
api_source_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}'))['apiSourceDir'])")
|
||||
commit=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('commit') or '')")
|
||||
# Frontend paths — optional (older self-updaters don't write these). When
|
||||
# present, this script also syncs the dashboard files (Caddy serves them
|
||||
# directly from the host; the container path /app/dashboard isn't mounted).
|
||||
frontend_staging_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('frontendStagingDir') or '')")
|
||||
frontend_target_dir=$(python3 -c "import json; print(json.load(open('${TRIGGER_FILE}')).get('frontendTargetDir') or '')")
|
||||
# Handle action=rollback (no new version to deploy)
|
||||
local to_version="${version}"
|
||||
|
||||
log "=== ${action^^}: v${from_version} -> v${version} ==="
|
||||
log "=== ${action^^}: v${from_version} -> v${to_version} ==="
|
||||
log "Staging: ${staging_dir}"
|
||||
log "API source: ${api_source_dir}"
|
||||
|
||||
# Consume the trigger immediately so we don't re-process on failure
|
||||
mv "$TRIGGER_FILE" "${TRIGGER_FILE}.processing"
|
||||
|
||||
# 2. Validate staging directory
|
||||
# ── Handle rollback ────────────────────────────────────────────────────────
|
||||
if [[ "$action" == "rollback" ]]; then
|
||||
local backup_dir="${BACKUPS_DIR}/${version}"
|
||||
if [[ ! -d "$backup_dir" ]]; then
|
||||
log "ERROR: No backup found for version ${version}"
|
||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "No backup found for version ${version}"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Performing rollback to v${version}..."
|
||||
rollback_restore "$backup_dir"
|
||||
|
||||
# Rebuild old code
|
||||
log "Rebuilding container..."
|
||||
cd "$api_source_dir"
|
||||
docker build -t dashcaddy-dashcaddy-api:latest . 2>&1 | tail -1 || true
|
||||
|
||||
restart_container "dashcaddy-dashcaddy-api:latest"
|
||||
wait_for_health || log "WARNING: Health check failed after rollback"
|
||||
|
||||
write_result "true" "$version" "$(( $(date +%s) - start_time ))"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
log "=== Rollback complete ==="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Handle update ───────────────────────────────────────────────────────────
|
||||
if [[ ! -d "$staging_dir" ]]; then
|
||||
log "ERROR: Staging directory not found: ${staging_dir}"
|
||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "Staging directory not found"
|
||||
write_result "false" "$to_version" "$(( $(date +%s) - start_time ))" "Staging directory not found"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 3. Backup current API files
|
||||
# 2. Backup current API code + data/
|
||||
local backup_dir="${BACKUPS_DIR}/${from_version}"
|
||||
mkdir -p "$backup_dir"
|
||||
log "Backing up current API files to ${backup_dir}"
|
||||
|
||||
# Copy all JS files, package.json, Dockerfile, and tracked subdirs
|
||||
for item in "$api_source_dir"/*.js "$api_source_dir"/package.json "$api_source_dir"/package-lock.json "$api_source_dir"/Dockerfile "$api_source_dir"/openapi.yaml "$api_source_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$backup_dir/" 2>/dev/null || true
|
||||
done
|
||||
[[ -d "$api_source_dir/routes" ]] && cp -rf "$api_source_dir/routes" "$backup_dir/"
|
||||
[[ -d "$api_source_dir/src" ]] && cp -rf "$api_source_dir/src" "$backup_dir/"
|
||||
# VERSION (commit hash) was copied from api_source_dir above; preserve as-is
|
||||
# so a rollback restores the original commit marker. The version *string* is
|
||||
# already encoded in the backup dir name (${from_version}).
|
||||
|
||||
# Backup data/ directory (services.json, config.json, credentials, etc.)
|
||||
backup_data_dir "$backup_dir"
|
||||
|
||||
cleanup_old_backups
|
||||
|
||||
# 4. Copy new files from staging to API source
|
||||
# 3. Copy new files from staging to API source
|
||||
log "Deploying new API files..."
|
||||
for item in "$staging_dir"/*.js "$staging_dir"/package.json "$staging_dir"/package-lock.json "$staging_dir"/Dockerfile "$staging_dir"/openapi.yaml "$staging_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
@@ -145,19 +254,11 @@ main() {
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$staging_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
|
||||
# Belt-and-suspenders: always write the commit from trigger.json to VERSION,
|
||||
# even if the tarball didn't include one. The container's self-updater uses
|
||||
# this to detect the "same version, different commit" case.
|
||||
if [[ -n "$commit" ]]; then
|
||||
echo "$commit" > "$api_source_dir/VERSION"
|
||||
fi
|
||||
|
||||
# 4b. Sync frontend. Caddy serves the dashboard directly from the host
|
||||
# filesystem; the container-side copy in older self-updater.js builds wrote
|
||||
# to /app/dashboard which isn't always mounted, so the real sync happens
|
||||
# here. Trigger fields take precedence; if absent (older self-updater),
|
||||
# fall back to: staging dir's sibling status/ + first existing known target.
|
||||
# 3b. Sync frontend
|
||||
if [[ -z "$frontend_staging_dir" ]]; then
|
||||
parent_staging=$(dirname "$staging_dir")
|
||||
[[ -d "$parent_staging/status" ]] && frontend_staging_dir="$parent_staging/status"
|
||||
@@ -175,107 +276,55 @@ main() {
|
||||
for sub in dist css vendor js; do
|
||||
if [[ -d "$frontend_staging_dir/$sub" ]]; then
|
||||
mkdir -p "$frontend_target_dir/$sub"
|
||||
cp -rf "$frontend_staging_dir/$sub"/* "$frontend_target_dir/$sub/" 2>/dev/null || true
|
||||
cp -rf "$frontend_staging_dir/$sub/"* "$frontend_target_dir/$sub/" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
# assets/ is mounted into the container; usually already in sync via bind
|
||||
# mount, but if a release ships new assets we want them on disk too.
|
||||
if [[ -d "$frontend_staging_dir/assets" ]]; then
|
||||
mkdir -p "$frontend_target_dir/assets"
|
||||
cp -rf "$frontend_staging_dir/assets"/* "$frontend_target_dir/assets/" 2>/dev/null || true
|
||||
cp -rf "$frontend_staging_dir/assets/"* "$frontend_target_dir/assets/" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# 5. Rebuild container
|
||||
# 4. Rebuild container
|
||||
log "Rebuilding container..."
|
||||
cd "$api_source_dir"
|
||||
|
||||
local build_ok=false
|
||||
if docker compose build --quiet 2>&1; then
|
||||
build_ok=true
|
||||
elif docker-compose build --quiet 2>&1; then
|
||||
local image_tag="dashcaddy-dashcaddy-api:latest"
|
||||
|
||||
if docker build -t "$image_tag" . 2>&1; then
|
||||
build_ok=true
|
||||
fi
|
||||
|
||||
if [[ "$build_ok" != "true" ]]; then
|
||||
log "ERROR: Docker build failed — rolling back"
|
||||
|
||||
# Restore backup
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
|
||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "Docker build failed"
|
||||
log "ERROR: Docker build failed — rolling back code + data"
|
||||
code_restore "$backup_dir"
|
||||
docker build -t "$image_tag" . 2>&1 | tail -3 || true
|
||||
restart_container "$image_tag"
|
||||
wait_for_health || true
|
||||
write_result "false" "$to_version" "$(( $(date +%s) - start_time ))" "Docker build failed"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 6. Restart container
|
||||
log "Restarting container..."
|
||||
if docker compose up -d 2>&1 || docker-compose up -d 2>&1; then
|
||||
log "Container restarted"
|
||||
else
|
||||
log "ERROR: Container restart failed — rolling back"
|
||||
# 5. Restart container (rm + run so new env vars take effect)
|
||||
restart_container "$image_tag"
|
||||
|
||||
# Restore backup
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
|
||||
docker compose build --quiet 2>&1 || docker-compose build --quiet 2>&1 || true
|
||||
docker compose up -d 2>&1 || docker-compose up -d 2>&1 || true
|
||||
|
||||
write_result "false" "$version" "$(( $(date +%s) - start_time ))" "Container restart failed"
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 7. Health check
|
||||
# 6. Health check
|
||||
if wait_for_health; then
|
||||
local duration=$(( $(date +%s) - start_time ))
|
||||
log "=== Update successful: v${version} in ${duration}s ==="
|
||||
write_result "true" "$version" "$duration"
|
||||
log "=== Update successful: v${to_version} in ${duration}s ==="
|
||||
write_result "true" "$to_version" "$duration"
|
||||
else
|
||||
local duration=$(( $(date +%s) - start_time ))
|
||||
log "ERROR: Health check failed after update — rolling back"
|
||||
|
||||
# Restore backup
|
||||
for item in "$backup_dir"/*.js "$backup_dir"/package.json "$backup_dir"/package-lock.json "$backup_dir"/Dockerfile "$backup_dir"/openapi.yaml "$backup_dir"/VERSION; do
|
||||
[[ -f "$item" ]] && cp -f "$item" "$api_source_dir/" 2>/dev/null || true
|
||||
done
|
||||
if [[ -d "$backup_dir/routes" ]]; then
|
||||
rm -rf "$api_source_dir/routes"
|
||||
cp -rf "$backup_dir/routes" "$api_source_dir/routes"
|
||||
fi
|
||||
if [[ -d "$backup_dir/src" ]]; then
|
||||
rm -rf "$api_source_dir/src"
|
||||
cp -rf "$backup_dir/src" "$api_source_dir/src"
|
||||
fi
|
||||
|
||||
docker compose build --quiet 2>&1 || docker-compose build --quiet 2>&1 || true
|
||||
docker compose up -d 2>&1 || docker-compose up -d 2>&1 || true
|
||||
log "ERROR: Health check failed after update — rolling back code + data"
|
||||
rollback_restore "$backup_dir"
|
||||
docker build -t "$image_tag" . 2>&1 | tail -3 || true
|
||||
restart_container "$image_tag"
|
||||
wait_for_health || log "WARNING: Rollback health check also failed"
|
||||
|
||||
write_result "false" "$version" "$duration" "Health check failed after update"
|
||||
write_result "false" "$to_version" "$duration" "Health check failed after update"
|
||||
fi
|
||||
|
||||
# 8. Cleanup
|
||||
# 7. Cleanup
|
||||
rm -f "${TRIGGER_FILE}.processing"
|
||||
rm -rf "${UPDATES_DIR}/staging" 2>/dev/null || true
|
||||
|
||||
|
||||
@@ -77,6 +77,15 @@ const themesRoutes = require('../routes/themes');
|
||||
const dockerResourcesRoutes = require('../routes/docker-resources');
|
||||
const eventsRoutes = require('../routes/events');
|
||||
const workflowsRoutes = require('../routes/workflows');
|
||||
const dependenciesRoutes = require('../routes/dependencies');
|
||||
const DependencyManager = require('../dependency-manager');
|
||||
const autoRestartRoutes = require('../routes/auto-restart');
|
||||
const configDriftRoutes = require('../routes/config-drift');
|
||||
const sslMonitorRoutes = require('../routes/ssl-monitor');
|
||||
const { AutoRestartManager } = require('../auto-restart-manager');
|
||||
const { ConfigDriftDetector } = require('../config-drift-detector');
|
||||
const { SSLMonitor } = require('../ssl-monitor');
|
||||
const { DNSPropagationChecker } = require('../dns-propagation');
|
||||
|
||||
// Constants
|
||||
const { APP } = require('../constants');
|
||||
@@ -335,6 +344,39 @@ async function createApp() {
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize dependency manager
|
||||
const dependencyManager = new DependencyManager({
|
||||
servicesStateManager,
|
||||
docker: ctx.docker,
|
||||
notification: ctx.notification,
|
||||
log,
|
||||
});
|
||||
ctx.dependencyManager = dependencyManager;
|
||||
log.info('app', 'Dependency manager initialized');
|
||||
|
||||
// Initialize auto-restart manager
|
||||
const autoRestartManager = new AutoRestartManager(ctx);
|
||||
ctx.autoRestartManager = autoRestartManager;
|
||||
autoRestartManager.start();
|
||||
log.info('app', 'Auto-restart manager initialized');
|
||||
|
||||
// Initialize config drift detector
|
||||
const driftDetector = new ConfigDriftDetector(ctx);
|
||||
ctx.driftDetector = driftDetector;
|
||||
driftDetector.startPolling(300000); // 5 min
|
||||
log.info('app', 'Config drift detector initialized');
|
||||
|
||||
// Initialize SSL monitor
|
||||
const sslMonitor = new SSLMonitor(ctx);
|
||||
ctx.sslMonitor = sslMonitor;
|
||||
sslMonitor.start(3600000); // 1 hour
|
||||
log.info('app', 'SSL monitor initialized');
|
||||
|
||||
// Initialize DNS propagation checker
|
||||
const dnsPropagationChecker = new DNSPropagationChecker(ctx);
|
||||
ctx.dnsPropagationChecker = dnsPropagationChecker;
|
||||
log.info('app', 'DNS propagation checker initialized');
|
||||
|
||||
// Build versioned API router
|
||||
const apiRouter = express.Router();
|
||||
|
||||
@@ -375,7 +417,8 @@ async function createApp() {
|
||||
log: ctx.log,
|
||||
safeErrorMessage: ctx.safeErrorMessage,
|
||||
fetchT: ctx.fetchT,
|
||||
credentialManager: ctx.credentialManager
|
||||
credentialManager: ctx.credentialManager,
|
||||
dnsPropagationChecker: ctx.dnsPropagationChecker
|
||||
}));
|
||||
apiRouter.use('/notifications', notificationRoutes({
|
||||
notification: ctx.notification,
|
||||
@@ -384,7 +427,8 @@ async function createApp() {
|
||||
apiRouter.use('/containers', containerRoutes({
|
||||
docker: ctx.docker,
|
||||
log: ctx.log,
|
||||
asyncHandler: ctx.asyncHandler
|
||||
asyncHandler: ctx.asyncHandler,
|
||||
workflowEngine: ctx.workflowEngine
|
||||
}));
|
||||
apiRouter.use(serviceRoutes({
|
||||
servicesStateManager: ctx.servicesStateManager,
|
||||
@@ -488,13 +532,42 @@ async function createApp() {
|
||||
resourceMonitor: ctx.resourceMonitor,
|
||||
healthChecker: ctx.healthChecker,
|
||||
updateManager: ctx.updateManager,
|
||||
logError: ctx.logError
|
||||
logError: ctx.logError,
|
||||
dependencyManager: ctx.dependencyManager,
|
||||
autoRestartManager: ctx.autoRestartManager,
|
||||
driftDetector: ctx.driftDetector,
|
||||
sslMonitor: ctx.sslMonitor,
|
||||
dnsPropagationChecker: ctx.dnsPropagationChecker
|
||||
}));
|
||||
apiRouter.use(workflowsRoutes({
|
||||
workflowEngine: ctx.workflowEngine,
|
||||
licenseManager: ctx.licenseManager,
|
||||
asyncHandler: ctx.asyncHandler
|
||||
}));
|
||||
apiRouter.use('/dependencies', dependenciesRoutes({
|
||||
dependencyManager: ctx.dependencyManager,
|
||||
servicesStateManager: ctx.servicesStateManager,
|
||||
docker: ctx.docker,
|
||||
asyncHandler: ctx.asyncHandler,
|
||||
logError: ctx.logError,
|
||||
resyncHealthChecker: ctx.resyncHealthChecker,
|
||||
log: ctx.log,
|
||||
}));
|
||||
apiRouter.use(autoRestartRoutes({
|
||||
autoRestartManager: ctx.autoRestartManager,
|
||||
asyncHandler: ctx.asyncHandler,
|
||||
logError: ctx.logError,
|
||||
}));
|
||||
apiRouter.use(configDriftRoutes({
|
||||
driftDetector: ctx.driftDetector,
|
||||
asyncHandler: ctx.asyncHandler,
|
||||
logError: ctx.logError,
|
||||
}));
|
||||
apiRouter.use(sslMonitorRoutes({
|
||||
sslMonitor: ctx.sslMonitor,
|
||||
asyncHandler: ctx.asyncHandler,
|
||||
logError: ctx.logError,
|
||||
}));
|
||||
|
||||
// Inline API routes
|
||||
apiRouter.get('/health', (req, res) => {
|
||||
|
||||
@@ -0,0 +1,411 @@
|
||||
/**
|
||||
* SSL Certificate Monitor
|
||||
* Periodically checks SSL certificates on services with HTTPS URLs.
|
||||
* Alerts at 30, 14, and 7 days before expiry.
|
||||
*
|
||||
* @module ssl-monitor
|
||||
*/
|
||||
|
||||
const tls = require('tls');
|
||||
const EventEmitter = require('events');
|
||||
const path = require('path');
|
||||
const { readJsonFile, writeJsonFile } = require('./fs-helpers');
|
||||
const { resolveServiceUrl } = require('./url-resolver');
|
||||
|
||||
/** Default check interval: 1 hour */
|
||||
const DEFAULT_INTERVAL_MS = 3600000;
|
||||
|
||||
/** Alert thresholds in days */
|
||||
const THRESHOLDS = {
|
||||
WARNING: 30,
|
||||
URGENT: 14,
|
||||
CRITICAL: 7
|
||||
};
|
||||
|
||||
/** TLS connection timeout in milliseconds */
|
||||
const TLS_TIMEOUT_MS = 10000;
|
||||
|
||||
class SSLMonitor extends EventEmitter {
|
||||
/**
|
||||
* Create an SSLMonitor instance.
|
||||
* @param {Object} ctx - Shared application context
|
||||
* @param {Object} ctx.servicesStateManager - State manager for reading services
|
||||
* @param {Function} ctx.buildServiceUrl - URL builder helper
|
||||
* @param {Object} ctx.siteConfig - Site configuration
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
* @param {string} [ctx.SSL_CACHE_FILE] - Path to persist SSL cache
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.ctx = ctx;
|
||||
this.log = ctx.log || console;
|
||||
|
||||
/** @type {Map<string, Object>} hostname → last cert check result */
|
||||
this.certStatus = new Map();
|
||||
|
||||
/** @type {Map<string, number>} hostname → last notified threshold level */
|
||||
this.notifiedThresholds = new Map();
|
||||
|
||||
/** @type {Map<string, string>} hostname → service ID mapping */
|
||||
this.hostnameToServiceId = new Map();
|
||||
|
||||
/** @type {NodeJS.Timeout|null} */
|
||||
this.intervalHandle = null;
|
||||
|
||||
/** Current config */
|
||||
this.config = {
|
||||
enabled: true,
|
||||
intervalMs: DEFAULT_INTERVAL_MS
|
||||
};
|
||||
|
||||
/** Cache file path */
|
||||
this.cacheFile = ctx.SSL_CACHE_FILE ||
|
||||
path.join(path.dirname(ctx.SERVICES_FILE || './data'), 'ssl-cache.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the SSL certificate for a given hostname and port.
|
||||
* Connects via TLS with rejectUnauthorized: false to retrieve certificate info.
|
||||
*
|
||||
* @param {string} hostname - The hostname to check
|
||||
* @param {number} [port=443] - The port to connect to
|
||||
* @returns {Promise<Object>} Certificate information
|
||||
*/
|
||||
async checkCert(hostname, port = 443) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = tls.connect({
|
||||
host: hostname,
|
||||
port,
|
||||
rejectUnauthorized: false,
|
||||
servername: hostname,
|
||||
timeout: TLS_TIMEOUT_MS
|
||||
}, () => {
|
||||
try {
|
||||
const cert = socket.getPeerCertificate();
|
||||
|
||||
if (!cert || Object.keys(cert).length === 0) {
|
||||
socket.destroy();
|
||||
return reject(new Error(`No certificate returned for ${hostname}:${port}`));
|
||||
}
|
||||
|
||||
const validFrom = new Date(cert.valid_from);
|
||||
const validTo = new Date(cert.valid_to);
|
||||
const now = new Date();
|
||||
const msRemaining = validTo.getTime() - now.getTime();
|
||||
const daysRemaining = Math.ceil(msRemaining / (1000 * 60 * 60 * 24));
|
||||
|
||||
const result = {
|
||||
hostname,
|
||||
port,
|
||||
subject: cert.subject?.CN || cert.subject?.O || 'Unknown',
|
||||
issuer: cert.issuer?.CN || cert.issuer?.O || 'Unknown',
|
||||
validFrom: cert.valid_from,
|
||||
validTo: cert.valid_to,
|
||||
daysRemaining,
|
||||
fingerprint: cert.fingerprint || null,
|
||||
isExpiring: daysRemaining <= THRESHOLDS.WARNING,
|
||||
checkedAt: new Date().toISOString()
|
||||
};
|
||||
|
||||
socket.destroy();
|
||||
resolve(result);
|
||||
} catch (err) {
|
||||
socket.destroy();
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('error', (err) => {
|
||||
reject(new Error(`TLS connect error for ${hostname}:${port}: ${err.message}`));
|
||||
});
|
||||
|
||||
socket.setTimeout(TLS_TIMEOUT_MS, () => {
|
||||
socket.destroy(new Error(`TLS connection timeout for ${hostname}:${port}`));
|
||||
reject(new Error(`TLS connection timeout for ${hostname}:${port}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check SSL certificates for all services that have HTTPS URLs.
|
||||
* Reads services from ctx.servicesStateManager, resolves URLs, and checks each HTTPS cert.
|
||||
*
|
||||
* @returns {Promise<Object>} Map of hostname → cert status
|
||||
*/
|
||||
async checkAll() {
|
||||
if (!this.config.enabled) {
|
||||
this.log.info('ssl-monitor', 'SSL monitoring is disabled, skipping check');
|
||||
return this.getStatus();
|
||||
}
|
||||
|
||||
let servicesData;
|
||||
try {
|
||||
servicesData = await this.ctx.servicesStateManager.read();
|
||||
} catch (err) {
|
||||
this.log.error('ssl-monitor', 'Failed to read services', { error: err.message });
|
||||
return this.getStatus();
|
||||
}
|
||||
|
||||
const services = Array.isArray(servicesData) ? servicesData : (servicesData.services || []);
|
||||
|
||||
for (const service of services) {
|
||||
const serviceId = service.id || service.name?.toLowerCase();
|
||||
if (!serviceId) continue;
|
||||
|
||||
try {
|
||||
const url = resolveServiceUrl(serviceId, service, this.ctx.siteConfig, this.ctx.buildServiceUrl);
|
||||
if (!url) continue;
|
||||
|
||||
const parsed = new URL(url);
|
||||
if (parsed.protocol !== 'https:') continue;
|
||||
|
||||
const hostname = parsed.hostname;
|
||||
const port = parseInt(parsed.port) || 443;
|
||||
|
||||
// Map hostname back to service ID
|
||||
this.hostnameToServiceId.set(hostname, serviceId);
|
||||
|
||||
const result = await this.checkCert(hostname, port);
|
||||
|
||||
// Store result
|
||||
this.certStatus.set(hostname, result);
|
||||
|
||||
// Emit check event
|
||||
this.emit('cert-check', { serviceId, hostname, result });
|
||||
|
||||
// Check alert thresholds
|
||||
await this._checkAndNotify(hostname, result, serviceId);
|
||||
} catch (err) {
|
||||
this.log.warn('ssl-monitor', `Failed to check cert for service ${serviceId}`, {
|
||||
error: err.message
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Persist results
|
||||
await this._saveCache();
|
||||
|
||||
return this.getStatus();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic SSL certificate checking.
|
||||
*
|
||||
* @param {number} [intervalMs=3600000] - Check interval in milliseconds
|
||||
*/
|
||||
start(intervalMs) {
|
||||
if (intervalMs !== undefined) {
|
||||
this.config.intervalMs = intervalMs;
|
||||
}
|
||||
if (this.intervalHandle) {
|
||||
this.log.warn('ssl-monitor', 'SSL monitor is already running');
|
||||
return;
|
||||
}
|
||||
|
||||
this.config.enabled = true;
|
||||
|
||||
// Load cached data
|
||||
this._loadCache().catch(err => {
|
||||
this.log.warn('ssl-monitor', 'Failed to load SSL cache', { error: err.message });
|
||||
});
|
||||
|
||||
// Initial check (non-blocking)
|
||||
this.checkAll().catch(err => {
|
||||
this.log.error('ssl-monitor', 'Initial SSL check failed', { error: err.message });
|
||||
});
|
||||
|
||||
// Schedule periodic checks
|
||||
this.intervalHandle = setInterval(() => {
|
||||
this.checkAll().catch(err => {
|
||||
this.log.error('ssl-monitor', 'Periodic SSL check failed', { error: err.message });
|
||||
});
|
||||
}, this.config.intervalMs);
|
||||
|
||||
this.log.info('ssl-monitor', 'SSL monitoring started', {
|
||||
intervalMs: this.config.intervalMs
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop periodic SSL certificate checking.
|
||||
*/
|
||||
stop() {
|
||||
if (this.intervalHandle) {
|
||||
clearInterval(this.intervalHandle);
|
||||
this.intervalHandle = null;
|
||||
}
|
||||
this.config.enabled = false;
|
||||
this.log.info('ssl-monitor', 'SSL monitoring stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current SSL certificate status for all checked hostnames.
|
||||
*
|
||||
* @returns {Object} Map of hostname → cert status
|
||||
*/
|
||||
getStatus() {
|
||||
const status = {};
|
||||
for (const [hostname, cert] of this.certStatus.entries()) {
|
||||
status[hostname] = { ...cert };
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the SSL certificate status for a specific service.
|
||||
*
|
||||
* @param {string} serviceId - The service ID to look up
|
||||
* @returns {Object|null} Certificate status or null if not found
|
||||
*/
|
||||
getServiceCertStatus(serviceId) {
|
||||
// Find hostname mapped to this service
|
||||
for (const [hostname, id] of this.hostnameToServiceId.entries()) {
|
||||
if (id === serviceId) {
|
||||
const cert = this.certStatus.get(hostname);
|
||||
return cert ? { ...cert, serviceId } : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current monitoring configuration.
|
||||
*
|
||||
* @returns {Object} Config with interval and enabled state
|
||||
*/
|
||||
getConfig() {
|
||||
return { ...this.config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update monitoring configuration.
|
||||
*
|
||||
* @param {Object} updates - Config updates
|
||||
* @param {boolean} [updates.enabled] - Enable/disable monitoring
|
||||
* @param {number} [updates.intervalMs] - Check interval in milliseconds
|
||||
*/
|
||||
updateConfig(updates) {
|
||||
if (typeof updates.enabled === 'boolean') {
|
||||
this.config.enabled = updates.enabled;
|
||||
if (!updates.enabled && this.intervalHandle) {
|
||||
this.stop();
|
||||
}
|
||||
}
|
||||
if (typeof updates.intervalMs === 'number' && updates.intervalMs >= 60000) {
|
||||
this.config.intervalMs = updates.intervalMs;
|
||||
// Restart interval if running
|
||||
if (this.intervalHandle) {
|
||||
clearInterval(this.intervalHandle);
|
||||
this.intervalHandle = setInterval(() => {
|
||||
this.checkAll().catch(err => {
|
||||
this.log.error('ssl-monitor', 'Periodic SSL check failed', { error: err.message });
|
||||
});
|
||||
}, this.config.intervalMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Private Methods =====
|
||||
|
||||
/**
|
||||
* Check alert thresholds and send notifications if thresholds are crossed.
|
||||
* Only sends one notification per threshold per hostname.
|
||||
*
|
||||
* @param {string} hostname
|
||||
* @param {Object} certResult
|
||||
* @param {string} serviceId
|
||||
*/
|
||||
async _checkAndNotify(hostname, certResult, serviceId) {
|
||||
const { daysRemaining } = certResult;
|
||||
const key = hostname;
|
||||
const lastNotified = this.notifiedThresholds.get(key) || Infinity;
|
||||
|
||||
let level = null;
|
||||
let eventType = null;
|
||||
let message = null;
|
||||
|
||||
if (daysRemaining <= THRESHOLDS.CRITICAL) {
|
||||
level = THRESHOLDS.CRITICAL;
|
||||
eventType = 'cert-critical';
|
||||
message = `🔒 CRITICAL: SSL certificate for ${hostname} expires in ${daysRemaining} days!`;
|
||||
} else if (daysRemaining <= THRESHOLDS.URGENT) {
|
||||
level = THRESHOLDS.URGENT;
|
||||
eventType = 'cert-expiring';
|
||||
message = `⚠️ URGENT: SSL certificate for ${hostname} expires in ${daysRemaining} days`;
|
||||
} else if (daysRemaining <= THRESHOLDS.WARNING) {
|
||||
level = THRESHOLDS.WARNING;
|
||||
eventType = 'cert-expiring';
|
||||
message = `⚠️ SSL certificate for ${hostname} expires in ${daysRemaining} days`;
|
||||
}
|
||||
|
||||
if (level !== null && level < lastNotified) {
|
||||
// New threshold crossed — send notification
|
||||
this.notifiedThresholds.set(key, level);
|
||||
this.emit(eventType, { hostname, serviceId, daysRemaining, level });
|
||||
|
||||
if (this.ctx.notification) {
|
||||
try {
|
||||
await this.ctx.notification.send('ssl-cert-expiry', {
|
||||
text: message,
|
||||
hostname,
|
||||
serviceId,
|
||||
daysRemaining,
|
||||
level,
|
||||
validTo: certResult.validTo
|
||||
}, level <= THRESHOLDS.CRITICAL ? 'error' : 'warning');
|
||||
} catch (err) {
|
||||
this.log.error('ssl-monitor', 'Failed to send SSL notification', { error: err.message });
|
||||
}
|
||||
}
|
||||
} else if (level === null) {
|
||||
// Cert is healthy — reset notification tracking
|
||||
this.notifiedThresholds.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist cert status cache to disk.
|
||||
*/
|
||||
async _saveCache() {
|
||||
try {
|
||||
const data = {
|
||||
lastChecked: new Date().toISOString(),
|
||||
certs: {},
|
||||
hostnameToServiceId: Object.fromEntries(this.hostnameToServiceId)
|
||||
};
|
||||
for (const [hostname, cert] of this.certStatus.entries()) {
|
||||
data.certs[hostname] = cert;
|
||||
}
|
||||
await writeJsonFile(this.cacheFile, data);
|
||||
} catch (err) {
|
||||
this.log.warn('ssl-monitor', 'Failed to save SSL cache', { error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load cert status cache from disk.
|
||||
*/
|
||||
async _loadCache() {
|
||||
try {
|
||||
const data = await readJsonFile(this.cacheFile, null);
|
||||
if (data && data.certs) {
|
||||
for (const [hostname, cert] of Object.entries(data.certs)) {
|
||||
this.certStatus.set(hostname, cert);
|
||||
}
|
||||
if (data.hostnameToServiceId) {
|
||||
for (const [hostname, serviceId] of Object.entries(data.hostnameToServiceId)) {
|
||||
this.hostnameToServiceId.set(hostname, serviceId);
|
||||
}
|
||||
}
|
||||
this.log.info('ssl-monitor', 'Loaded SSL cache', {
|
||||
certCount: this.certStatus.size
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
this.log.warn('ssl-monitor', 'Failed to load SSL cache', { error: err.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = SSLMonitor;
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# DashCaddy Gitea — Off-host backup to Dropbox
|
||||
# =============================================================================
|
||||
# - Stops gitea container briefly to ensure SQLite DB consistency
|
||||
# - Syncs /var/lib/docker/volumes/gitea-data to dropbox:/Apps/dashcaddy-gitea-backups/<date>/
|
||||
# - Date-stamped snapshots (one per day), kept for 7 days locally
|
||||
# - Restarts gitea even if sync fails
|
||||
# - Logs to /var/log/gitea-backup.log
|
||||
# =============================================================================
|
||||
set -u # don't use -e: we want to always restart gitea
|
||||
|
||||
LOG=/var/log/gitea-backup.log
|
||||
DATA_SRC=/var/lib/docker/volumes/gitea-data/_data
|
||||
DEST="dropbox:/Apps/dashcaddy-gitea-backups"
|
||||
TODAY=$(date -u +%Y-%m-%d)
|
||||
BACKUP_PATH="${DEST}/${TODAY}"
|
||||
RETENTION_DAYS=7
|
||||
|
||||
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG"; }
|
||||
|
||||
log "=== Backup start ==="
|
||||
|
||||
# 0. Sanity checks
|
||||
if [ ! -d "$DATA_SRC" ]; then
|
||||
log "ERROR: data dir $DATA_SRC missing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 1. Stop gitea to flush SQLite
|
||||
log "Stopping gitea container..."
|
||||
docker stop gitea >> "$LOG" 2>&1
|
||||
STOP_RC=$?
|
||||
if [ $STOP_RC -ne 0 ]; then
|
||||
log "WARNING: docker stop returned $STOP_RC — container may not be running"
|
||||
fi
|
||||
|
||||
# 2. Sync (use copy so source files are preserved as-is, no --delete)
|
||||
log "Syncing $DATA_SRC -> $BACKUP_PATH"
|
||||
rclone copy "$DATA_SRC" "$BACKUP_PATH" \
|
||||
--transfers 4 \
|
||||
--checkers 8 \
|
||||
--retries 3 \
|
||||
--low-level-retries 10 \
|
||||
--stats 30s \
|
||||
--log-file "$LOG" \
|
||||
--log-level INFO
|
||||
SYNC_RC=$?
|
||||
|
||||
# 3. Always restart gitea
|
||||
log "Starting gitea container..."
|
||||
docker start gitea >> "$LOG" 2>&1
|
||||
START_RC=$?
|
||||
|
||||
# Wait for gitea to be ready
|
||||
for i in {1..30}; do
|
||||
if curl -sf http://localhost:3000/api/v1/version > /dev/null 2>&1; then
|
||||
log "Gitea is up after ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# 4. Cleanup old backups (older than RETENTION_DAYS)
|
||||
log "Pruning local + remote snapshots older than ${RETENTION_DAYS} days..."
|
||||
CUTOFF=$(date -u -d "${RETENTION_DAYS} days ago" +%Y-%m-%d)
|
||||
rclone lsf "$DEST/" --dirs-only 2>/dev/null | while read -r d; do
|
||||
# rclone returns names with trailing /
|
||||
name="${d%/}"
|
||||
if [[ "$name" < "$CUTOFF" ]]; then
|
||||
log " removing old: $name"
|
||||
rclone purge "${DEST}/${name}" >> "$LOG" 2>&1
|
||||
fi
|
||||
done
|
||||
|
||||
# 5. Report
|
||||
if [ $SYNC_RC -eq 0 ] && [ $START_RC -eq 0 ]; then
|
||||
log "=== Backup OK ==="
|
||||
exit 0
|
||||
else
|
||||
log "=== Backup completed with errors (sync=$SYNC_RC, start=$START_RC) ==="
|
||||
exit 1
|
||||
fi
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/bin/bash
|
||||
# Samihost fail2ban watchdog — auto-unban whitelisted IPs and keep ignoreip list in sync.
|
||||
# Deployed to /usr/local/bin/samihost-fail2ban-watchdog.sh on 194.163.161.162
|
||||
# Cron: every 30 min (0,30 * * * *)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
JAIL_LOCAL=/etc/fail2ban/jail.local
|
||||
BACKUP=/etc/fail2ban/jail.local.watchdog.bak
|
||||
EXPECTED_IGNOREIP="127.0.0.1/8 ::1 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7 fe80::/10 100.64.0.0/10 100.121.150.22 100.85.236.10 100.71.97.12 100.81.59.99 100.98.123.59 194.233.88.206 173.212.201.200 194.163.161.162"
|
||||
LOG=/var/log/samihost-fail2ban-watchdog.log
|
||||
TELEGRAM_LOG=/tmp/fail2ban-watchdog-last-action
|
||||
|
||||
ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
|
||||
log() { echo "$(ts) $*" | tee -a "$LOG"; }
|
||||
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
touch "$LOG"
|
||||
|
||||
# --- 1. Verify ignoreip line is intact and matches expected ---
|
||||
CURRENT=$(grep '^ignoreip' "$JAIL_LOCAL" | sed 's/^ignoreip[[:space:]]*=[[:space:]]*//' || true)
|
||||
EXPECTED_NORMALIZED=$(echo "$EXPECTED_IGNOREIP" | tr ' ' '\n' | sort -u | tr '\n' ' ' | sed 's/ $//')
|
||||
CURRENT_NORMALIZED=$(echo "$CURRENT" | tr ' ' '\n' | sort -u | tr '\n' ' ' | sed 's/ $//')
|
||||
|
||||
if [ "$CURRENT_NORMALIZED" != "$EXPECTED_NORMALIZED" ]; then
|
||||
log "ALERT: ignoreip line drifted. Restoring."
|
||||
cp "$JAIL_LOCAL" "$BACKUP"
|
||||
sed -i "s|^ignoreip = .*|ignoreip = $EXPECTED_IGNOREIP|" "$JAIL_LOCAL"
|
||||
fail2ban-client reload
|
||||
echo "ignoreip restored at $(ts)" > "$TELEGRAM_LOG"
|
||||
log "ignoreip restored, fail2ban reloaded"
|
||||
fi
|
||||
|
||||
# --- 2. Unban any currently-banned IPs that match our trusted set ---
|
||||
BANNED=$(fail2ban-client status sshd 2>/dev/null | awk -F: '/Banned IP list/{print $2}' | tr ' ' '\n' | grep -v '^$' || true)
|
||||
UNBANNED=0
|
||||
for ip in $BANNED; do
|
||||
# Match against any trusted network
|
||||
is_trusted=0
|
||||
for net in 127.0.0.0/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 100.64.0.0/10 ::1 fc00::/7 fe80::/10 100.121.150.22 100.85.236.10 100.71.97.12 100.81.59.99 100.98.123.59 194.233.88.206 173.212.201.200 194.163.161.162; do
|
||||
if [[ "$net" == *"/"* ]]; then
|
||||
# CIDR match (simple IPv4 only — IPv6 needs python or ipcalc, skip for now)
|
||||
base="${net%/*}"
|
||||
mask="${net#*/}"
|
||||
if [[ "$ip" == "$base"* ]] || python3 -c "import ipaddress,sys; sys.exit(0 if ipaddress.ip_address('$ip') in ipaddress.ip_network('$net', strict=False) else 1)" 2>/dev/null; then
|
||||
is_trusted=1
|
||||
break
|
||||
fi
|
||||
else
|
||||
if [ "$ip" = "$net" ]; then
|
||||
is_trusted=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if [ "$is_trusted" = "1" ]; then
|
||||
if fail2ban-client set sshd unbanip "$ip" >/dev/null 2>&1; then
|
||||
log "auto-unbanned trusted IP: $ip"
|
||||
UNBANNED=$((UNBANNED+1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
[ "$UNBANNED" -gt 0 ] && echo "auto-unbanned $UNBANNED trusted IPs at $(ts)" > "$TELEGRAM_LOG"
|
||||
|
||||
# --- 3. Cap the ban count — if more than 200 are banned, mass-unban stale ones ---
|
||||
TOTAL_BANNED=$(fail2ban-client status sshd 2>/dev/null | awk '/Currently banned/{print $NF}' || echo 0)
|
||||
if [ "$TOTAL_BANNED" -gt 200 ]; then
|
||||
log "ALERT: $TOTAL_BANNED IPs banned. Mass-unbanning all."
|
||||
for ip in $BANNED; do
|
||||
fail2ban-client set sshd unbanip "$ip" >/dev/null 2>&1 || true
|
||||
done
|
||||
echo "mass-unbanned $TOTAL_BANNED stale bans at $(ts)" > "$TELEGRAM_LOG"
|
||||
fi
|
||||
|
||||
log "watchdog run complete (unbanned=$UNBANNED, total_banned=$TOTAL_BANNED)"
|
||||
@@ -72,6 +72,7 @@ const bundles = {
|
||||
],
|
||||
'init.js': [
|
||||
JS('core', 'init.js'),
|
||||
JS('monitoring-widgets.js'),
|
||||
JS('keyboard-shortcuts.js'),
|
||||
],
|
||||
};
|
||||
|
||||
Vendored
+114
-87
File diff suppressed because one or more lines are too long
Vendored
+308
-233
File diff suppressed because one or more lines are too long
Vendored
+129
-18
File diff suppressed because one or more lines are too long
@@ -256,6 +256,9 @@
|
||||
<option value="on">🟢 Online</option>
|
||||
<option value="off">🔴 Offline</option>
|
||||
</select>
|
||||
<select id="service-filter-category" style="padding: 8px 12px; background: var(--bg); border: 1px solid var(--border); border-radius: 6px; color: var(--fg); font-size: 0.9rem;">
|
||||
<option value="all">All Categories</option>
|
||||
</select>
|
||||
<button id="batch-operations-btn" class="btn-sm" style="padding: 8px 12px;">☰ Batch Operations</button>
|
||||
<span id="service-filter-count" style="color: var(--muted); font-size: 0.85rem; white-space: nowrap;"></span>
|
||||
</div>
|
||||
|
||||
@@ -41,8 +41,11 @@
|
||||
dismissedUpdates = new Set();
|
||||
}
|
||||
|
||||
// Track global update state for cross-component access
|
||||
let knownUpdates = [];
|
||||
|
||||
// Fetch update data and show badges
|
||||
async function refreshCardUpdates() {
|
||||
async function refreshCardUpdates(notifyNew) {
|
||||
try {
|
||||
const res = await fetch('/api/v1/updates/available');
|
||||
const data = await res.json();
|
||||
@@ -51,9 +54,21 @@
|
||||
// Clear all update badges first
|
||||
document.querySelectorAll('.update-available-badge').forEach(el => el.classList.remove('visible'));
|
||||
|
||||
if (!data.updates?.length) return;
|
||||
const updates = data.updates || [];
|
||||
knownUpdates = updates; // store globally
|
||||
|
||||
for (const upd of data.updates) {
|
||||
// Notify if new updates appeared (periodic check with notification)
|
||||
if (notifyNew && updates.length > 0) {
|
||||
const prev = window._lastKnownUpdateCount || 0;
|
||||
if (prev > 0 && updates.length > prev) {
|
||||
showNotification(`${updates.length} container update(s) available — click Update Management to review.`, 'info');
|
||||
}
|
||||
window._lastKnownUpdateCount = updates.length;
|
||||
}
|
||||
|
||||
if (!updates.length) return;
|
||||
|
||||
for (const upd of updates) {
|
||||
// Try to match by container name to service id
|
||||
const apps = window.APPS || [];
|
||||
for (const app of apps) {
|
||||
@@ -61,17 +76,24 @@
|
||||
// Skip dismissed updates
|
||||
if (dismissedUpdates.has(app.id)) break;
|
||||
const badge = document.getElementById('update-badge-' + app.id);
|
||||
const updateBtn = document.getElementById('update-btn-' + app.id);
|
||||
if (badge) {
|
||||
badge.classList.add('visible');
|
||||
badge.title = `Image digest changed. Click to dismiss if already up to date.\n${upd.imageName || ''}`;
|
||||
badge.title = `Update available — click to open Update Management.`;
|
||||
badge.style.cursor = 'pointer';
|
||||
badge.onclick = (e) => {
|
||||
e.stopPropagation();
|
||||
badge.classList.remove('visible');
|
||||
dismissedUpdates.add(app.id);
|
||||
safeSessionSet('dismissed-updates', JSON.stringify([...dismissedUpdates]));
|
||||
// Open Update Management modal focused on this app
|
||||
if (window.openUpdateModal) window.openUpdateModal(app.id);
|
||||
};
|
||||
}
|
||||
// Highlight update button if update is available
|
||||
if (updateBtn) {
|
||||
updateBtn.style.background = '#f97316';
|
||||
updateBtn.style.borderColor = '#f97316';
|
||||
updateBtn.style.boxShadow = '0 0 6px #f9731688';
|
||||
updateBtn.title = `Update available — click to open Update Management.`;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -90,10 +112,10 @@
|
||||
refreshCardUpdates();
|
||||
}, 5000);
|
||||
|
||||
// Periodic refresh every 60 seconds
|
||||
// Periodic refresh every 60 seconds — notify on new updates detected
|
||||
setInterval(() => {
|
||||
refreshCardHealth();
|
||||
refreshCardUpdates();
|
||||
refreshCardUpdates(true); // true = notify if new updates found
|
||||
}, 60000);
|
||||
}
|
||||
|
||||
|
||||
@@ -95,6 +95,8 @@
|
||||
const card = el('div', 'card');
|
||||
card.setAttribute('data-app', s.id);
|
||||
card.setAttribute('data-status', 'off'); // Initial status
|
||||
if (s.containerId) card.setAttribute('data-container-id', s.containerId);
|
||||
if (s.category) card.setAttribute('data-category', s.category);
|
||||
if (s.recipeId) card.setAttribute('data-recipe-id', s.recipeId);
|
||||
|
||||
const dot = el('span', 'dot bad at-bl'); dot.id = 'dot-' + s.id + '-grid'; card.appendChild(dot);
|
||||
@@ -156,6 +158,16 @@
|
||||
nameSpan.appendChild(tsBadge);
|
||||
}
|
||||
|
||||
// Add Category badge if service has one (colored pill with icon)
|
||||
if (s.category) {
|
||||
const cats = (typeof DC !== 'undefined' && DC.CATEGORIES) || window.DC_CATEGORIES || {};
|
||||
const catInfo = cats[s.category] || {};
|
||||
const catBadge = el('span', 'cat-badge', `${catInfo.icon || ''} ${s.category}`.trim());
|
||||
catBadge.title = `Category: ${s.category}`;
|
||||
catBadge.style.cssText = `margin-left: 6px; font-size: 0.65rem; padding: 1px 6px; border-radius: 999px; background: color-mix(in srgb, ${catInfo.color || '#7f8c8d'} 25%, transparent); color: ${catInfo.color || '#7f8c8d'}; border: 1px solid color-mix(in srgb, ${catInfo.color || '#7f8c8d'} 50%, transparent); white-space: nowrap; font-weight: 500;`;
|
||||
nameSpan.appendChild(catBadge);
|
||||
}
|
||||
|
||||
row.appendChild(el('span', 'spacer'));
|
||||
|
||||
const pill = el('span', 'badge off', 'OFF'); pill.id = 'badge-' + s.id; row.appendChild(pill);
|
||||
@@ -282,6 +294,9 @@
|
||||
|
||||
// Group recipe cards visually after grid is built
|
||||
if (window.groupRecipeCards) requestAnimationFrame(() => window.groupRecipeCards());
|
||||
|
||||
// Refresh the service filter so the category dropdown reflects new services
|
||||
if (window.refreshServiceFilter) window.refreshServiceFilter();
|
||||
}
|
||||
|
||||
function setBadge(id, up, responseTime = null) {
|
||||
|
||||
@@ -59,11 +59,13 @@
|
||||
}
|
||||
_dashboardInitialized = true;
|
||||
await window.loadServices();
|
||||
await loadTemplateCategories();
|
||||
window.buildGrid();
|
||||
animateTopCards();
|
||||
window.refreshAll();
|
||||
setInterval(window.refreshAll, DC.POLL.DASHBOARD);
|
||||
if (typeof window.refreshCredsButtons === 'function') window.refreshCredsButtons();
|
||||
if (typeof window.refreshMonitoringWidgets === 'function') window.refreshMonitoringWidgets();
|
||||
// Update auth card (may have already been updated by the auto-load IIFE but ensure it's correct)
|
||||
if (typeof window._updateAuthCard === 'function') {
|
||||
try {
|
||||
@@ -200,6 +202,55 @@
|
||||
window.loadCustomServices = loadCustomServices;
|
||||
registerServiceWorker();
|
||||
|
||||
// ===== TEMPLATE CATEGORIES =====
|
||||
// Cached template categories from /api/v1/templates for use across the UI
|
||||
// (service create/edit, filter dropdown, category badges, etc.)
|
||||
async function loadTemplateCategories() {
|
||||
try {
|
||||
const r = await fetch('/api/v1/templates', { cache: 'no-store' });
|
||||
if (!r.ok) return;
|
||||
const data = await r.json();
|
||||
if (data && data.categories) {
|
||||
window.DC_CATEGORIES = data.categories;
|
||||
// Also expose via globals.js constant for convenience
|
||||
if (typeof DC !== 'undefined') DC.CATEGORIES = data.categories;
|
||||
// Populate any category <select> that's already in the DOM
|
||||
populateCategorySelects();
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[init] Failed to load template categories:', e);
|
||||
}
|
||||
}
|
||||
|
||||
function populateCategorySelects() {
|
||||
const cats = window.DC_CATEGORIES || (typeof DC !== 'undefined' && DC.CATEGORIES);
|
||||
if (!cats) return;
|
||||
document.querySelectorAll('select[data-role="service-category"]').forEach(select => {
|
||||
const current = select.dataset.current || '';
|
||||
// Clear options but keep the first (placeholder)
|
||||
const placeholder = select.querySelector('option[value=""]');
|
||||
select.innerHTML = '';
|
||||
if (placeholder) select.appendChild(placeholder);
|
||||
else {
|
||||
const ph = document.createElement('option');
|
||||
ph.value = '';
|
||||
ph.textContent = '— Select category —';
|
||||
select.appendChild(ph);
|
||||
}
|
||||
Object.entries(cats).forEach(([name, info]) => {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = name;
|
||||
opt.textContent = `${info.icon || ''} ${name}`.trim();
|
||||
if (name === current) opt.selected = true;
|
||||
select.appendChild(opt);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Allow other modules to re-run population after they (re)inject selects
|
||||
window.populateCategorySelects = populateCategorySelects;
|
||||
window.loadTemplateCategories = loadTemplateCategories;
|
||||
|
||||
// TOTP-gated initialization
|
||||
(async () => {
|
||||
try {
|
||||
|
||||
@@ -262,6 +262,7 @@
|
||||
const proxyIp = document.getElementById('external-proxy-ip').value.trim() || SITE.dnsIp || 'localhost';
|
||||
const preserveHost = document.getElementById('external-preserve-host').checked;
|
||||
const followRedirects = document.getElementById('external-follow-redirects').checked;
|
||||
const category = document.getElementById('external-service-category')?.value || '';
|
||||
|
||||
if (!name || !externalUrl) {
|
||||
showNotification('Please fill in Name and External URL', 'warning');
|
||||
@@ -341,6 +342,8 @@
|
||||
isExternal: true,
|
||||
isCustom: true
|
||||
};
|
||||
// Only attach category if user actually picked one
|
||||
if (category) newService.category = category;
|
||||
|
||||
window.APPS.push(newService);
|
||||
results.dashboard = true;
|
||||
@@ -457,6 +460,13 @@
|
||||
const healthCheck = document.getElementById('health-check-input')?.value || '';
|
||||
const timeout = document.getElementById('timeout-input')?.value || 30;
|
||||
|
||||
// Category is optional — pulled from either local or external select by the
|
||||
// openAddServiceModal reset. If user doesn't choose one, it stays undefined
|
||||
// and we don't send it (so the backend keeps the existing behavior).
|
||||
const categoryEl = document.getElementById('service-category-input')
|
||||
|| document.getElementById('external-service-category');
|
||||
const category = categoryEl?.value || '';
|
||||
|
||||
const dnsToken = window.getToken(getPrimaryDnsId(), 'admin');
|
||||
|
||||
if (!name || !port || !ip) {
|
||||
@@ -525,6 +535,8 @@
|
||||
logo: logo || `/assets/${subdomain}.png`,
|
||||
tailscaleOnly: tailscaleOnly || false
|
||||
};
|
||||
// Only include category if user actually picked one
|
||||
if (category) serviceConfig.category = category;
|
||||
|
||||
await window.addServiceToConfig(serviceConfig);
|
||||
results.dashboard = true;
|
||||
|
||||
@@ -19,6 +19,16 @@
|
||||
document.getElementById('edit-tailscale-only').checked = service.tailscaleOnly || false;
|
||||
document.getElementById('edit-logo-url').value = service.logo || '';
|
||||
|
||||
// Populate the category select for this service, then set the current value.
|
||||
// populateCategorySelects() uses data-current so we set it first, then call.
|
||||
const categorySelect = document.getElementById('edit-service-category');
|
||||
if (categorySelect) {
|
||||
categorySelect.dataset.current = service.category || '';
|
||||
if (typeof window.populateCategorySelects === 'function') {
|
||||
window.populateCategorySelects();
|
||||
}
|
||||
}
|
||||
|
||||
modal.classList.add('show');
|
||||
}
|
||||
|
||||
@@ -36,6 +46,7 @@
|
||||
const newIp = document.getElementById('edit-ip').value.trim() || 'localhost';
|
||||
const tailscaleOnly = document.getElementById('edit-tailscale-only').checked;
|
||||
const newLogo = document.getElementById('edit-logo-url').value.trim();
|
||||
const newCategory = document.getElementById('edit-service-category')?.value || '';
|
||||
|
||||
if (!newSubdomain) {
|
||||
showNotification('Subdomain is required', 'warning');
|
||||
@@ -51,6 +62,7 @@
|
||||
if (newIp !== currentEditService.ip) changes.push('ip');
|
||||
if (tailscaleOnly !== (currentEditService.tailscaleOnly || false)) changes.push('tailscale');
|
||||
if (newLogo && newLogo !== currentEditService.logo) changes.push('logo');
|
||||
if (newCategory !== (currentEditService.category || '')) changes.push('category');
|
||||
|
||||
if (changes.length === 0) {
|
||||
closeServiceEditModal();
|
||||
@@ -72,7 +84,8 @@
|
||||
port: newPort || currentEditService.port,
|
||||
ip: newIp,
|
||||
tailscaleOnly,
|
||||
logo: newLogo || undefined
|
||||
logo: newLogo || undefined,
|
||||
category: newCategory
|
||||
})
|
||||
});
|
||||
|
||||
@@ -91,7 +104,8 @@
|
||||
port: newPort || window.APPS[appIndex].port,
|
||||
ip: newIp,
|
||||
tailscaleOnly,
|
||||
logo: newLogo || window.APPS[appIndex].logo
|
||||
logo: newLogo || window.APPS[appIndex].logo,
|
||||
category: newCategory || undefined
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -187,6 +187,9 @@
|
||||
name: serviceConfig.name,
|
||||
logo: serviceConfig.logo || `/assets/${serviceConfig.subdomain}.png`
|
||||
};
|
||||
// Forward optional metadata fields if provided
|
||||
if (serviceConfig.category) newService.category = serviceConfig.category;
|
||||
if (serviceConfig.containerId) newService.containerId = serviceConfig.containerId;
|
||||
|
||||
try {
|
||||
const response = await secureFetch('/api/v1/services', {
|
||||
|
||||
@@ -82,6 +82,16 @@
|
||||
Enter a URL or upload an image file (PNG, JPG, SVG)
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Category -->
|
||||
<div>
|
||||
<label for="edit-service-category" class="form-label-accent-sm">
|
||||
Category
|
||||
</label>
|
||||
<select id="edit-service-category" data-role="service-category" class="form-input-md">
|
||||
<option value="">— No category —</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="weather-modal-buttons" style="margin-top: 24px;">
|
||||
@@ -239,6 +249,15 @@
|
||||
Reload Caddy after adding
|
||||
</label>
|
||||
|
||||
<!-- Category -->
|
||||
<div>
|
||||
<label for="service-category-input" style="font-size: 0.8rem; color: var(--muted); margin-bottom: 4px; display: block;">Category</label>
|
||||
<select id="service-category-input" data-role="service-category" style="width: 100%;">
|
||||
<option value="">— No category —</option>
|
||||
</select>
|
||||
<div style="font-size: 0.7rem; color: var(--muted); margin-top: 3px;">Group services on the dashboard by purpose (Media, Productivity, etc.)</div>
|
||||
</div>
|
||||
|
||||
<hr style="border: none; border-top: 1px solid var(--border); margin: 4px 0;" />
|
||||
|
||||
<div class="grid-2col">
|
||||
@@ -326,6 +345,14 @@
|
||||
Follow Redirects
|
||||
</label>
|
||||
|
||||
<!-- Category (external) -->
|
||||
<div>
|
||||
<label for="external-service-category" style="font-size: 0.8rem; color: var(--muted); margin-bottom: 4px; display: block;">Category</label>
|
||||
<select id="external-service-category" data-role="service-category" style="width: 100%;">
|
||||
<option value="">— No category —</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
// ========== MONITORING WIDGETS ==========
|
||||
// Embeds a compact system-resource + health summary panel directly on the
|
||||
// main dashboard. Replaces the need for a separate monitoring-dashboard.html
|
||||
// page — quick at-a-glance stats where you already are.
|
||||
(function () {
|
||||
|
||||
// ----- Style injection (scoped to .dc-monitor so it doesn't leak) -----
|
||||
const styleEl = document.createElement('style');
|
||||
styleEl.textContent = `
|
||||
.dc-monitor {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
|
||||
gap: 12px;
|
||||
margin-bottom: 16px;
|
||||
padding: 12px 16px;
|
||||
background: var(--card-base);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
.dc-monitor-card {
|
||||
padding: 10px 12px;
|
||||
background: var(--card-bg, rgba(255,255,255,0.04));
|
||||
border-radius: 8px;
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
.dc-monitor-label {
|
||||
font-size: 0.7rem;
|
||||
color: var(--muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.dc-monitor-value {
|
||||
font-size: 1.4rem;
|
||||
font-weight: 600;
|
||||
color: var(--fg);
|
||||
}
|
||||
.dc-monitor-sub {
|
||||
font-size: 0.7rem;
|
||||
color: var(--muted);
|
||||
margin-top: 4px;
|
||||
}
|
||||
.dc-monitor-bar {
|
||||
margin-top: 6px;
|
||||
width: 100%;
|
||||
height: 4px;
|
||||
background: color-mix(in srgb, var(--muted) 20%, transparent);
|
||||
border-radius: 2px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.dc-monitor-bar-fill {
|
||||
height: 100%;
|
||||
width: 0%;
|
||||
background: var(--ok-fg, #27ae60);
|
||||
transition: width 0.3s ease, background 0.3s ease;
|
||||
}
|
||||
.dc-monitor-bar-fill.warn { background: #f39c12; }
|
||||
.dc-monitor-bar-fill.bad { background: #e74c3c; }
|
||||
.dc-monitor-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.dc-monitor-title {
|
||||
font-size: 0.85rem;
|
||||
font-weight: 500;
|
||||
color: var(--muted);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
.dc-monitor-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
padding: 2px 8px;
|
||||
border-radius: 999px;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
.dc-monitor-pill.ok { background: color-mix(in srgb, #27ae60 20%, transparent); color: #27ae60; }
|
||||
.dc-monitor-pill.warn { background: color-mix(in srgb, #f39c12 20%, transparent); color: #f39c12; }
|
||||
.dc-monitor-pill.bad { background: color-mix(in srgb, #e74c3c 20%, transparent); color: #e74c3c; }
|
||||
.dc-monitor-refresh {
|
||||
font-size: 0.7rem;
|
||||
color: var(--muted);
|
||||
opacity: 0.7;
|
||||
}
|
||||
`;
|
||||
document.head.appendChild(styleEl);
|
||||
|
||||
// ----- Container element (inserted above service-filter-bar) -----
|
||||
const filterBar = document.getElementById('service-filter-bar');
|
||||
if (!filterBar) return;
|
||||
|
||||
const panel = document.createElement('div');
|
||||
panel.className = 'dc-monitor';
|
||||
panel.id = 'dc-monitor-panel';
|
||||
panel.innerHTML = `
|
||||
<div class="dc-monitor-header" style="grid-column: 1 / -1;">
|
||||
<div class="dc-monitor-title">📊 System Overview</div>
|
||||
<span class="dc-monitor-refresh" id="dc-monitor-refresh-stamp">—</span>
|
||||
</div>
|
||||
<div class="dc-monitor-card">
|
||||
<div class="dc-monitor-label">Services</div>
|
||||
<div class="dc-monitor-value" id="dc-monitor-services">—</div>
|
||||
<div class="dc-monitor-sub" id="dc-monitor-services-sub">loading…</div>
|
||||
</div>
|
||||
<div class="dc-monitor-card">
|
||||
<div class="dc-monitor-label">Containers Up</div>
|
||||
<div class="dc-monitor-value" id="dc-monitor-containers">—</div>
|
||||
<div class="dc-monitor-sub" id="dc-monitor-containers-sub">loading…</div>
|
||||
</div>
|
||||
<div class="dc-monitor-card">
|
||||
<div class="dc-monitor-label">Avg CPU</div>
|
||||
<div class="dc-monitor-value" id="dc-monitor-cpu">—</div>
|
||||
<div class="dc-monitor-bar"><div class="dc-monitor-bar-fill" id="dc-monitor-cpu-bar"></div></div>
|
||||
</div>
|
||||
<div class="dc-monitor-card">
|
||||
<div class="dc-monitor-label">Avg Memory</div>
|
||||
<div class="dc-monitor-value" id="dc-monitor-mem">—</div>
|
||||
<div class="dc-monitor-bar"><div class="dc-monitor-bar-fill" id="dc-monitor-mem-bar"></div></div>
|
||||
</div>
|
||||
<div class="dc-monitor-card">
|
||||
<div class="dc-monitor-label">Health</div>
|
||||
<div class="dc-monitor-value" id="dc-monitor-health">—</div>
|
||||
<div class="dc-monitor-sub" id="dc-monitor-health-sub">—</div>
|
||||
</div>
|
||||
`;
|
||||
// Insert ABOVE the filter bar
|
||||
filterBar.parentNode.insertBefore(panel, filterBar);
|
||||
|
||||
// ----- Helpers -----
|
||||
function setBar(id, pct) {
|
||||
const el = document.getElementById(id);
|
||||
if (!el) return;
|
||||
const p = Math.max(0, Math.min(100, Number(pct) || 0));
|
||||
el.style.width = p + '%';
|
||||
el.classList.remove('warn', 'bad');
|
||||
if (p >= 85) el.classList.add('bad');
|
||||
else if (p >= 65) el.classList.add('warn');
|
||||
}
|
||||
|
||||
function fmtPct(v) {
|
||||
if (v == null || isNaN(v)) return '—';
|
||||
return (Math.round(v * 10) / 10) + '%';
|
||||
}
|
||||
|
||||
function fmtBytes(b) {
|
||||
if (b == null || isNaN(b)) return '—';
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let i = 0;
|
||||
while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
|
||||
return b.toFixed(1) + ' ' + units[i];
|
||||
}
|
||||
|
||||
function setServicesCard() {
|
||||
const total = (window.APPS || []).length;
|
||||
let up = 0;
|
||||
document.querySelectorAll('#cards .card').forEach(c => {
|
||||
if (c.dataset.status === 'on') up++;
|
||||
});
|
||||
const el = document.getElementById('dc-monitor-services');
|
||||
const sub = document.getElementById('dc-monitor-services-sub');
|
||||
if (el) el.textContent = `${up} / ${total}`;
|
||||
if (sub) sub.textContent = total === 0 ? 'no services yet' : `${up} online · ${total - up} offline`;
|
||||
}
|
||||
|
||||
function applyHealthSummary(data) {
|
||||
const el = document.getElementById('dc-monitor-health');
|
||||
const sub = document.getElementById('dc-monitor-health-sub');
|
||||
if (!el) return;
|
||||
if (!data || data.summary == null) {
|
||||
el.textContent = '—';
|
||||
if (sub) sub.textContent = 'no data';
|
||||
return;
|
||||
}
|
||||
const s = data.summary;
|
||||
const healthy = s.healthy ?? s.up ?? 0;
|
||||
const unhealthy = s.unhealthy ?? s.down ?? 0;
|
||||
const total = s.total ?? (healthy + unhealthy);
|
||||
el.textContent = `${healthy}/${total}`;
|
||||
if (sub) {
|
||||
if (unhealthy === 0) {
|
||||
sub.innerHTML = '<span class="dc-monitor-pill ok">● all healthy</span>';
|
||||
} else if (unhealthy <= 2) {
|
||||
sub.innerHTML = `<span class="dc-monitor-pill warn">● ${unhealthy} degraded</span>`;
|
||||
} else {
|
||||
sub.innerHTML = `<span class="dc-monitor-pill bad">● ${unhealthy} down</span>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ----- Data fetches -----
|
||||
async function fetchStats() {
|
||||
try {
|
||||
const r = await fetch('/api/v1/monitoring/stats', { cache: 'no-store' });
|
||||
if (!r.ok) return null;
|
||||
const data = await r.json();
|
||||
return (data && data.stats) ? data.stats : null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchHealth() {
|
||||
try {
|
||||
const r = await fetch('/api/v1/health-checks/status', { cache: 'no-store' });
|
||||
if (!r.ok) return null;
|
||||
return await r.json();
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function applyStats(stats) {
|
||||
const containers = document.getElementById('dc-monitor-containers');
|
||||
const containersSub = document.getElementById('dc-monitor-containers-sub');
|
||||
const cpuEl = document.getElementById('dc-monitor-cpu');
|
||||
const memEl = document.getElementById('dc-monitor-mem');
|
||||
|
||||
if (!stats) {
|
||||
if (containers) containers.textContent = '—';
|
||||
if (cpuEl) cpuEl.textContent = '—';
|
||||
if (memEl) memEl.textContent = '—';
|
||||
return;
|
||||
}
|
||||
|
||||
const entries = Object.values(stats);
|
||||
if (entries.length === 0) {
|
||||
if (containers) containers.textContent = '0';
|
||||
if (containersSub) containersSub.textContent = 'no containers reporting';
|
||||
if (cpuEl) cpuEl.textContent = '0%';
|
||||
if (memEl) memEl.textContent = '0%';
|
||||
setBar('dc-monitor-cpu-bar', 0);
|
||||
setBar('dc-monitor-mem-bar', 0);
|
||||
return;
|
||||
}
|
||||
|
||||
let cpuSum = 0, memSum = 0, memBytes = 0, cpuCount = 0, memCount = 0;
|
||||
entries.forEach(s => {
|
||||
// CPU may be percentage (0-100) or fraction (0-1) — handle both
|
||||
if (s.cpu != null) {
|
||||
const cpu = Number(s.cpu);
|
||||
if (!isNaN(cpu)) {
|
||||
cpuSum += cpu > 1 ? cpu : cpu * 100;
|
||||
cpuCount++;
|
||||
}
|
||||
}
|
||||
if (s.memory != null) {
|
||||
const mem = Number(s.memory);
|
||||
if (!isNaN(mem)) {
|
||||
memSum += mem;
|
||||
memBytes += Number(s.memoryUsage || 0);
|
||||
memCount++;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const avgCpu = cpuCount ? cpuSum / cpuCount : 0;
|
||||
const avgMem = memCount ? memSum / memCount : 0;
|
||||
|
||||
if (containers) containers.textContent = String(entries.length);
|
||||
if (containersSub) {
|
||||
const memTxt = memBytes ? ` · ${fmtBytes(memBytes)} RAM` : '';
|
||||
containersSub.textContent = `running${memTxt}`;
|
||||
}
|
||||
if (cpuEl) cpuEl.textContent = fmtPct(avgCpu);
|
||||
if (memEl) memEl.textContent = fmtPct(avgMem);
|
||||
setBar('dc-monitor-cpu-bar', avgCpu);
|
||||
setBar('dc-monitor-mem-bar', avgMem);
|
||||
}
|
||||
|
||||
// ----- Public refresh function -----
|
||||
let inFlight = false;
|
||||
async function refresh() {
|
||||
if (inFlight) return;
|
||||
inFlight = true;
|
||||
try {
|
||||
setServicesCard();
|
||||
const [stats, health] = await Promise.all([fetchStats(), fetchHealth()]);
|
||||
applyStats(stats);
|
||||
applyHealthSummary(health);
|
||||
const stamp = document.getElementById('dc-monitor-refresh-stamp');
|
||||
if (stamp) {
|
||||
const now = new Date();
|
||||
stamp.textContent = `updated ${now.toLocaleTimeString()}`;
|
||||
}
|
||||
} finally {
|
||||
inFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Expose for init.js to call once and re-call after each refreshAll cycle
|
||||
window.refreshMonitoringWidgets = refresh;
|
||||
|
||||
// Auto-refresh on the STATS interval (separate from full DASHBOARD refresh)
|
||||
setInterval(refresh, (typeof DC !== 'undefined' && DC.POLL && DC.POLL.STATS) || 5000);
|
||||
|
||||
// Refresh once on first script load (init.js also calls this; double-call is harmless)
|
||||
setTimeout(refresh, 200);
|
||||
|
||||
})();
|
||||
@@ -2,11 +2,50 @@
|
||||
(function() {
|
||||
const searchInput = document.getElementById('service-filter-search');
|
||||
const statusSelect = document.getElementById('service-filter-status');
|
||||
const categorySelect = document.getElementById('service-filter-category');
|
||||
const countSpan = document.getElementById('service-filter-count');
|
||||
|
||||
// Build a single category list from both the API categories and any
|
||||
// categories present on the actual rendered cards (covers custom services
|
||||
// whose category isn't in TEMPLATE_CATEGORIES).
|
||||
function getCategoryList() {
|
||||
const seen = new Set();
|
||||
const fromCards = new Set();
|
||||
document.querySelectorAll('#cards .card[data-category]').forEach(c => {
|
||||
const cat = c.dataset.category.trim();
|
||||
if (cat) fromCards.add(cat);
|
||||
});
|
||||
const apiCats = (window.DC_CATEGORIES || (typeof DC !== 'undefined' && DC.CATEGORIES)) || {};
|
||||
const all = Object.keys(apiCats).concat([...fromCards].filter(c => !apiCats[c]));
|
||||
all.forEach(c => seen.add(c));
|
||||
return { list: [...seen], apiCats };
|
||||
}
|
||||
|
||||
function refreshCategoryDropdown() {
|
||||
if (!categorySelect) return;
|
||||
const { list, apiCats } = getCategoryList();
|
||||
const current = categorySelect.value;
|
||||
categorySelect.innerHTML = '<option value="all">All Categories</option>';
|
||||
list.sort().forEach(name => {
|
||||
const info = apiCats[name];
|
||||
const opt = document.createElement('option');
|
||||
opt.value = name;
|
||||
opt.textContent = info ? `${info.icon || ''} ${name}`.trim() : name;
|
||||
categorySelect.appendChild(opt);
|
||||
});
|
||||
// Restore selection if it still exists
|
||||
if (current && [...categorySelect.options].some(o => o.value === current)) {
|
||||
categorySelect.value = current;
|
||||
} else {
|
||||
categorySelect.value = 'all';
|
||||
}
|
||||
}
|
||||
|
||||
function updateFilter() {
|
||||
refreshCategoryDropdown();
|
||||
const query = searchInput.value.toLowerCase().trim();
|
||||
const statusFilter = statusSelect.value; // 'all', 'on', or 'off'
|
||||
const categoryFilter = categorySelect ? categorySelect.value : 'all';
|
||||
|
||||
const cards = document.querySelectorAll('#cards .card');
|
||||
let visibleCount = 0;
|
||||
@@ -15,11 +54,13 @@
|
||||
const name = card.querySelector('.name')?.textContent?.toLowerCase() || '';
|
||||
const app = card.dataset.app?.toLowerCase() || '';
|
||||
const status = card.dataset.status || 'off'; // 'on' or 'off'
|
||||
const category = card.dataset.category || '';
|
||||
|
||||
const matchesSearch = !query || name.includes(query) || app.includes(query);
|
||||
const matchesStatus = statusFilter === 'all' || status === statusFilter;
|
||||
const matchesCategory = categoryFilter === 'all' || category === categoryFilter;
|
||||
|
||||
if (matchesSearch && matchesStatus) {
|
||||
if (matchesSearch && matchesStatus && matchesCategory) {
|
||||
card.style.display = '';
|
||||
visibleCount++;
|
||||
} else {
|
||||
@@ -44,6 +85,7 @@
|
||||
|
||||
searchInput?.addEventListener('input', debounce(updateFilter, 200));
|
||||
statusSelect?.addEventListener('change', updateFilter);
|
||||
categorySelect?.addEventListener('change', updateFilter);
|
||||
|
||||
// Initial count on page load
|
||||
if (document.readyState === 'loading') {
|
||||
@@ -52,6 +94,7 @@
|
||||
setTimeout(updateFilter, 500);
|
||||
}
|
||||
|
||||
// Expose for external triggers
|
||||
// Expose for external triggers (called after buildGrid to repopulate categories)
|
||||
window.refreshServiceFilter = updateFilter;
|
||||
window.refreshCategoryDropdown = refreshCategoryDropdown;
|
||||
})();
|
||||
|
||||
@@ -17,8 +17,10 @@
|
||||
|
||||
<!-- Tab: Available Updates -->
|
||||
<div id="updates-available" class="panel-section active">
|
||||
<div style="margin-bottom: 12px;">
|
||||
<div style="margin-bottom: 12px; display: flex; gap: 8px; align-items: center;">
|
||||
<button id="updates-check-btn" class="btn-accent-solid">🔍 Check for Updates</button>
|
||||
<button id="updates-update-all-btn" style="display: none; padding: 6px 14px; font-size: 0.82rem; background: #f97316; color: #fff; border: 1px solid #f97316; border-radius: 6px; cursor: pointer;">⬆️ Update All</button>
|
||||
<span id="updates-count-badge" style="display: none; padding: 4px 10px; border-radius: 12px; font-size: 0.78rem; font-weight: 600; background: var(--accent); color: var(--bg);"></span>
|
||||
</div>
|
||||
<div id="updates-available-container" style="max-height: 450px; overflow-y: auto;">
|
||||
<div class="panel-empty"><span class="empty-icon">📦</span> Click "Check for Updates" to scan containers.</div>
|
||||
@@ -94,13 +96,24 @@
|
||||
if (updates.length === 0) {
|
||||
availableContainer.innerHTML = '<div class="panel-empty"><span class="empty-icon">✅</span>All containers are up to date.</div>';
|
||||
lastCheckSpan.textContent = '';
|
||||
document.getElementById('updates-update-all-btn').style.display = 'none';
|
||||
document.getElementById('updates-count-badge').style.display = 'none';
|
||||
window._pendingUpdates = [];
|
||||
return;
|
||||
}
|
||||
|
||||
let html = '<table style="width: 100%; border-collapse: collapse; font-size: 0.85rem;">';
|
||||
html += '<tr style="border-bottom: 1px solid var(--border); color: var(--muted);"><th style="padding: 8px; text-align: left;">Container</th><th style="padding: 8px; text-align: left;">Image</th><th style="padding: 8px; text-align: left;">Current</th><th style="padding: 8px; text-align: left;">Latest</th><th style="padding: 8px; text-align: right;">Actions</th></tr>';
|
||||
for (const u of updates) {
|
||||
html += `<tr style="border-bottom: 1px solid var(--border);">`;
|
||||
// Match app by containerId first, then name
|
||||
const appId = (() => {
|
||||
const apps = window.APPS || [];
|
||||
for (const a of apps) {
|
||||
if (a.containerId === u.containerId || a.name === u.containerName || a.id === u.containerName) return a.id;
|
||||
}
|
||||
return u.containerName;
|
||||
})();
|
||||
html += `<tr data-app-id="${escapeHtml(appId)}" style="border-bottom: 1px solid var(--border);">`;
|
||||
html += `<td style="padding: 8px; font-weight: 500;">${escapeHtml(u.containerName)}</td>`;
|
||||
html += `<td style="padding: 8px; color: var(--muted);">${escapeHtml(u.imageName)}</td>`;
|
||||
html += `<td style="padding: 8px;"><code style="font-size: 0.78rem; background: var(--bg); padding: 2px 6px; border-radius: 4px;">${escapeHtml(u.currentDigest)}</code></td>`;
|
||||
@@ -114,6 +127,20 @@
|
||||
availableContainer.innerHTML = html;
|
||||
lastCheckSpan.textContent = updates.length + ' update(s) available';
|
||||
|
||||
// Show count badge and Update All button
|
||||
const countBadge = document.getElementById('updates-count-badge');
|
||||
const updateAllBtn = document.getElementById('updates-update-all-btn');
|
||||
if (countBadge) {
|
||||
countBadge.textContent = updates.length + ' pending';
|
||||
countBadge.style.display = '';
|
||||
}
|
||||
if (updateAllBtn && updates.length > 0) {
|
||||
updateAllBtn.style.display = '';
|
||||
}
|
||||
|
||||
// Store updates for Update All button
|
||||
window._pendingUpdates = updates;
|
||||
|
||||
// Wire update buttons
|
||||
availableContainer.querySelectorAll('.update-now-btn').forEach(btn => {
|
||||
btn.addEventListener('click', async () => {
|
||||
@@ -174,6 +201,38 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Update All — sequentially, skip failures
|
||||
async function updateAllContainers() {
|
||||
const updates = window._pendingUpdates || [];
|
||||
if (!updates.length) return;
|
||||
const btn = document.getElementById('updates-update-all-btn');
|
||||
if (!confirm(`Update all ${updates.length} containers? Each will restart.`)) return;
|
||||
btn.textContent = '⏳ Updating...';
|
||||
btn.disabled = true;
|
||||
let success = 0, failed = 0;
|
||||
for (const u of updates) {
|
||||
try {
|
||||
const r = await secureFetch(`/api/v1/updates/update/${encodeURIComponent(u.containerId)}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ autoRollback: true })
|
||||
});
|
||||
const d = await r.json();
|
||||
if (d.success) success++;
|
||||
else failed++;
|
||||
} catch (_) { failed++; }
|
||||
}
|
||||
btn.textContent = `✅ Done`;
|
||||
showNotification(`Update all: ${success} succeeded, ${failed} failed.`, success > 0 && failed === 0 ? 'success' : 'error');
|
||||
setTimeout(() => {
|
||||
btn.textContent = '⬆️ Update All';
|
||||
btn.disabled = false;
|
||||
loadAvailable();
|
||||
}, 3000);
|
||||
}
|
||||
|
||||
document.getElementById('updates-update-all-btn')?.addEventListener('click', updateAllContainers);
|
||||
|
||||
async function checkForUpdates() {
|
||||
checkBtn.textContent = '🔍 Checking...';
|
||||
checkBtn.disabled = true;
|
||||
@@ -499,6 +558,21 @@
|
||||
});
|
||||
wireModal(modal, cancelBtn);
|
||||
|
||||
// Open Update Management modal, optionally scrolled to a specific app
|
||||
window.openUpdateModal = function(appId) {
|
||||
modal?.classList.add('show');
|
||||
loadAvailable().then(() => {
|
||||
if (!appId) return;
|
||||
// Scroll to and highlight the matching row
|
||||
const row = availableContainer.querySelector(`[data-app-id="${appId}"]`);
|
||||
if (row) {
|
||||
row.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
row.style.background = 'rgba(249,115,22,0.15)';
|
||||
setTimeout(() => { row.style.background = ''; }, 3000);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Lazy-load tabs
|
||||
document.querySelector('[data-panel="updates-history"]')?.addEventListener('click', loadHistory);
|
||||
document.querySelector('[data-panel="updates-auto"]')?.addEventListener('click', loadAutoConfig);
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
const CACHE = 'dashcaddy-shell-8ef9c82616';
|
||||
const CACHE = 'dashcaddy-shell-43a872cc40';
|
||||
const PRECACHE = [
|
||||
'/',
|
||||
'/index.html',
|
||||
|
||||
Reference in New Issue
Block a user