Three coordinated fixes for the System Overview widget:
1. routes/monitoring.js — flatten getAllStats() shape from
{current:{cpu:{percent},memory:{percent}}} to {cpu,memory,memoryUsage}
so the widget's Number() coercion actually produces numbers, not NaN.
Skill reference: references/totp-and-system-overview-pitfalls.md §3.
2. routes/health.js — add summary block to /health-checks/status response.
Widget looks for {healthy, unhealthy, total} but only per-service objects
existed. Permissive on healthy side (up|healthy|online), strict on
unhealthy (down|unhealthy|offline|error); anything else counted as
unknown. Same skill §3 reference.
3. middleware.js — add /api/v1/monitoring/stats to PUBLIC_ROUTES and the
rate-limit skip list. The widget polls it every 5s from the dashboard;
cookie-auth works but listing it explicitly makes it future-proof
against auth-cookie expiry and prevents per-second 429s.
End-to-end test (unauthenticated):
GET /api/v1/monitoring/stats -> {cpu: 8.71, memory: 0.37, ...}
GET /api/v1/health-checks/status -> {summary: {healthy:11, unhealthy:4, total:15}}
286 lines
11 KiB
JavaScript
286 lines
11 KiB
JavaScript
const express = require('express');
|
|
const { success } = require('../response-helpers');
|
|
|
|
/**
|
|
* Monitoring routes factory
|
|
* @param {Object} deps - Explicit dependencies
|
|
* @param {Object} deps.resourceMonitor - Resource monitoring manager
|
|
* @param {Object} deps.docker - Docker client wrapper
|
|
* @param {Function} deps.asyncHandler - Async route handler wrapper
|
|
* @param {Object} deps.log - Logger instance
|
|
* @returns {express.Router}
|
|
*/
|
|
module.exports = function({ resourceMonitor, docker, asyncHandler, log, notificationManager }) {
|
|
const router = express.Router();
|
|
|
|
// ===== RESOURCE MONITORING ENDPOINTS =====
|
|
|
|
// Get all container stats (from resource monitor module)
|
|
// Flattened for the System Overview widget — see skill references/totp-and-system-overview-pitfalls.md §3
|
|
router.get('/monitoring/stats', asyncHandler(async (req, res) => {
|
|
const raw = resourceMonitor.getAllStats();
|
|
const stats = {};
|
|
for (const [id, data] of Object.entries(raw || {})) {
|
|
const cur = data.current || {};
|
|
const cpuObj = (cur.cpu && typeof cur.cpu === 'object') ? cur.cpu : null;
|
|
const memObj = (cur.memory && typeof cur.memory === 'object') ? cur.memory : null;
|
|
stats[id] = {
|
|
name: data.name,
|
|
cpu: cpuObj ? (cpuObj.percent ?? 0) : (Number(cur.cpu) || 0),
|
|
memory: memObj ? (memObj.percent ?? 0) : (Number(cur.memory) || 0),
|
|
memoryUsage: memObj ? (memObj.usage ?? 0) : 0,
|
|
};
|
|
}
|
|
success(res, { stats });
|
|
}, 'monitoring-stats'));
|
|
|
|
// Get stats for specific container
|
|
router.get('/monitoring/stats/:containerId', asyncHandler(async (req, res) => {
|
|
const stats = resourceMonitor.getCurrentStats(req.params.containerId);
|
|
if (!stats) {
|
|
const { NotFoundError } = require('../errors');
|
|
throw new NotFoundError('Container');
|
|
}
|
|
success(res, { stats });
|
|
}, 'monitoring-stats-container'));
|
|
|
|
// Get historical stats — supports either ?hours=24 (legacy raw) OR ?startTime=...&endTime=...
|
|
// (range mode auto-selects raw / hourly / daily tier)
|
|
router.get('/monitoring/history/:containerId', asyncHandler(async (req, res) => {
|
|
const containerId = req.params.containerId;
|
|
|
|
// Range mode (preferred)
|
|
if (req.query.startTime && req.query.endTime) {
|
|
const startTime = parseInt(req.query.startTime, 10);
|
|
const endTime = parseInt(req.query.endTime, 10);
|
|
if (Number.isNaN(startTime) || Number.isNaN(endTime) || startTime >= endTime) {
|
|
const { ValidationError } = require('../errors');
|
|
throw new ValidationError('Invalid startTime/endTime');
|
|
}
|
|
const result = resourceMonitor.getHistoryByRange(containerId, startTime, endTime);
|
|
success(res, { ...result, startTime, endTime });
|
|
return;
|
|
}
|
|
|
|
// Legacy hours-based mode (raw samples only)
|
|
const hours = parseInt(req.query.hours) || 24;
|
|
const history = resourceMonitor.getHistoricalStats(containerId, hours);
|
|
success(res, { history, hours, tier: 'raw', samples: history, unit: '10s' });
|
|
}, 'monitoring-history'));
|
|
|
|
// Get aggregated stats
|
|
router.get('/monitoring/aggregated/:containerId', asyncHandler(async (req, res) => {
|
|
const hours = parseInt(req.query.hours) || 24;
|
|
const aggregated = resourceMonitor.getAggregatedStats(req.params.containerId, hours);
|
|
if (!aggregated) {
|
|
const { NotFoundError } = require('../errors');
|
|
throw new NotFoundError('Monitoring data');
|
|
}
|
|
success(res, { aggregated, hours });
|
|
}, 'monitoring-aggregated'));
|
|
|
|
// ===== ALERT CONFIGURATION (bulk) =====
|
|
|
|
// Get all alert configs
|
|
router.get('/monitoring/alerts/config', asyncHandler(async (req, res) => {
|
|
const configs = resourceMonitor.getAllAlertConfigs();
|
|
success(res, { configs });
|
|
}, 'monitoring-alerts-config-get'));
|
|
|
|
// Set all alert configs (bulk update)
|
|
router.post('/monitoring/alerts/config', asyncHandler(async (req, res) => {
|
|
const { configs } = req.body;
|
|
if (!configs || typeof configs !== 'object') {
|
|
const { ValidationError } = require('../errors');
|
|
throw new ValidationError('configs object required');
|
|
}
|
|
for (const [containerId, config] of Object.entries(configs)) {
|
|
resourceMonitor.setAlertConfig(containerId, config);
|
|
}
|
|
success(res, { message: 'Alert configurations saved' });
|
|
}, 'monitoring-alerts-config-set'));
|
|
|
|
// Get alert history
|
|
router.get('/monitoring/alerts', asyncHandler(async (req, res) => {
|
|
const limit = parseInt(req.query.limit) || 50;
|
|
const history = resourceMonitor.getAlertHistory(limit);
|
|
success(res, { history });
|
|
}, 'monitoring-alerts-history'));
|
|
|
|
// Send test alert notification for a container
|
|
router.post('/monitoring/alerts/:containerId/test', asyncHandler(async (req, res) => {
|
|
const { containerId } = req.params;
|
|
|
|
// Get container name from docker
|
|
let containerName = containerId;
|
|
try {
|
|
const containers = await docker.client.listContainers({ all: false });
|
|
const containerInfo = containers.find(c => c.Id === containerId || c.Id.startsWith(containerId));
|
|
if (containerInfo) {
|
|
containerName = containerInfo.Names[0]?.replace(/^\//, '') || containerId;
|
|
}
|
|
} catch (_) {}
|
|
|
|
const testAlert = {
|
|
containerId,
|
|
containerName,
|
|
timestamp: new Date().toISOString(),
|
|
alerts: [{
|
|
type: 'test',
|
|
severity: 'info',
|
|
message: 'This is a test alert notification',
|
|
value: 0,
|
|
threshold: 0
|
|
}],
|
|
stats: null,
|
|
config: resourceMonitor.getAlertConfig(containerId) || {}
|
|
};
|
|
|
|
if (notificationManager) {
|
|
await notificationManager.sendAlert(testAlert);
|
|
}
|
|
|
|
// Also log to alert history
|
|
resourceMonitor.addAlertHistoryEntry({
|
|
id: `test-${Date.now()}`,
|
|
timestamp: new Date().toISOString(),
|
|
containerId,
|
|
containerName,
|
|
type: 'test',
|
|
metric: 'test',
|
|
value: 0,
|
|
threshold: 0,
|
|
severity: 'info',
|
|
notified: true,
|
|
autoRestartTriggered: false
|
|
});
|
|
|
|
success(res, { message: 'Test alert sent', alert: testAlert });
|
|
}, 'monitoring-alerts-test'));
|
|
|
|
// Configure alerts for a container
|
|
router.post('/monitoring/alerts/:containerId', asyncHandler(async (req, res) => {
|
|
resourceMonitor.setAlertConfig(req.params.containerId, req.body);
|
|
success(res, { message: 'Alert configuration saved' });
|
|
}, 'monitoring-alerts-set'));
|
|
|
|
// Get alert configuration
|
|
router.get('/monitoring/alerts/:containerId', asyncHandler(async (req, res) => {
|
|
const config = resourceMonitor.getAlertConfig(req.params.containerId);
|
|
success(res, { config: config || {} });
|
|
}, 'monitoring-alerts-get'));
|
|
|
|
// Delete alert configuration
|
|
router.delete('/monitoring/alerts/:containerId', asyncHandler(async (req, res) => {
|
|
resourceMonitor.removeAlertConfig(req.params.containerId);
|
|
success(res, { message: 'Alert configuration removed' });
|
|
}, 'monitoring-alerts-delete'));
|
|
|
|
// ===== CONTAINER STATS ENDPOINTS (legacy /stats/) =====
|
|
|
|
// Get all container stats (live Docker stats)
|
|
router.get('/stats/containers', asyncHandler(async (req, res) => {
|
|
const containers = await docker.client.listContainers({ all: false });
|
|
const stats = [];
|
|
|
|
for (const containerInfo of containers) {
|
|
try {
|
|
const container = docker.client.getContainer(containerInfo.Id);
|
|
const containerStats = await container.stats({ stream: false });
|
|
|
|
// Calculate CPU percentage
|
|
const cpuDelta = containerStats.cpu_stats.cpu_usage.total_usage -
|
|
(containerStats.precpu_stats.cpu_usage?.total_usage || 0);
|
|
const systemDelta = containerStats.cpu_stats.system_cpu_usage -
|
|
(containerStats.precpu_stats.system_cpu_usage || 0);
|
|
const cpuPercent = systemDelta > 0 ? (cpuDelta / systemDelta) * 100 * (containerStats.cpu_stats.online_cpus || 1) : 0;
|
|
|
|
// Calculate memory usage
|
|
const memUsage = containerStats.memory_stats.usage || 0;
|
|
const memLimit = containerStats.memory_stats.limit || 1;
|
|
const memPercent = (memUsage / memLimit) * 100;
|
|
|
|
// Network stats
|
|
let netRx = 0, netTx = 0;
|
|
if (containerStats.networks) {
|
|
for (const net of Object.values(containerStats.networks)) {
|
|
netRx += net.rx_bytes || 0;
|
|
netTx += net.tx_bytes || 0;
|
|
}
|
|
}
|
|
|
|
stats.push({
|
|
id: containerInfo.Id.slice(0, 12),
|
|
name: containerInfo.Names[0]?.replace(/^\//, '') || 'unknown',
|
|
image: containerInfo.Image,
|
|
status: containerInfo.State,
|
|
cpu: {
|
|
percent: Math.round(cpuPercent * 100) / 100
|
|
},
|
|
memory: {
|
|
used: memUsage,
|
|
limit: memLimit,
|
|
percent: Math.round(memPercent * 100) / 100
|
|
},
|
|
network: {
|
|
rx: netRx,
|
|
tx: netTx
|
|
}
|
|
});
|
|
} catch (e) {
|
|
// Skip containers we can't get stats for
|
|
log.warn('monitoring', `Could not get stats for ${containerInfo.Names[0]}`, { error: e.message });
|
|
}
|
|
}
|
|
|
|
success(res, { stats, timestamp: new Date().toISOString() });
|
|
}, 'stats-containers'));
|
|
|
|
// Get single container stats
|
|
router.get('/stats/container/:id', asyncHandler(async (req, res) => {
|
|
const container = docker.client.getContainer(req.params.id);
|
|
const containerStats = await container.stats({ stream: false });
|
|
const info = await container.inspect();
|
|
|
|
// Calculate CPU percentage
|
|
const cpuDelta = containerStats.cpu_stats.cpu_usage.total_usage -
|
|
(containerStats.precpu_stats.cpu_usage?.total_usage || 0);
|
|
const systemDelta = containerStats.cpu_stats.system_cpu_usage -
|
|
(containerStats.precpu_stats.system_cpu_usage || 0);
|
|
const cpuPercent = systemDelta > 0 ? (cpuDelta / systemDelta) * 100 * (containerStats.cpu_stats.online_cpus || 1) : 0;
|
|
|
|
// Memory
|
|
const memUsage = containerStats.memory_stats.usage || 0;
|
|
const memLimit = containerStats.memory_stats.limit || 1;
|
|
|
|
// Network
|
|
let netRx = 0, netTx = 0;
|
|
if (containerStats.networks) {
|
|
for (const net of Object.values(containerStats.networks)) {
|
|
netRx += net.rx_bytes || 0;
|
|
netTx += net.tx_bytes || 0;
|
|
}
|
|
}
|
|
|
|
success(res, {
|
|
stats: {
|
|
name: info.Name.replace(/^\//, ''),
|
|
image: info.Config.Image,
|
|
status: info.State.Status,
|
|
started: info.State.StartedAt,
|
|
cpu: {
|
|
percent: Math.round(cpuPercent * 100) / 100
|
|
},
|
|
memory: {
|
|
used: memUsage,
|
|
limit: memLimit,
|
|
percent: Math.round((memUsage / memLimit) * 100 * 100) / 100
|
|
},
|
|
network: { rx: netRx, tx: netTx }
|
|
}
|
|
});
|
|
}, 'stats-container'));
|
|
|
|
return router;
|
|
};
|