DC-101: Disk Space Monitor + Product Vision
Backend: - src/monitoring/disk-space-monitor.js: monitors Docker disk usage against user-configured budget, auto-cleans at thresholds, breaks down by category - routes/disk-space.js: GET /disk, GET /disk/breakdown, POST /disk/config, POST /disk/cleanup endpoints - src/app.js: wire DiskSpaceMonitor into startup, 10-min check interval - All 1539 tests pass Product Vision (PRODUCT-VISION.md): - DashCaddy is a self-hosting platform, not just a dashboard - Core value: 'Self-host anything in 30 seconds' - Three pillars: One-click deploy, zero-config networking, self-healing infra - vs Portainer/CasaOS/Yunohost positioning New backlog tasks (P5 tier, DC-101–108): - Disk budget, one-click deploy with auto Caddyfile+DNS, container auto-discovery, app catalog, smart wizard, visual Caddy builder, disaster recovery, multi-host fleet management 47 total backlog tasks, ~110 hr of work, cron running every 2h.
This commit is contained in:
Hermes
@@ -0,0 +1,392 @@
|
||||
/**
|
||||
* Disk Space Monitor
|
||||
*
|
||||
* Tracks Docker + system disk usage against a user-configured budget.
|
||||
* When usage exceeds thresholds, triggers automatic cleanup and notifications.
|
||||
*
|
||||
* Key concepts:
|
||||
* - diskBudgetGB: How much disk the user is willing to give DashCaddy (default 10)
|
||||
* - The monitor calculates Docker's footprint (images, volumes, containers, build cache)
|
||||
* - Breakdown shows where space goes so users can make informed decisions
|
||||
* - Auto-cleanup triggers at 80% (warning), 90% (aggressive), 95% (critical)
|
||||
*/
|
||||
|
||||
const EventEmitter = require('events');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { execFile } = require('child_process');
|
||||
const { promisify } = require('util');
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const DEFAULT_BUDGET_GB = 10;
|
||||
const DEFAULT_CONFIG = {
|
||||
enabled: true,
|
||||
diskBudgetGB: DEFAULT_BUDGET_GB,
|
||||
warningThresholdPct: 80,
|
||||
criticalThresholdPct: 90,
|
||||
autoCleanup: true,
|
||||
cleanupAggressivePct: 95,
|
||||
};
|
||||
|
||||
class DiskSpaceMonitor extends EventEmitter {
|
||||
constructor({ log, config }) {
|
||||
super();
|
||||
this.log = log;
|
||||
this.config = config;
|
||||
this.lastSnapshot = null;
|
||||
this.lastCleanup = null;
|
||||
this.intervalHandle = null;
|
||||
this.diskConfig = { ...DEFAULT_CONFIG };
|
||||
this._loadConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* Load disk budget config from the site config file
|
||||
* Stored under `diskSpace` key in config.json
|
||||
*/
|
||||
_loadConfig() {
|
||||
try {
|
||||
const raw = this.config?.diskSpace;
|
||||
if (raw) {
|
||||
this.diskConfig = {
|
||||
...DEFAULT_CONFIG,
|
||||
...raw,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Use defaults
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update disk space settings
|
||||
*/
|
||||
configure(updates) {
|
||||
const prev = { ...this.diskConfig };
|
||||
this.diskConfig = { ...this.diskConfig, ...updates };
|
||||
this._persistConfig();
|
||||
this.emit('config-changed', { prev, current: this.diskConfig });
|
||||
return this.diskConfig;
|
||||
}
|
||||
|
||||
_persistConfig() {
|
||||
// The config is persisted by the caller (settings route) which merges
|
||||
// into config.json. We just expose the current state.
|
||||
if (this.config) {
|
||||
this.config.diskSpace = this.diskConfig;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a disk usage snapshot using `df` and `docker system df -v`
|
||||
*/
|
||||
async getSnapshot() {
|
||||
const [diskInfo, dockerInfo] = await Promise.all([
|
||||
this._getDiskInfo(),
|
||||
this._getDockerInfo(),
|
||||
]);
|
||||
|
||||
const snapshot = {
|
||||
timestamp: new Date().toISOString(),
|
||||
system: diskInfo,
|
||||
docker: dockerInfo,
|
||||
budget: {
|
||||
configuredGB: this.diskConfig.diskBudgetGB,
|
||||
dockerUsageGB: dockerInfo.totalGB,
|
||||
remainingBudgetGB: Math.max(0, this.diskConfig.diskBudgetGB - dockerInfo.totalGB),
|
||||
budgetUsedPct: Math.min(100, Math.round((dockerInfo.totalGB / this.diskConfig.diskBudgetGB) * 100)),
|
||||
status: this._getBudgetStatus(dockerInfo.totalGB),
|
||||
},
|
||||
config: { ...this.diskConfig },
|
||||
lastCleanup: this.lastCleanup,
|
||||
};
|
||||
|
||||
this.lastSnapshot = snapshot;
|
||||
|
||||
// Check thresholds and emit events
|
||||
this._checkThresholds(snapshot);
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
_getBudgetStatus(dockerUsageGB) {
|
||||
const pct = (dockerUsageGB / this.diskConfig.diskBudgetGB) * 100;
|
||||
if (pct >= this.diskConfig.cleanupAggressivePct) return 'critical';
|
||||
if (pct >= this.diskConfig.criticalThresholdPct) return 'aggressive';
|
||||
if (pct >= this.diskConfig.warningThresholdPct) return 'warning';
|
||||
return 'healthy';
|
||||
}
|
||||
|
||||
_checkThresholds(snapshot) {
|
||||
const { status, budgetUsedPct } = snapshot.budget;
|
||||
if (status === 'critical' || status === 'aggressive') {
|
||||
this.emit('budget-exceeded', snapshot);
|
||||
if (this.diskConfig.autoCleanup) {
|
||||
this.performCleanup(status === 'critical' ? 'aggressive' : 'standard').catch(() => {});
|
||||
}
|
||||
} else if (status === 'warning') {
|
||||
this.emit('budget-warning', snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
async _getDiskInfo() {
|
||||
try {
|
||||
const { stdout } = await execFileAsync('df', ['-B1', '/']);
|
||||
const lines = stdout.trim().split('\n');
|
||||
const parts = lines[1].split(/\s+/);
|
||||
return {
|
||||
totalBytes: parseInt(parts[1], 10),
|
||||
usedBytes: parseInt(parts[2], 10),
|
||||
availableBytes: parseInt(parts[3], 10),
|
||||
usedPct: parseInt(parts[4], 10),
|
||||
mount: parts[5],
|
||||
totalGB: Math.round(parseInt(parts[1], 10) / 1073741824 * 10) / 10,
|
||||
usedGB: Math.round(parseInt(parts[2], 10) / 1073741824 * 10) / 10,
|
||||
availableGB: Math.round(parseInt(parts[3], 10) / 1073741824 * 10) / 10,
|
||||
};
|
||||
} catch {
|
||||
return { totalBytes: 0, usedBytes: 0, availableBytes: 0, usedPct: 0, totalGB: 0, usedGB: 0, availableGB: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async _getDockerInfo() {
|
||||
try {
|
||||
const { stdout } = await execFileAsync('docker', ['system', 'df', '--format', '{{json .}}']);
|
||||
const lines = stdout.trim().split('\n').filter(Boolean);
|
||||
|
||||
let images = { count: 0, totalGB: 0, reclaimableGB: 0 };
|
||||
let containers = { count: 0, totalGB: 0, reclaimableGB: 0 };
|
||||
let volumes = { count: 0, totalGB: 0, reclaimableGB: 0 };
|
||||
let buildCache = { count: 0, totalGB: 0, reclaimableGB: 0 };
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const d = JSON.parse(line);
|
||||
const type = d.Type?.toLowerCase() || '';
|
||||
const sizeGB = this._parseSizeToGB(d.Size);
|
||||
const reclaimGB = this._parseSizeToGB(d.Reclaimable);
|
||||
|
||||
if (type === 'images') images = { count: parseInt(d.TotalCount, 10) || 0, totalGB: sizeGB, reclaimableGB: reclaimGB };
|
||||
else if (type === 'containers') containers = { count: parseInt(d.TotalCount, 10) || 0, totalGB: sizeGB, reclaimableGB: reclaimGB };
|
||||
else if (type === 'local volumes') volumes = { count: parseInt(d.TotalCount, 10) || 0, totalGB: sizeGB, reclaimableGB: reclaimGB };
|
||||
else if (type === 'build cache') buildCache = { count: parseInt(d.TotalCount, 10) || 0, totalGB: sizeGB, reclaimableGB: reclaimGB };
|
||||
} catch { /* skip unparseable lines */ }
|
||||
}
|
||||
|
||||
const totalGB = Math.round((images.totalGB + containers.totalGB + volumes.totalGB + buildCache.totalGB) * 100) / 100;
|
||||
const reclaimableGB = Math.round((images.reclaimableGB + containers.reclaimableGB + volumes.reclaimableGB + buildCache.reclaimableGB) * 100) / 100;
|
||||
|
||||
return {
|
||||
images,
|
||||
containers,
|
||||
volumes,
|
||||
buildCache,
|
||||
totalGB,
|
||||
reclaimableGB,
|
||||
};
|
||||
} catch {
|
||||
return { images: {}, containers: {}, volumes: {}, buildCache: {}, totalGB: 0, reclaimableGB: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Docker's human-readable size strings (e.g., "2.519GB", "8.108MB", "0B")
|
||||
*/
|
||||
_parseSizeToGB(str) {
|
||||
if (!str || str === '0B') return 0;
|
||||
const match = str.match(/^([\d.]+)(B|KB|MB|GB|TB)$/i);
|
||||
if (!match) return 0;
|
||||
const value = parseFloat(match[1]);
|
||||
const unit = match[2].toUpperCase();
|
||||
const multipliers = { B: 1e-9, KB: 1e-6, MB: 1e-3, GB: 1, TB: 1e3 };
|
||||
return Math.round(value * (multipliers[unit] || 0) * 1000) / 1000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get per-container log file sizes (the hidden disk hog)
|
||||
*/
|
||||
async _getContainerLogs() {
|
||||
try {
|
||||
const { stdout } = await execFileAsync('sh', ['-c', 'for f in /var/lib/docker/containers/*/*-json.log; do [ -f "$f" ] && stat -c "%s %n" "$f"; done 2>/dev/null | sort -rn | head -10']);
|
||||
const entries = [];
|
||||
for (const line of stdout.trim().split('\n').filter(Boolean)) {
|
||||
const [sizeStr, ...fileParts] = line.split(' ');
|
||||
const sizeBytes = parseInt(sizeStr, 10);
|
||||
entries.push({
|
||||
sizeBytes,
|
||||
sizeMB: Math.round(sizeBytes / 1048576 * 10) / 10,
|
||||
file: fileParts.join(' '),
|
||||
});
|
||||
}
|
||||
return entries;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform cleanup
|
||||
* @param {string} level - 'standard' | 'aggressive' | 'logs-only'
|
||||
* @returns {Object} cleanup result with bytes reclaimed
|
||||
*/
|
||||
async performCleanup(level = 'standard') {
|
||||
const startTime = Date.now();
|
||||
const result = {
|
||||
level,
|
||||
startedAt: new Date(startTime).toISOString(),
|
||||
actions: [],
|
||||
bytesReclaimed: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
// Always: truncate oversized container logs
|
||||
const logsBefore = await this._getContainerLogs();
|
||||
let logBytesFreed = 0;
|
||||
for (const log of logsBefore) {
|
||||
if (log.sizeBytes > 100 * 1048576) { // > 100MB
|
||||
try {
|
||||
await execFileAsync('truncate', ['-s', '0', log.file]);
|
||||
logBytesFreed += log.sizeBytes;
|
||||
result.actions.push({ action: 'truncate-log', file: log.file, freedBytes: log.sizeBytes });
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
}
|
||||
result.bytesReclaimed += logBytesFreed;
|
||||
|
||||
// Always: vacuum journald to 200MB
|
||||
try {
|
||||
const { stdout } = await execFileAsync('journalctl', ['--vacuum-size=200M']);
|
||||
const freedMatch = stdout.match(/freed ([\d.]+[KMGT]?B)/i);
|
||||
if (freedMatch) {
|
||||
const freedBytes = this._humanToBytes(freedMatch[1]);
|
||||
result.bytesReclaimed += freedBytes;
|
||||
result.actions.push({ action: 'vacuum-journal', freedBytes, freedHuman: freedMatch[1] });
|
||||
}
|
||||
} catch { /* skip */ }
|
||||
|
||||
if (level === 'standard' || level === 'aggressive') {
|
||||
// Prune dangling images
|
||||
try {
|
||||
const { stdout } = await execFileAsync('docker', ['image', 'prune', '-f', '--filter', 'dangling=true']);
|
||||
const reclaimed = this._extractDockerReclaimed(stdout);
|
||||
result.bytesReclaimed += reclaimed;
|
||||
result.actions.push({ action: 'prune-dangling-images', freedBytes: reclaimed });
|
||||
} catch { /* skip */ }
|
||||
|
||||
// Prune unused volumes
|
||||
try {
|
||||
const { stdout } = await execFileAsync('docker', ['volume', 'prune', '-f']);
|
||||
const reclaimed = this._extractDockerReclaimed(stdout);
|
||||
result.bytesReclaimed += reclaimed;
|
||||
result.actions.push({ action: 'prune-unused-volumes', freedBytes: reclaimed });
|
||||
} catch { /* skip */ }
|
||||
|
||||
// Prune build cache (keep last 500MB)
|
||||
try {
|
||||
const { stdout } = await execFileAsync('docker', ['builder', 'prune', '-f', '--keep-storage', '500m']);
|
||||
const reclaimed = this._extractDockerReclaimed(stdout);
|
||||
result.bytesReclaimed += reclaimed;
|
||||
result.actions.push({ action: 'prune-build-cache', freedBytes: reclaimed });
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
|
||||
if (level === 'aggressive') {
|
||||
// Remove ALL images not used by running containers
|
||||
try {
|
||||
const { stdout } = await execFileAsync('docker', ['image', 'prune', '-a', '-f']);
|
||||
const reclaimed = this._extractDockerReclaimed(stdout);
|
||||
result.bytesReclaimed += reclaimed;
|
||||
result.actions.push({ action: 'prune-all-unused-images', freedBytes: reclaimed });
|
||||
} catch { /* skip */ }
|
||||
|
||||
// Prune stopped containers older than 24h
|
||||
try {
|
||||
const { stdout } = await execFileAsync('docker', ['container', 'prune', '-f', '--filter', 'until=24h']);
|
||||
const reclaimed = this._extractDockerReclaimed(stdout);
|
||||
result.bytesReclaimed += reclaimed;
|
||||
result.actions.push({ action: 'prune-old-containers', freedBytes: reclaimed });
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
|
||||
result.completedAt = new Date().toISOString();
|
||||
result.durationMs = Date.now() - startTime;
|
||||
result.bytesReclaimedGB = Math.round(result.bytesReclaimed / 1073741824 * 100) / 100;
|
||||
|
||||
this.lastCleanup = result;
|
||||
this.emit('cleanup-complete', result);
|
||||
|
||||
if (this.log) {
|
||||
this.log.info('disk', 'Disk cleanup completed', {
|
||||
level,
|
||||
bytesReclaimed: result.bytesReclaimed,
|
||||
GBReclaimed: result.bytesReclaimedGB,
|
||||
durationMs: result.durationMs,
|
||||
actions: result.actions.length,
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (err) {
|
||||
result.error = err.message;
|
||||
result.completedAt = new Date().toISOString();
|
||||
if (this.log) {
|
||||
this.log.error('disk', 'Disk cleanup failed', { error: err.message, level });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
_humanToBytes(str) {
|
||||
const match = str.match(/^([\d.]+)(B|KB|MB|GB|TB)$/i);
|
||||
if (!match) return 0;
|
||||
const value = parseFloat(match[1]);
|
||||
const unit = match[2].toUpperCase();
|
||||
const multipliers = { B: 1, KB: 1024, MB: 1048576, GB: 1073741824, TB: 1099511627776 };
|
||||
return Math.round(value * (multipliers[unit] || 0));
|
||||
}
|
||||
|
||||
_extractDockerReclaimed(stdout) {
|
||||
const match = stdout.match(/reclaimed\s+([\d.]+[KMGT]?B)/i) || stdout.match(/Total reclaimed space:\s*([\d.]+[KMGT]?B)/i);
|
||||
if (match) return this._humanToBytes(match[1]);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic monitoring
|
||||
* @param {number} intervalMs - check interval (default 10 minutes)
|
||||
*/
|
||||
start(intervalMs = 600000) {
|
||||
if (this.intervalHandle) return;
|
||||
this.log?.info?.('disk', 'Disk space monitor started', { intervalMs });
|
||||
// Initial check
|
||||
this.getSnapshot().catch(() => {});
|
||||
this.intervalHandle = setInterval(() => {
|
||||
this.getSnapshot().catch(() => {});
|
||||
}, intervalMs);
|
||||
}
|
||||
|
||||
stop() {
|
||||
if (this.intervalHandle) {
|
||||
clearInterval(this.intervalHandle);
|
||||
this.intervalHandle = null;
|
||||
}
|
||||
}
|
||||
|
||||
getConfig() {
|
||||
return { ...this.diskConfig };
|
||||
}
|
||||
|
||||
async getDetailedBreakdown() {
|
||||
const [snapshot, containerLogs] = await Promise.all([
|
||||
this.getSnapshot(),
|
||||
this._getContainerLogs(),
|
||||
]);
|
||||
return {
|
||||
...snapshot,
|
||||
containerLogs,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { DiskSpaceMonitor, DEFAULT_DISK_CONFIG: DEFAULT_CONFIG };
|
||||
Reference in New Issue
Block a user