feat: Log Insights panel — plain English activity summary + safe log disposal
- New route /api/v1/log-insights: analyzes audit logs + security events
- Shows top IPs with request counts, failures, and top actions
- Plain English insights (heavy users, auth failures, security alerts)
- Summary stats: total requests, unique IPs, failed actions
- Storage info showing log file sizes and entry counts
- New route POST /api/v1/log-insights/dispose: preview-then-confirm cleanup
- First call shows what would be deleted (preview mode)
- Second call with confirm:true actually deletes
- Configurable retention period (default 30 days)
- Frontend panel with modal UI showing insights as cards
- Period selector (1h, 6h, 24h, 7d)
- Top visitors table with IP, requests, failures, actions, last seen
- Storage info footer
- Clean Old Logs button with preview confirmation dialog
- Wired into app.js and dashboard navbar (🔍 Insights button)
- Addresses QA issue: users need to see who is accessing before cleanup
This commit is contained in:
Krystie
@@ -0,0 +1,153 @@
|
||||
const express = require('express');
|
||||
const fs = require('fs').promises;
|
||||
|
||||
module.exports = function({ asyncHandler, ok, auditLogger, securityEventStore }) {
|
||||
const router = express.Router();
|
||||
|
||||
// GET /api/v1/log-insights — Plain English summary of who's doing what
|
||||
router.get('/log-insights', asyncHandler(async (req, res) => {
|
||||
const hours = parseInt(req.query.hours) || 24;
|
||||
const since = new Date(Date.now() - hours * 60 * 60 * 1000).toISOString();
|
||||
|
||||
// --- Collect data ---
|
||||
const auditEntries = await auditLogger.query({ limit: 10000 });
|
||||
const recentAudit = auditEntries.filter(e => e.timestamp >= since);
|
||||
|
||||
let securityEvents = [];
|
||||
try { securityEvents = securityEventStore.query({ since, limit: 10000 }); } catch {}
|
||||
|
||||
// --- Analyze IPs ---
|
||||
const ipMap = {};
|
||||
recentAudit.forEach(e => {
|
||||
const ip = e.ip || 'unknown';
|
||||
if (!ipMap[ip]) ipMap[ip] = { count: 0, actions: {}, resources: new Set(), first: e.timestamp, last: e.timestamp, failures: 0 };
|
||||
const s = ipMap[ip];
|
||||
s.count++;
|
||||
const cat = (e.action || 'unknown').split('.')[0];
|
||||
s.actions[cat] = (s.actions[cat] || 0) + 1;
|
||||
if (e.resource) s.resources.add(e.resource);
|
||||
if (e.timestamp < s.first) s.first = e.timestamp;
|
||||
if (e.timestamp > s.last) s.last = e.timestamp;
|
||||
if (e.outcome === 'failure' || e.outcome === 'denied') s.failures++;
|
||||
});
|
||||
|
||||
// --- Build plain-English insights ---
|
||||
const insights = [];
|
||||
const ipArray = Object.entries(ipMap).sort((a, b) => b[1].count - a[1].count);
|
||||
|
||||
// Heavy users
|
||||
ipArray.slice(0, 3).forEach(([ip, s]) => {
|
||||
const topAction = Object.entries(s.actions).sort((a, b) => b[1] - a[1])[0];
|
||||
insights.push({
|
||||
severity: s.count > 500 ? 'warning' : 'info',
|
||||
title: ip + ' — ' + s.count + ' requests in ' + hours + 'h',
|
||||
plain: ip + ' made ' + s.count + ' requests (mostly ' + (topAction ? topAction[0] : 'unknown') + ')' +
|
||||
(s.failures > 0 ? ', ' + s.failures + ' failed' : '') + '.'
|
||||
});
|
||||
});
|
||||
|
||||
// Auth failures
|
||||
const totalFailures = recentAudit.filter(e => e.outcome === 'failure' || e.outcome === 'denied').length;
|
||||
if (totalFailures > 5) {
|
||||
insights.push({
|
||||
severity: totalFailures > 50 ? 'warning' : 'info',
|
||||
title: totalFailures + ' failed actions',
|
||||
plain: totalFailures + ' requests were denied or failed in the last ' + hours + ' hours.' +
|
||||
(totalFailures > 50 ? ' This could indicate someone trying to brute-force access.' : '')
|
||||
});
|
||||
}
|
||||
|
||||
// Security events
|
||||
const secBySev = {};
|
||||
securityEvents.forEach(e => { secBySev[e.severity] = (secBySev[e.severity] || 0) + 1; });
|
||||
if (secBySev.critical || secBySev.error) {
|
||||
insights.push({
|
||||
severity: 'warning',
|
||||
title: ((secBySev.critical || 0) + (secBySev.error || 0)) + ' security alerts',
|
||||
plain: (secBySev.critical || 0) + ' critical and ' + (secBySev.error || 0) + ' error-level security events were logged.'
|
||||
});
|
||||
}
|
||||
|
||||
// Quiet / nothing
|
||||
if (insights.length === 0) {
|
||||
insights.push({ severity: 'ok', title: 'All quiet', plain: 'No notable activity in the last ' + hours + ' hours.' });
|
||||
}
|
||||
|
||||
// --- Storage info ---
|
||||
const auditPath = process.env.AUDIT_LOG_FILE || '/opt/dashcaddy/dashcaddy-api/data/audit-log.json';
|
||||
const secPath = process.env.SECURITY_EVENT_LOG_FILE || '/opt/dashcaddy/dashcaddy-api/data/security-events.jsonl';
|
||||
let storage = {};
|
||||
try {
|
||||
const a = await fs.stat(auditPath);
|
||||
storage.auditLog = { sizeMB: +(a.size / 1048576).toFixed(2), entries: auditEntries.length };
|
||||
} catch {}
|
||||
try {
|
||||
const s = await fs.stat(secPath);
|
||||
storage.securityEvents = { sizeMB: +(s.size / 1048576).toFixed(2), entries: securityEvents.length };
|
||||
} catch {}
|
||||
|
||||
ok(res, {
|
||||
period: { hours, since, until: new Date().toISOString() },
|
||||
summary: {
|
||||
totalRequests: recentAudit.length,
|
||||
uniqueIPs: ipArray.length,
|
||||
securityEvents: securityEvents.length,
|
||||
failedActions: totalFailures
|
||||
},
|
||||
topIPs: ipArray.slice(0, 10).map(([ip, s]) => ({
|
||||
ip: ip,
|
||||
count: s.count,
|
||||
failures: s.failures,
|
||||
topActions: Object.entries(s.actions).sort((a, b) => b[1] - a[1]).slice(0, 3),
|
||||
activeFrom: s.first,
|
||||
lastSeen: s.last
|
||||
})),
|
||||
insights: insights,
|
||||
storage: storage
|
||||
});
|
||||
}));
|
||||
|
||||
// POST /api/v1/log-insights/dispose — Preview then confirm cleanup
|
||||
router.post('/log-insights/dispose', asyncHandler(async (req, res) => {
|
||||
const keepDays = parseInt(req.body.keepDays) || 30;
|
||||
const confirm = req.body.confirm === true;
|
||||
const cutoff = new Date(Date.now() - keepDays * 86400000).toISOString();
|
||||
|
||||
const auditPath = process.env.AUDIT_LOG_FILE || '/opt/dashcaddy/dashcaddy-api/data/audit-log.json';
|
||||
const secPath = process.env.SECURITY_EVENT_LOG_FILE || '/opt/dashcaddy/dashcaddy-api/data/security-events.jsonl';
|
||||
|
||||
const auditRaw = await fs.readFile(auditPath, 'utf8').catch(function () { return '[]'; });
|
||||
const auditData = JSON.parse(auditRaw);
|
||||
const oldAudit = auditData.filter(function (e) { return e.timestamp < cutoff; });
|
||||
|
||||
const secRaw = await fs.readFile(secPath, 'utf8').catch(function () { return ''; });
|
||||
const secLines = secRaw.split('\n').filter(Boolean);
|
||||
const oldSec = secLines.filter(function (l) { try { return JSON.parse(l).timestamp < cutoff; } catch (e) { return false; } });
|
||||
|
||||
if (!confirm) {
|
||||
ok(res, {
|
||||
preview: true,
|
||||
message: 'This will delete ' + oldAudit.length + ' audit entries and ' + oldSec.length + ' security events older than ' + keepDays + ' days. Send {confirm: true} to proceed.',
|
||||
wouldDelete: { auditEntries: oldAudit.length, securityEvents: oldSec.length },
|
||||
cutoffDate: cutoff
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Execute cleanup
|
||||
const keptAudit = auditData.filter(function (e) { return e.timestamp >= cutoff; });
|
||||
await fs.writeFile(auditPath, JSON.stringify(keptAudit, null, 2));
|
||||
|
||||
const keptSec = secLines.filter(function (l) { try { return JSON.parse(l).timestamp >= cutoff; } catch (e) { return false; } });
|
||||
await fs.writeFile(secPath, keptSec.join('\n') + '\n');
|
||||
|
||||
ok(res, {
|
||||
disposed: true,
|
||||
deleted: { auditEntries: oldAudit.length, securityEvents: oldSec.length },
|
||||
remaining: { auditEntries: keptAudit.length, securityEvents: keptSec.length },
|
||||
cutoffDate: cutoff
|
||||
});
|
||||
}));
|
||||
|
||||
return router;
|
||||
};
|
||||
@@ -93,6 +93,7 @@ const eventsRoutes = require('../routes/events');
|
||||
const workflowsRoutes = require('../routes/workflows');
|
||||
const dependenciesRoutes = require('../routes/dependencies');
|
||||
const securityRoutes = require('../routes/security');
|
||||
const logInsightsRoutes = require('../routes/log-insights');
|
||||
const billingRoutes = require('../routes/billing');
|
||||
const DependencyManager = require('./managers/dependency-manager');
|
||||
const autoRestartRoutes = require('../routes/auto-restart');
|
||||
@@ -753,6 +754,20 @@ async function createApp() {
|
||||
apiRouter.use('/security', securityRoutes({
|
||||
log: ctx.log,
|
||||
}));
|
||||
|
||||
// Log Insights — plain English activity summary + safe log disposal
|
||||
apiRouter.use(logInsightsRoutes({
|
||||
asyncHandler: ctx.asyncHandler,
|
||||
ok: ctx.ok,
|
||||
auditLogger: ctx.auditLogger,
|
||||
securityEventStore: (function() {
|
||||
try {
|
||||
var getStore = require('./security/event-store').getStore;
|
||||
return getStore();
|
||||
} catch (e) { return null; }
|
||||
})()
|
||||
}));
|
||||
|
||||
apiRouter.use('/dependencies', dependenciesRoutes({
|
||||
dependencyManager: ctx.dependencyManager,
|
||||
servicesStateManager: ctx.servicesStateManager,
|
||||
|
||||
@@ -206,6 +206,7 @@
|
||||
<button id="manage-notifications" aria-label="Manage notifications">🔔 Alerts</button>
|
||||
<button id="audit-log-btn" aria-label="Audit log">📜 Audit</button>
|
||||
<button id="security-center-btn" aria-label="Security Center">🛡️ Security</button>
|
||||
<button id="log-insights-btn" aria-label="Log Insights">🔍 Insights</button>
|
||||
<button id="docker-resources-btn" aria-label="Docker resources">🐳 Docker</button>
|
||||
</div>
|
||||
</div>
|
||||
@@ -950,6 +951,7 @@
|
||||
<script src="/js/xterm-fit.min.js" defer></script>
|
||||
|
||||
<!-- Tailscale device list panel (self-contained, polls /api/v1/tailscale/devices) -->
|
||||
<script src="/js/log-insights.js" defer></script>
|
||||
<script src="/js/tailscale-devices.js" defer></script>
|
||||
|
||||
<!-- Bundled JS (built with: npm run build) -->
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
// ========== LOG INSIGHTS PANEL ==========
|
||||
(function() {
|
||||
injectModal('log-insights-modal', `<div id="log-insights-modal" class="weather-modal">
|
||||
<div class="weather-modal-content" style="min-width: 800px; max-width: 1000px;">
|
||||
<h3>🔍 Log Insights</h3>
|
||||
<p class="modal-subtitle">Who's accessing your server and what they're doing — in plain English.</p>
|
||||
|
||||
<div style="display: flex; gap: 12px; margin-bottom: 16px; align-items: center;">
|
||||
<label class="text-muted-sm">Period:</label>
|
||||
<select id="li-period" style="padding: 6px 10px; border-radius: 6px; border: 1px solid var(--border); background: var(--bg); color: var(--fg); font-size: 0.85rem;">
|
||||
<option value="1">Last 1 hour</option>
|
||||
<option value="6">Last 6 hours</option>
|
||||
<option value="24" selected>Last 24 hours</option>
|
||||
<option value="168">Last 7 days</option>
|
||||
</select>
|
||||
<button id="li-refresh" class="btn-sm">🔄 Refresh</button>
|
||||
<span style="flex: 1;"></span>
|
||||
<button id="li-dispose-btn" style="padding: 6px 12px; font-size: 0.8rem; color: var(--warn-fg, #f0c674); border-color: var(--warn-fg, #f0c674);">🧹 Clean Old Logs</button>
|
||||
</div>
|
||||
|
||||
<!-- Plain English Insights -->
|
||||
<div id="li-insights" style="margin-bottom: 16px;"></div>
|
||||
|
||||
<!-- Summary Stats -->
|
||||
<div id="li-summary" style="display: grid; grid-template-columns: repeat(4, 1fr); gap: 12px; margin-bottom: 16px;"></div>
|
||||
|
||||
<!-- Top IPs Table -->
|
||||
<div id="li-ips-section">
|
||||
<h4 style="margin: 12px 0 8px; font-size: 0.95rem;">Top Visitors</h4>
|
||||
<div id="li-ips-table" class="scroll-container" style="max-height: 300px;"></div>
|
||||
</div>
|
||||
|
||||
<!-- Storage Info -->
|
||||
<div id="li-storage" style="margin-top: 16px; padding: 12px; background: var(--card-base); border-radius: 8px; border: 1px solid var(--border);"></div>
|
||||
|
||||
<div class="weather-modal-buttons">
|
||||
<button id="li-close">Close</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>`);
|
||||
|
||||
const modal = document.getElementById('log-insights-modal');
|
||||
const openBtn = document.getElementById('log-insights-btn');
|
||||
const closeBtn = document.getElementById('li-close');
|
||||
const refreshBtn = document.getElementById('li-refresh');
|
||||
const disposeBtn = document.getElementById('li-dispose-btn');
|
||||
const periodSel = document.getElementById('li-period');
|
||||
const insightsDiv = document.getElementById('li-insights');
|
||||
const summaryDiv = document.getElementById('li-summary');
|
||||
const ipsDiv = document.getElementById('li-ips-table');
|
||||
const storageDiv = document.getElementById('li-storage');
|
||||
|
||||
if (openBtn) {
|
||||
openBtn.addEventListener('click', () => { modal.style.display = 'flex'; loadInsights(); });
|
||||
}
|
||||
closeBtn.addEventListener('click', () => modal.style.display = 'none');
|
||||
refreshBtn.addEventListener('click', loadInsights);
|
||||
periodSel.addEventListener('change', loadInsights);
|
||||
disposeBtn.addEventListener('click', showDisposePreview);
|
||||
|
||||
async function loadInsights() {
|
||||
const hours = periodSel.value;
|
||||
insightsDiv.innerHTML = '<div class="panel-empty"><span class="brand-spinner"></span> Analyzing logs...</div>';
|
||||
summaryDiv.innerHTML = '';
|
||||
ipsDiv.innerHTML = '';
|
||||
storageDiv.innerHTML = '';
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/v1/log-insights?hours=' + hours);
|
||||
const data = await res.json();
|
||||
if (!data.success) { insightsDiv.innerHTML = '<div class="panel-empty">Error: ' + data.error + '</div>'; return; }
|
||||
|
||||
// Render insights as plain English cards
|
||||
let insightsHtml = '';
|
||||
(data.insights || []).forEach(function(ins) {
|
||||
const sevColor = ins.severity === 'warning' ? 'var(--warn-fg, #f0c674)' :
|
||||
ins.severity === 'critical' ? 'var(--bad-fg, #ff6b6b)' :
|
||||
ins.severity === 'ok' ? 'var(--good-fg, #98c379)' : 'var(--muted)';
|
||||
insightsHtml += '<div style="padding: 10px 14px; margin-bottom: 8px; background: var(--bg); border-radius: 6px; border-left: 3px solid ' + sevColor + ';">' +
|
||||
'<strong style="font-size: 0.9rem;">' + ins.title + '</strong><br>' +
|
||||
'<span style="font-size: 0.85rem; color: var(--muted);">' + ins.plain + '</span></div>';
|
||||
});
|
||||
insightsDiv.innerHTML = insightsHtml;
|
||||
|
||||
// Summary stats
|
||||
var s = data.summary;
|
||||
summaryDiv.innerHTML =
|
||||
statCard('Requests', s.totalRequests) +
|
||||
statCard('Unique IPs', s.uniqueIPs) +
|
||||
statCard('Security Events', s.securityEvents) +
|
||||
statCard('Failed Actions', s.failedActions);
|
||||
|
||||
// Top IPs table
|
||||
var ips = data.topIPs || [];
|
||||
if (ips.length === 0) {
|
||||
ipsDiv.innerHTML = '<div class="panel-empty">No activity in this period.</div>';
|
||||
} else {
|
||||
var html = '<table style="width: 100%; font-size: 0.85rem; border-collapse: collapse;">';
|
||||
html += '<tr style="border-bottom: 1px solid var(--border);"><th style="text-align:left; padding: 6px;">IP Address</th><th style="text-align:right; padding: 6px;">Requests</th><th style="text-align:right; padding: 6px;">Failures</th><th style="text-align:left; padding: 6px;">Top Actions</th><th style="text-align:left; padding: 6px;">Last Seen</th></tr>';
|
||||
ips.forEach(function(ip) {
|
||||
var failStyle = ip.failures > 0 ? 'color: var(--bad-fg, #ff6b6b); font-weight: 600;' : '';
|
||||
var actions = (ip.topActions || []).map(function(a) { return a[0]; }).join(', ');
|
||||
var lastSeen = ip.lastSeen ? new Date(ip.lastSeen).toLocaleString() : '?';
|
||||
html += '<tr style="border-bottom: 1px solid var(--border);">' +
|
||||
'<td style="padding: 6px; font-family: monospace;">' + ip.ip + '</td>' +
|
||||
'<td style="padding: 6px; text-align: right;">' + ip.count + '</td>' +
|
||||
'<td style="padding: 6px; text-align: right; ' + failStyle + '">' + ip.failures + '</td>' +
|
||||
'<td style="padding: 6px;">' + actions + '</td>' +
|
||||
'<td style="padding: 6px; color: var(--muted);">' + lastSeen + '</td>' +
|
||||
'</tr>';
|
||||
});
|
||||
html += '</table>';
|
||||
ipsDiv.innerHTML = html;
|
||||
}
|
||||
|
||||
// Storage info
|
||||
var st = data.storage || {};
|
||||
var stHtml = '<strong style="font-size: 0.85rem;">Log Storage</strong><br>';
|
||||
if (st.auditLog) stHtml += '<span style="font-size: 0.8rem; color: var(--muted);">Audit log: ' + st.auditLog.sizeMB + ' MB (' + st.auditLog.entries + ' entries)</span><br>';
|
||||
if (st.securityEvents) stHtml += '<span style="font-size: 0.8rem; color: var(--muted);">Security events: ' + st.securityEvents.sizeMB + ' MB (' + st.securityEvents.entries + ' entries)</span>';
|
||||
storageDiv.innerHTML = stHtml;
|
||||
|
||||
} catch (e) {
|
||||
insightsDiv.innerHTML = '<div class="panel-empty">Failed to load: ' + e.message + '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
function statCard(label, value) {
|
||||
return '<div style="text-align: center; padding: 12px; background: var(--card-base); border-radius: 8px; border: 1px solid var(--border);">' +
|
||||
'<div style="font-size: 1.5rem; font-weight: 700;">' + value + '</div>' +
|
||||
'<div style="font-size: 0.75rem; color: var(--muted);">' + label + '</div></div>';
|
||||
}
|
||||
|
||||
async function showDisposePreview() {
|
||||
var keepDays = prompt('Delete logs older than how many days?', '30');
|
||||
if (!keepDays) return;
|
||||
keepDays = parseInt(keepDays);
|
||||
if (isNaN(keepDays) || keepDays < 1) { alert('Invalid number'); return; }
|
||||
|
||||
try {
|
||||
var res = await fetch('/api/v1/log-insights/dispose', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ keepDays: keepDays })
|
||||
});
|
||||
var data = await res.json();
|
||||
if (!data.success) { alert('Error: ' + data.error); return; }
|
||||
|
||||
var msg = data.message + '\n\n' +
|
||||
'Audit entries to delete: ' + data.wouldDelete.auditEntries + '\n' +
|
||||
'Security events to delete: ' + data.wouldDelete.securityEvents + '\n\n' +
|
||||
'Click OK to confirm deletion.';
|
||||
if (confirm(msg)) {
|
||||
await executeDispose(keepDays);
|
||||
}
|
||||
} catch (e) {
|
||||
alert('Failed: ' + e.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function executeDispose(keepDays) {
|
||||
try {
|
||||
var res = await fetch('/api/v1/log-insights/dispose', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ keepDays: keepDays, confirm: true })
|
||||
});
|
||||
var data = await res.json();
|
||||
if (!data.success) { alert('Error: ' + data.error); return; }
|
||||
|
||||
alert('Cleaned up!\n\nDeleted: ' + data.deleted.auditEntries + ' audit entries, ' + data.deleted.securityEvents + ' security events.\nRemaining: ' + data.remaining.auditEntries + ' audit, ' + data.remaining.securityEvents + ' security.');
|
||||
loadInsights();
|
||||
} catch (e) {
|
||||
alert('Failed: ' + e.message);
|
||||
}
|
||||
}
|
||||
|
||||
function injectModal(id, html) {
|
||||
if (document.getElementById(id)) return;
|
||||
var div = document.createElement('div');
|
||||
div.innerHTML = html;
|
||||
document.body.appendChild(div.firstElementChild);
|
||||
}
|
||||
})();
|
||||
Reference in New Issue
Block a user