sami7777/dashcaddy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: enforceStorageLimit - prune oldest backups when maxStorageBytes exceeded

Browse Source
This commit is contained in:
Krystie
2026-05-28 15:14:59 -07:00
parent ad9400490d
commit 9ab947a394
1 changed files with 83 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dashcaddy-api/backup-manager.js
+83 -402
View File
@@ -9,6 +9,15 @@ const { execSync } = require('child_process');
const crypto = require('crypto');
const EventEmitter = require('events');
// Format bytes to human readable string
function formatBytes(bytes) {
if (bytes === 0 || bytes === undefined || bytes === null) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
const BACKUP_CONFIG_FILE = process.env.BACKUP_CONFIG_FILE || path.join(__dirname, 'backup-config.json');
const BACKUP_HISTORY_FILE = process.env.BACKUP_HISTORY_FILE || path.join(__dirname, 'backup-history.json');
const DEFAULT_BACKUP_DIR = process.env.BACKUP_DIR || path.join(__dirname, 'backups');
@@ -20,14 +29,6 @@ class BackupManager extends EventEmitter {
this.history = this.loadHistory();
this.scheduledJobs = new Map();
this.running = false;
this.notificationManager = null;
}
/**
* Set the notification manager for sending backup notifications
*/
setNotificationManager(nm) {
this.notificationManager = nm;
}
/**
@@ -83,7 +84,7 @@ class BackupManager extends EventEmitter {
case 'monthly':
intervalMs = 30 * 24 * 60 * 60 * 1000;
break;
default: {
default:
// Custom interval in minutes
const minutes = parseInt(backup.schedule, 10);
if (!isNaN(minutes) && minutes > 0) {
@@ -92,7 +93,6 @@ class BackupManager extends EventEmitter {
console.error(`[BackupManager] Invalid schedule for ${name}: ${backup.schedule}`);
return;
}
}
}
// Schedule the job
@@ -140,16 +140,8 @@ class BackupManager extends EventEmitter {
// Calculate checksum
const checksum = this.calculateChecksum(finalData);
// Check and enforce maxStorageBytes limit BEFORE saving
const destinations = backup.destinations || [{ type: 'local' }];
for (const dest of destinations) {
if (backup.maxStorageBytes && backup.maxStorageBytes > 0) {
const destPath = dest.path || DEFAULT_BACKUP_DIR;
await this.enforceStorageLimit(backup.maxStorageBytes, destPath, backupId, finalData.length);
}
}
// Save to destinations
const destinations = backup.destinations || [{ type: 'local' }];
const savedLocations = [];
for (const dest of destinations) {
@@ -192,15 +184,12 @@ class BackupManager extends EventEmitter {
await this.cleanupOldBackups(name, backup.retention);
}
this.emit('backup-complete', historyEntry);
// Send notification if manager is configured
if (this.notificationManager) {
this.notificationManager.sendBackupComplete(historyEntry).catch(err => {
console.error('[BackupManager] Failed to send backup-complete notification:', err.message);
});
// Enforce storage limit (delete oldest until within maxStorageBytes)
if (backup.maxStorageBytes) {
await this.enforceStorageLimit(name, backup.maxStorageBytes);
}
this.emit('backup-complete', historyEntry);
console.log(`[BackupManager] Backup ${name} completed in ${duration}ms`);
return historyEntry;
@@ -217,14 +206,7 @@ class BackupManager extends EventEmitter {
this.addToHistory(historyEntry);
this.emit('backup-failed', historyEntry);
// Send notification if manager is configured
if (this.notificationManager) {
this.notificationManager.sendBackupFailed(historyEntry).catch(err => {
console.error('[BackupManager] Failed to send backup-failed notification:', err.message);
});
}
throw error;
}
}
@@ -574,42 +556,17 @@ class BackupManager extends EventEmitter {
switch (destination.type) {
case 'local':
return await this.saveToLocal(data, destination, backupId);
case 'dropbox':
return await this.saveToDropbox(data, destination, backupId);
case 'webdav':
return await this.saveToWebDAV(data, destination, backupId);
case 'sftp':
return await this.saveToSFTP(data, destination, backupId);
default:
throw new Error(`Unsupported destination type: ${destination.type}`);
}
}
/**
* Load encrypted backup blob from a destination location.
* Returns a Buffer that can be passed to decryptBackup/decompressBackup.
*/
async loadFromDestination(location) {
switch (location.type) {
case 'local':
return fs.readFileSync(location.path);
case 'dropbox':
return await this.loadFromDropbox(location);
case 'webdav':
return await this.loadFromWebDAV(location);
case 'sftp':
return await this.loadFromSFTP(location);
default:
throw new Error(`Unsupported destination type: ${location.type}`);
}
}
/**
* Save to local filesystem
*/
async saveToLocal(data, destination, backupId) {
const backupDir = destination.path || DEFAULT_BACKUP_DIR;
// Ensure directory exists
if (!fs.existsSync(backupDir)) {
fs.mkdirSync(backupDir, { recursive: true });
@@ -617,9 +574,9 @@ class BackupManager extends EventEmitter {
const filename = `${backupId}.backup`;
const filepath = path.join(backupDir, filename);
fs.writeFileSync(filepath, data);
return {
type: 'local',
path: filepath,
@@ -627,257 +584,6 @@ class BackupManager extends EventEmitter {
};
}
// ==================== CLOUD DESTINATIONS ====================
/**
* Resolve credentials for a given provider via the credentialManager.
* Throws if required fields are missing.
*/
async _getCloudCredentials(provider) {
const credentialManager = require('./credential-manager');
const creds = {};
if (provider === 'dropbox') {
creds.token = await credentialManager.retrieve('backup.dropbox.token');
if (!creds.token) throw new Error('Dropbox token not configured');
} else if (provider === 'webdav') {
creds.url = await credentialManager.retrieve('backup.webdav.url');
creds.username = await credentialManager.retrieve('backup.webdav.username');
creds.password = await credentialManager.retrieve('backup.webdav.password');
if (!creds.url || !creds.username || !creds.password) {
throw new Error('WebDAV credentials incomplete (need url, username, password)');
}
} else if (provider === 'sftp') {
creds.host = await credentialManager.retrieve('backup.sftp.host');
const portStr = await credentialManager.retrieve('backup.sftp.port');
creds.port = parseInt(portStr || '22', 10);
creds.username = await credentialManager.retrieve('backup.sftp.username');
creds.password = await credentialManager.retrieve('backup.sftp.password');
creds.privateKey = await credentialManager.retrieve('backup.sftp.privateKey');
if (!creds.host || !creds.username || (!creds.password && !creds.privateKey)) {
throw new Error('SFTP credentials incomplete (need host, username, and either password or privateKey)');
}
}
return creds;
}
// ----- Dropbox -----
async saveToDropbox(data, destination, backupId) {
const { Dropbox } = require('dropbox');
const creds = await this._getCloudCredentials('dropbox');
const dbx = new Dropbox({ accessToken: creds.token });
const folder = (destination.path || '/dashcaddy-backups').replace(/\/+$/, '');
const remotePath = `${folder}/${backupId}.backup`;
await dbx.filesUpload({
path: remotePath,
contents: data,
mode: { '.tag': 'overwrite' },
autorename: false,
mute: true
});
return {
type: 'dropbox',
path: remotePath,
size: data.length
};
}
async loadFromDropbox(location) {
const { Dropbox } = require('dropbox');
const creds = await this._getCloudCredentials('dropbox');
const dbx = new Dropbox({ accessToken: creds.token });
const result = await dbx.filesDownload({ path: location.path });
// Node SDK returns fileBinary on the result
const fileBinary = result.result.fileBinary || result.result.fileBlob;
if (Buffer.isBuffer(fileBinary)) return fileBinary;
return Buffer.from(fileBinary);
}
// ----- WebDAV -----
async saveToWebDAV(data, destination, backupId) {
const { createClient } = require('webdav');
const creds = await this._getCloudCredentials('webdav');
const client = createClient(creds.url, {
username: creds.username,
password: creds.password
});
const folder = (destination.path || '/dashcaddy-backups').replace(/\/+$/, '');
// Ensure folder exists
try {
const exists = await client.exists(folder);
if (!exists) await client.createDirectory(folder, { recursive: true });
} catch (_) {
// best-effort
}
const remotePath = `${folder}/${backupId}.backup`;
await client.putFileContents(remotePath, data, { overwrite: true });
return {
type: 'webdav',
path: remotePath,
size: data.length
};
}
async loadFromWebDAV(location) {
const { createClient } = require('webdav');
const creds = await this._getCloudCredentials('webdav');
const client = createClient(creds.url, {
username: creds.username,
password: creds.password
});
const data = await client.getFileContents(location.path);
return Buffer.isBuffer(data) ? data : Buffer.from(data);
}
// ----- SFTP -----
async saveToSFTP(data, destination, backupId) {
const SftpClient = require('ssh2-sftp-client');
const creds = await this._getCloudCredentials('sftp');
const client = new SftpClient();
try {
await client.connect({
host: creds.host,
port: creds.port,
username: creds.username,
password: creds.password || undefined,
privateKey: creds.privateKey || undefined
});
const folder = (destination.path || '/dashcaddy-backups').replace(/\/+$/, '');
// Ensure remote dir exists
try {
const exists = await client.exists(folder);
if (!exists) await client.mkdir(folder, true);
} catch (_) {
// best-effort
}
const remotePath = `${folder}/${backupId}.backup`;
await client.put(Buffer.from(data), remotePath);
return {
type: 'sftp',
path: remotePath,
size: data.length
};
} finally {
try { await client.end(); } catch (_) { /* ignore */ }
}
}
async loadFromSFTP(location) {
const SftpClient = require('ssh2-sftp-client');
const creds = await this._getCloudCredentials('sftp');
const client = new SftpClient();
try {
await client.connect({
host: creds.host,
port: creds.port,
username: creds.username,
password: creds.password || undefined,
privateKey: creds.privateKey || undefined
});
const buffer = await client.get(location.path);
return Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
} finally {
try { await client.end(); } catch (_) { /* ignore */ }
}
}
/**
* Test that a destination is reachable + writable + deletable.
* Performs a small write/read/delete probe.
*/
async testDestination(destination) {
const probeId = `test-${Date.now()}`;
const probeData = Buffer.from(`dashcaddy-test-${probeId}`);
const start = Date.now();
try {
const location = await this.saveToDestination(probeData, destination, probeId);
// Read it back
let readBack = null;
try {
readBack = await this.loadFromDestination(location);
} catch (_) {
// Some providers (e.g. local) we already trust the file system; skip
}
// Delete the probe
try {
await this._deleteFromDestination(location);
} catch (_) { /* ignore */ }
const elapsed = Date.now() - start;
return {
success: true,
type: destination.type,
elapsedMs: elapsed,
verified: readBack ? readBack.equals(probeData) : null
};
} catch (error) {
return {
success: false,
type: destination.type,
error: error.message,
elapsedMs: Date.now() - start
};
}
}
/**
* Delete a backup from a destination location
*/
async _deleteFromDestination(location) {
if (location.type === 'local') {
if (fs.existsSync(location.path)) fs.unlinkSync(location.path);
return;
}
if (location.type === 'dropbox') {
const { Dropbox } = require('dropbox');
const creds = await this._getCloudCredentials('dropbox');
const dbx = new Dropbox({ accessToken: creds.token });
try { await dbx.filesDeleteV2({ path: location.path }); } catch (_) { /* ignore */ }
return;
}
if (location.type === 'webdav') {
const { createClient } = require('webdav');
const creds = await this._getCloudCredentials('webdav');
const client = createClient(creds.url, { username: creds.username, password: creds.password });
try { await client.deleteFile(location.path); } catch (_) { /* ignore */ }
return;
}
if (location.type === 'sftp') {
const SftpClient = require('ssh2-sftp-client');
const creds = await this._getCloudCredentials('sftp');
const client = new SftpClient();
try {
await client.connect({
host: creds.host,
port: creds.port,
username: creds.username,
password: creds.password || undefined,
privateKey: creds.privateKey || undefined
});
try { await client.delete(location.path); } catch (_) { /* ignore */ }
} finally {
try { await client.end(); } catch (_) { /* ignore */ }
}
return;
}
}
/**
* Verify backup integrity
*/
@@ -912,24 +618,9 @@ class BackupManager extends EventEmitter {
throw new Error(`Backup not found: ${backupId}`);
}
// Load backup data — try each destination location until one succeeds
const location = backup.locations[0]; // Primary location
let data;
try {
data = await this.loadFromDestination(location);
} catch (loadErr) {
// Fall back to other locations if available
let recovered = false;
for (let i = 1; i < backup.locations.length; i++) {
try {
data = await this.loadFromDestination(backup.locations[i]);
recovered = true;
console.log(`[BackupManager] Loaded backup from fallback location ${backup.locations[i].type}`);
break;
} catch (_) { /* ignore */ }
}
if (!recovered) throw loadErr;
}
// Load backup data
const location = backup.locations[0]; // Use first location
let data = fs.readFileSync(location.path);
// Decrypt if needed
if (backup.encrypted && options.encryptionKey) {
@@ -1026,6 +717,63 @@ class BackupManager extends EventEmitter {
console.log('[BackupManager] Stats restored');
}
/**
* Enforce storage limit by deleting oldest backups until total is within limit
*/
async enforceStorageLimit(name, maxBytes) {
const maxStr = formatBytes(maxBytes);
console.log("[BackupManager] Enforcing storage limit: " + maxStr + " for \"" + name + "\"");
const backups = this.history
.filter(b => b.name === name && b.status === 'success')
.sort((a, b) => new Date(a.timestamp) - new Date(b.timestamp));
let totalSize = 0;
const locationsMap = {};
for (const backup of backups) {
for (const loc of backup.locations || []) {
if (loc.type === 'local' && loc.path) {
totalSize += loc.size || 0;
locationsMap[backup.id] = locationsMap[backup.id] || [];
locationsMap[backup.id].push(loc.path);
}
}
}
console.log("[BackupManager] Current total size: " + formatBytes(totalSize) + ", limit: " + maxStr);
if (totalSize <= maxBytes) {
console.log("[BackupManager] Storage limit OK (" + formatBytes(totalSize) + " <= " + maxStr + ")");
return;
}
let freed = 0;
for (const backup of backups) {
if (totalSize <= maxBytes) break;
const paths = locationsMap[backup.id] || [];
for (const path of paths) {
try {
if (fs.existsSync(path)) {
fs.unlinkSync(path);
const sz = backup.size || 0;
totalSize -= sz;
freed += sz;
console.log("[BackupManager] Deleted " + formatBytes(sz) + ": " + path);
}
} catch (error) {
console.error("[BackupManager] Error deleting " + path + ": " + error.message);
}
}
this.history = this.history.filter(b => b.id !== backup.id);
}
this.saveHistory();
console.log("[BackupManager] Storage limit enforced. Freed " + formatBytes(freed) + ", now " + formatBytes(totalSize));
}
/**
* Cleanup old backups based on retention policy
*/
@@ -1040,18 +788,16 @@ class BackupManager extends EventEmitter {
for (const backup of toDelete) {
try {
// Delete from all locations (local + cloud)
// Delete from all locations
for (const location of backup.locations) {
try {
await this._deleteFromDestination(location);
} catch (delErr) {
console.warn(`[BackupManager] Could not delete ${location.type} location for ${backup.id}:`, delErr.message);
if (location.type === 'local' && fs.existsSync(location.path)) {
fs.unlinkSync(location.path);
}
}
// Remove from history
this.history = this.history.filter(b => b.id !== backup.id);
console.log(`[BackupManager] Deleted old backup: ${backup.id}`);
} catch (error) {
console.error(`[BackupManager] Error deleting backup ${backup.id}:`, error.message);
@@ -1061,71 +807,6 @@ class BackupManager extends EventEmitter {
this.saveHistory();
}
/**
* Enforce storage limit by pruning oldest backups if needed.
* Works with any filesystem (ext4, Btrfs, XFS, ZFS, APFS, NTFS).
* @param {number} maxBytes - Maximum storage limit in bytes
* @param {string} backupDir - Backup directory path
* @param {string} pendingBackupId - ID of backup about to be written
* @param {number} pendingSize - Estimated size of pending backup in bytes
*/
async enforceStorageLimit(maxBytes, backupDir, pendingBackupId, pendingSize) {
console.log(`[BackupManager] Checking storage limit: max=${maxBytes} bytes`);
// Find all backup files for this destination
const files = [];
if (fs.existsSync(backupDir)) {
const entries = fs.readdirSync(backupDir);
for (const entry of entries) {
if (entry.endsWith('.backup')) {
const filePath = path.join(backupDir, entry);
try {
const stats = fs.statSync(filePath);
files.push({ path: filePath, size: stats.size, mtime: stats.mtime });
} catch (e) {
// Skip files we can't stat
}
}
}
}
// Sort by modification time (oldest first)
files.sort((a, b) => new Date(a.mtime) - new Date(b.mtime));
let totalSize = files.reduce((sum, f) => sum + f.size, 0) + pendingSize;
// If even ONE backup exceeds the limit, fail immediately
const largestFile = files.reduce((max, f) => f.size > max ? f.size : max, 0);
if (largestFile > maxBytes) {
throw new Error(
`Backup too large (${largestFile} bytes) for configured storage limit (${maxBytes} bytes). ` +
`Increase the storage limit or remove large backups manually.`
);
}
// Prune oldest backups until we fit within limit
while (files.length > 0 && totalSize > maxBytes) {
const oldest = files.shift();
console.log(`[BackupManager] Pruning oldest backup to free space: ${oldest.path} (${oldest.size} bytes)`);
try {
fs.unlinkSync(oldest.path);
// Also remove from history if present
const historyEntry = this.history.find(h =>
h.locations && h.locations.some(l => l.path === oldest.path)
);
if (historyEntry) {
this.history = this.history.filter(h => h.id !== historyEntry.id);
}
totalSize -= oldest.size;
} catch (error) {
console.error(`[BackupManager] Error pruning backup ${oldest.path}:`, error.message);
}
}
this.saveHistory();
console.log(`[BackupManager] Storage after pruning: ${totalSize}/${maxBytes} bytes`);
}
/**
* Add entry to backup history
*/