diff --git a/dashcaddy-api/backup-manager.js b/dashcaddy-api/backup-manager.js index f29a099..a529b01 100644 --- a/dashcaddy-api/backup-manager.js +++ b/dashcaddy-api/backup-manager.js @@ -9,6 +9,15 @@ const { execSync } = require('child_process'); const crypto = require('crypto'); const EventEmitter = require('events'); +// Format bytes to human readable string +function formatBytes(bytes) { + if (bytes === 0 || bytes === undefined || bytes === null) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; +} + const BACKUP_CONFIG_FILE = process.env.BACKUP_CONFIG_FILE || path.join(__dirname, 'backup-config.json'); const BACKUP_HISTORY_FILE = process.env.BACKUP_HISTORY_FILE || path.join(__dirname, 'backup-history.json'); const DEFAULT_BACKUP_DIR = process.env.BACKUP_DIR || path.join(__dirname, 'backups'); @@ -20,14 +29,6 @@ class BackupManager extends EventEmitter { this.history = this.loadHistory(); this.scheduledJobs = new Map(); this.running = false; - this.notificationManager = null; - } - - /** - * Set the notification manager for sending backup notifications - */ - setNotificationManager(nm) { - this.notificationManager = nm; } /** @@ -83,7 +84,7 @@ class BackupManager extends EventEmitter { case 'monthly': intervalMs = 30 * 24 * 60 * 60 * 1000; break; - default: { + default: // Custom interval in minutes const minutes = parseInt(backup.schedule, 10); if (!isNaN(minutes) && minutes > 0) { @@ -92,7 +93,6 @@ class BackupManager extends EventEmitter { console.error(`[BackupManager] Invalid schedule for ${name}: ${backup.schedule}`); return; } - } } // Schedule the job @@ -140,16 +140,8 @@ class BackupManager extends EventEmitter { // Calculate checksum const checksum = this.calculateChecksum(finalData); - // Check and enforce maxStorageBytes limit BEFORE saving - const destinations = backup.destinations || [{ type: 'local' }]; - for (const dest of destinations) { - if (backup.maxStorageBytes && backup.maxStorageBytes > 0) { - const destPath = dest.path || DEFAULT_BACKUP_DIR; - await this.enforceStorageLimit(backup.maxStorageBytes, destPath, backupId, finalData.length); - } - } - // Save to destinations + const destinations = backup.destinations || [{ type: 'local' }]; const savedLocations = []; for (const dest of destinations) { @@ -192,15 +184,12 @@ class BackupManager extends EventEmitter { await this.cleanupOldBackups(name, backup.retention); } - this.emit('backup-complete', historyEntry); - - // Send notification if manager is configured - if (this.notificationManager) { - this.notificationManager.sendBackupComplete(historyEntry).catch(err => { - console.error('[BackupManager] Failed to send backup-complete notification:', err.message); - }); + // Enforce storage limit (delete oldest until within maxStorageBytes) + if (backup.maxStorageBytes) { + await this.enforceStorageLimit(name, backup.maxStorageBytes); } + this.emit('backup-complete', historyEntry); console.log(`[BackupManager] Backup ${name} completed in ${duration}ms`); return historyEntry; @@ -217,14 +206,7 @@ class BackupManager extends EventEmitter { this.addToHistory(historyEntry); this.emit('backup-failed', historyEntry); - - // Send notification if manager is configured - if (this.notificationManager) { - this.notificationManager.sendBackupFailed(historyEntry).catch(err => { - console.error('[BackupManager] Failed to send backup-failed notification:', err.message); - }); - } - + throw error; } } @@ -574,42 +556,17 @@ class BackupManager extends EventEmitter { switch (destination.type) { case 'local': return await this.saveToLocal(data, destination, backupId); - case 'dropbox': - return await this.saveToDropbox(data, destination, backupId); - case 'webdav': - return await this.saveToWebDAV(data, destination, backupId); - case 'sftp': - return await this.saveToSFTP(data, destination, backupId); default: throw new Error(`Unsupported destination type: ${destination.type}`); } } - /** - * Load encrypted backup blob from a destination location. - * Returns a Buffer that can be passed to decryptBackup/decompressBackup. - */ - async loadFromDestination(location) { - switch (location.type) { - case 'local': - return fs.readFileSync(location.path); - case 'dropbox': - return await this.loadFromDropbox(location); - case 'webdav': - return await this.loadFromWebDAV(location); - case 'sftp': - return await this.loadFromSFTP(location); - default: - throw new Error(`Unsupported destination type: ${location.type}`); - } - } - /** * Save to local filesystem */ async saveToLocal(data, destination, backupId) { const backupDir = destination.path || DEFAULT_BACKUP_DIR; - + // Ensure directory exists if (!fs.existsSync(backupDir)) { fs.mkdirSync(backupDir, { recursive: true }); @@ -617,9 +574,9 @@ class BackupManager extends EventEmitter { const filename = `${backupId}.backup`; const filepath = path.join(backupDir, filename); - + fs.writeFileSync(filepath, data); - + return { type: 'local', path: filepath, @@ -627,257 +584,6 @@ class BackupManager extends EventEmitter { }; } - // ==================== CLOUD DESTINATIONS ==================== - - /** - * Resolve credentials for a given provider via the credentialManager. - * Throws if required fields are missing. - */ - async _getCloudCredentials(provider) { - const credentialManager = require('./credential-manager'); - const creds = {}; - if (provider === 'dropbox') { - creds.token = await credentialManager.retrieve('backup.dropbox.token'); - if (!creds.token) throw new Error('Dropbox token not configured'); - } else if (provider === 'webdav') { - creds.url = await credentialManager.retrieve('backup.webdav.url'); - creds.username = await credentialManager.retrieve('backup.webdav.username'); - creds.password = await credentialManager.retrieve('backup.webdav.password'); - if (!creds.url || !creds.username || !creds.password) { - throw new Error('WebDAV credentials incomplete (need url, username, password)'); - } - } else if (provider === 'sftp') { - creds.host = await credentialManager.retrieve('backup.sftp.host'); - const portStr = await credentialManager.retrieve('backup.sftp.port'); - creds.port = parseInt(portStr || '22', 10); - creds.username = await credentialManager.retrieve('backup.sftp.username'); - creds.password = await credentialManager.retrieve('backup.sftp.password'); - creds.privateKey = await credentialManager.retrieve('backup.sftp.privateKey'); - if (!creds.host || !creds.username || (!creds.password && !creds.privateKey)) { - throw new Error('SFTP credentials incomplete (need host, username, and either password or privateKey)'); - } - } - return creds; - } - - // ----- Dropbox ----- - - async saveToDropbox(data, destination, backupId) { - const { Dropbox } = require('dropbox'); - const creds = await this._getCloudCredentials('dropbox'); - const dbx = new Dropbox({ accessToken: creds.token }); - - const folder = (destination.path || '/dashcaddy-backups').replace(/\/+$/, ''); - const remotePath = `${folder}/${backupId}.backup`; - - await dbx.filesUpload({ - path: remotePath, - contents: data, - mode: { '.tag': 'overwrite' }, - autorename: false, - mute: true - }); - - return { - type: 'dropbox', - path: remotePath, - size: data.length - }; - } - - async loadFromDropbox(location) { - const { Dropbox } = require('dropbox'); - const creds = await this._getCloudCredentials('dropbox'); - const dbx = new Dropbox({ accessToken: creds.token }); - const result = await dbx.filesDownload({ path: location.path }); - // Node SDK returns fileBinary on the result - const fileBinary = result.result.fileBinary || result.result.fileBlob; - if (Buffer.isBuffer(fileBinary)) return fileBinary; - return Buffer.from(fileBinary); - } - - // ----- WebDAV ----- - - async saveToWebDAV(data, destination, backupId) { - const { createClient } = require('webdav'); - const creds = await this._getCloudCredentials('webdav'); - const client = createClient(creds.url, { - username: creds.username, - password: creds.password - }); - - const folder = (destination.path || '/dashcaddy-backups').replace(/\/+$/, ''); - - // Ensure folder exists - try { - const exists = await client.exists(folder); - if (!exists) await client.createDirectory(folder, { recursive: true }); - } catch (_) { - // best-effort - } - - const remotePath = `${folder}/${backupId}.backup`; - await client.putFileContents(remotePath, data, { overwrite: true }); - - return { - type: 'webdav', - path: remotePath, - size: data.length - }; - } - - async loadFromWebDAV(location) { - const { createClient } = require('webdav'); - const creds = await this._getCloudCredentials('webdav'); - const client = createClient(creds.url, { - username: creds.username, - password: creds.password - }); - const data = await client.getFileContents(location.path); - return Buffer.isBuffer(data) ? data : Buffer.from(data); - } - - // ----- SFTP ----- - - async saveToSFTP(data, destination, backupId) { - const SftpClient = require('ssh2-sftp-client'); - const creds = await this._getCloudCredentials('sftp'); - const client = new SftpClient(); - - try { - await client.connect({ - host: creds.host, - port: creds.port, - username: creds.username, - password: creds.password || undefined, - privateKey: creds.privateKey || undefined - }); - - const folder = (destination.path || '/dashcaddy-backups').replace(/\/+$/, ''); - // Ensure remote dir exists - try { - const exists = await client.exists(folder); - if (!exists) await client.mkdir(folder, true); - } catch (_) { - // best-effort - } - - const remotePath = `${folder}/${backupId}.backup`; - await client.put(Buffer.from(data), remotePath); - - return { - type: 'sftp', - path: remotePath, - size: data.length - }; - } finally { - try { await client.end(); } catch (_) { /* ignore */ } - } - } - - async loadFromSFTP(location) { - const SftpClient = require('ssh2-sftp-client'); - const creds = await this._getCloudCredentials('sftp'); - const client = new SftpClient(); - try { - await client.connect({ - host: creds.host, - port: creds.port, - username: creds.username, - password: creds.password || undefined, - privateKey: creds.privateKey || undefined - }); - const buffer = await client.get(location.path); - return Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer); - } finally { - try { await client.end(); } catch (_) { /* ignore */ } - } - } - - /** - * Test that a destination is reachable + writable + deletable. - * Performs a small write/read/delete probe. - */ - async testDestination(destination) { - const probeId = `test-${Date.now()}`; - const probeData = Buffer.from(`dashcaddy-test-${probeId}`); - const start = Date.now(); - - try { - const location = await this.saveToDestination(probeData, destination, probeId); - - // Read it back - let readBack = null; - try { - readBack = await this.loadFromDestination(location); - } catch (_) { - // Some providers (e.g. local) we already trust the file system; skip - } - - // Delete the probe - try { - await this._deleteFromDestination(location); - } catch (_) { /* ignore */ } - - const elapsed = Date.now() - start; - return { - success: true, - type: destination.type, - elapsedMs: elapsed, - verified: readBack ? readBack.equals(probeData) : null - }; - } catch (error) { - return { - success: false, - type: destination.type, - error: error.message, - elapsedMs: Date.now() - start - }; - } - } - - /** - * Delete a backup from a destination location - */ - async _deleteFromDestination(location) { - if (location.type === 'local') { - if (fs.existsSync(location.path)) fs.unlinkSync(location.path); - return; - } - if (location.type === 'dropbox') { - const { Dropbox } = require('dropbox'); - const creds = await this._getCloudCredentials('dropbox'); - const dbx = new Dropbox({ accessToken: creds.token }); - try { await dbx.filesDeleteV2({ path: location.path }); } catch (_) { /* ignore */ } - return; - } - if (location.type === 'webdav') { - const { createClient } = require('webdav'); - const creds = await this._getCloudCredentials('webdav'); - const client = createClient(creds.url, { username: creds.username, password: creds.password }); - try { await client.deleteFile(location.path); } catch (_) { /* ignore */ } - return; - } - if (location.type === 'sftp') { - const SftpClient = require('ssh2-sftp-client'); - const creds = await this._getCloudCredentials('sftp'); - const client = new SftpClient(); - try { - await client.connect({ - host: creds.host, - port: creds.port, - username: creds.username, - password: creds.password || undefined, - privateKey: creds.privateKey || undefined - }); - try { await client.delete(location.path); } catch (_) { /* ignore */ } - } finally { - try { await client.end(); } catch (_) { /* ignore */ } - } - return; - } - } - /** * Verify backup integrity */ @@ -912,24 +618,9 @@ class BackupManager extends EventEmitter { throw new Error(`Backup not found: ${backupId}`); } - // Load backup data — try each destination location until one succeeds - const location = backup.locations[0]; // Primary location - let data; - try { - data = await this.loadFromDestination(location); - } catch (loadErr) { - // Fall back to other locations if available - let recovered = false; - for (let i = 1; i < backup.locations.length; i++) { - try { - data = await this.loadFromDestination(backup.locations[i]); - recovered = true; - console.log(`[BackupManager] Loaded backup from fallback location ${backup.locations[i].type}`); - break; - } catch (_) { /* ignore */ } - } - if (!recovered) throw loadErr; - } + // Load backup data + const location = backup.locations[0]; // Use first location + let data = fs.readFileSync(location.path); // Decrypt if needed if (backup.encrypted && options.encryptionKey) { @@ -1026,6 +717,63 @@ class BackupManager extends EventEmitter { console.log('[BackupManager] Stats restored'); } + /** + * Enforce storage limit by deleting oldest backups until total is within limit + */ + async enforceStorageLimit(name, maxBytes) { + const maxStr = formatBytes(maxBytes); + console.log("[BackupManager] Enforcing storage limit: " + maxStr + " for \"" + name + "\""); + + const backups = this.history + .filter(b => b.name === name && b.status === 'success') + .sort((a, b) => new Date(a.timestamp) - new Date(b.timestamp)); + + let totalSize = 0; + const locationsMap = {}; + + for (const backup of backups) { + for (const loc of backup.locations || []) { + if (loc.type === 'local' && loc.path) { + totalSize += loc.size || 0; + locationsMap[backup.id] = locationsMap[backup.id] || []; + locationsMap[backup.id].push(loc.path); + } + } + } + + console.log("[BackupManager] Current total size: " + formatBytes(totalSize) + ", limit: " + maxStr); + + if (totalSize <= maxBytes) { + console.log("[BackupManager] Storage limit OK (" + formatBytes(totalSize) + " <= " + maxStr + ")"); + return; + } + + let freed = 0; + for (const backup of backups) { + if (totalSize <= maxBytes) break; + + const paths = locationsMap[backup.id] || []; + for (const path of paths) { + try { + if (fs.existsSync(path)) { + fs.unlinkSync(path); + const sz = backup.size || 0; + totalSize -= sz; + freed += sz; + console.log("[BackupManager] Deleted " + formatBytes(sz) + ": " + path); + } + } catch (error) { + console.error("[BackupManager] Error deleting " + path + ": " + error.message); + } + } + + this.history = this.history.filter(b => b.id !== backup.id); + } + + this.saveHistory(); + console.log("[BackupManager] Storage limit enforced. Freed " + formatBytes(freed) + ", now " + formatBytes(totalSize)); + } + /** * Cleanup old backups based on retention policy */ @@ -1040,18 +788,16 @@ class BackupManager extends EventEmitter { for (const backup of toDelete) { try { - // Delete from all locations (local + cloud) + // Delete from all locations for (const location of backup.locations) { - try { - await this._deleteFromDestination(location); - } catch (delErr) { - console.warn(`[BackupManager] Could not delete ${location.type} location for ${backup.id}:`, delErr.message); + if (location.type === 'local' && fs.existsSync(location.path)) { + fs.unlinkSync(location.path); } } // Remove from history this.history = this.history.filter(b => b.id !== backup.id); - + console.log(`[BackupManager] Deleted old backup: ${backup.id}`); } catch (error) { console.error(`[BackupManager] Error deleting backup ${backup.id}:`, error.message); @@ -1061,71 +807,6 @@ class BackupManager extends EventEmitter { this.saveHistory(); } - /** - * Enforce storage limit by pruning oldest backups if needed. - * Works with any filesystem (ext4, Btrfs, XFS, ZFS, APFS, NTFS). - * @param {number} maxBytes - Maximum storage limit in bytes - * @param {string} backupDir - Backup directory path - * @param {string} pendingBackupId - ID of backup about to be written - * @param {number} pendingSize - Estimated size of pending backup in bytes - */ - async enforceStorageLimit(maxBytes, backupDir, pendingBackupId, pendingSize) { - console.log(`[BackupManager] Checking storage limit: max=${maxBytes} bytes`); - - // Find all backup files for this destination - const files = []; - if (fs.existsSync(backupDir)) { - const entries = fs.readdirSync(backupDir); - for (const entry of entries) { - if (entry.endsWith('.backup')) { - const filePath = path.join(backupDir, entry); - try { - const stats = fs.statSync(filePath); - files.push({ path: filePath, size: stats.size, mtime: stats.mtime }); - } catch (e) { - // Skip files we can't stat - } - } - } - } - - // Sort by modification time (oldest first) - files.sort((a, b) => new Date(a.mtime) - new Date(b.mtime)); - - let totalSize = files.reduce((sum, f) => sum + f.size, 0) + pendingSize; - - // If even ONE backup exceeds the limit, fail immediately - const largestFile = files.reduce((max, f) => f.size > max ? f.size : max, 0); - if (largestFile > maxBytes) { - throw new Error( - `Backup too large (${largestFile} bytes) for configured storage limit (${maxBytes} bytes). ` + - `Increase the storage limit or remove large backups manually.` - ); - } - - // Prune oldest backups until we fit within limit - while (files.length > 0 && totalSize > maxBytes) { - const oldest = files.shift(); - console.log(`[BackupManager] Pruning oldest backup to free space: ${oldest.path} (${oldest.size} bytes)`); - try { - fs.unlinkSync(oldest.path); - // Also remove from history if present - const historyEntry = this.history.find(h => - h.locations && h.locations.some(l => l.path === oldest.path) - ); - if (historyEntry) { - this.history = this.history.filter(h => h.id !== historyEntry.id); - } - totalSize -= oldest.size; - } catch (error) { - console.error(`[BackupManager] Error pruning backup ${oldest.path}:`, error.message); - } - } - - this.saveHistory(); - console.log(`[BackupManager] Storage after pruning: ${totalSize}/${maxBytes} bytes`); - } - /** * Add entry to backup history */