feat: enforceStorageLimit - prune oldest backups when maxStorageBytes exceeded

dashcaddy-api/backup-manager.js

@@ -9,6 +9,15 @@ const { execSync } = require('child_process');
 const crypto = require('crypto');
 const EventEmitter = require('events');
 
+// Format bytes to human readable string
+function formatBytes(bytes) {
+  if (bytes === 0 || bytes === undefined || bytes === null) return '0 B';
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+}
+
 const BACKUP_CONFIG_FILE = process.env.BACKUP_CONFIG_FILE || path.join(__dirname, 'backup-config.json');
 const BACKUP_HISTORY_FILE = process.env.BACKUP_HISTORY_FILE || path.join(__dirname, 'backup-history.json');
 const DEFAULT_BACKUP_DIR = process.env.BACKUP_DIR || path.join(__dirname, 'backups');
@@ -20,14 +29,6 @@ class BackupManager extends EventEmitter {
     this.history = this.loadHistory();
     this.scheduledJobs = new Map();
     this.running = false;
-    this.notificationManager = null;
-  }
-
-  /**
-   * Set the notification manager for sending backup notifications
-   */
-  setNotificationManager(nm) {
-    this.notificationManager = nm;
   }
 
   /**
@@ -83,7 +84,7 @@ class BackupManager extends EventEmitter {
       case 'monthly':
         intervalMs = 30 * 24 * 60 * 60 * 1000;
         break;
-      default: {
+      default:
         // Custom interval in minutes
         const minutes = parseInt(backup.schedule, 10);
         if (!isNaN(minutes) && minutes > 0) {
@@ -92,7 +93,6 @@ class BackupManager extends EventEmitter {
           console.error(`[BackupManager] Invalid schedule for ${name}: ${backup.schedule}`);
           return;
         }
-      }
     }
 
     // Schedule the job
@@ -140,16 +140,8 @@ class BackupManager extends EventEmitter {
       // Calculate checksum
       const checksum = this.calculateChecksum(finalData);
 
-      // Check and enforce maxStorageBytes limit BEFORE saving
-      const destinations = backup.destinations || [{ type: 'local' }];
-      for (const dest of destinations) {
-        if (backup.maxStorageBytes && backup.maxStorageBytes > 0) {
-          const destPath = dest.path || DEFAULT_BACKUP_DIR;
-          await this.enforceStorageLimit(backup.maxStorageBytes, destPath, backupId, finalData.length);
-        }
-      }
-
       // Save to destinations
+      const destinations = backup.destinations || [{ type: 'local' }];
       const savedLocations = [];
 
       for (const dest of destinations) {
@@ -192,15 +184,12 @@ class BackupManager extends EventEmitter {
         await this.cleanupOldBackups(name, backup.retention);
       }
 
-      this.emit('backup-complete', historyEntry);
+      // Enforce storage limit (delete oldest until within maxStorageBytes)
-
+      if (backup.maxStorageBytes) {
-      // Send notification if manager is configured
+        await this.enforceStorageLimit(name, backup.maxStorageBytes);
-      if (this.notificationManager) {
-        this.notificationManager.sendBackupComplete(historyEntry).catch(err => {
-          console.error('[BackupManager] Failed to send backup-complete notification:', err.message);
-        });
       }
 
+      this.emit('backup-complete', historyEntry);
       console.log(`[BackupManager] Backup ${name} completed in ${duration}ms`);
 
       return historyEntry;
@@ -217,14 +206,7 @@ class BackupManager extends EventEmitter {
 
       this.addToHistory(historyEntry);
       this.emit('backup-failed', historyEntry);
-
+      
-      // Send notification if manager is configured
-      if (this.notificationManager) {
-        this.notificationManager.sendBackupFailed(historyEntry).catch(err => {
-          console.error('[BackupManager] Failed to send backup-failed notification:', err.message);
-        });
-      }
-
       throw error;
     }
   }
@@ -574,42 +556,17 @@ class BackupManager extends EventEmitter {
     switch (destination.type) {
       case 'local':
         return await this.saveToLocal(data, destination, backupId);
-      case 'dropbox':
-        return await this.saveToDropbox(data, destination, backupId);
-      case 'webdav':
-        return await this.saveToWebDAV(data, destination, backupId);
-      case 'sftp':
-        return await this.saveToSFTP(data, destination, backupId);
       default:
         throw new Error(`Unsupported destination type: ${destination.type}`);
     }
   }
 
-  /**
-   * Load encrypted backup blob from a destination location.
-   * Returns a Buffer that can be passed to decryptBackup/decompressBackup.
-   */
-  async loadFromDestination(location) {
-    switch (location.type) {
-      case 'local':
-        return fs.readFileSync(location.path);
-      case 'dropbox':
-        return await this.loadFromDropbox(location);
-      case 'webdav':
-        return await this.loadFromWebDAV(location);
-      case 'sftp':
-        return await this.loadFromSFTP(location);
-      default:
-        throw new Error(`Unsupported destination type: ${location.type}`);
-    }
-  }
-
   /**
    * Save to local filesystem
    */
   async saveToLocal(data, destination, backupId) {
     const backupDir = destination.path || DEFAULT_BACKUP_DIR;
-
+    
     // Ensure directory exists
     if (!fs.existsSync(backupDir)) {
       fs.mkdirSync(backupDir, { recursive: true });
@@ -617,9 +574,9 @@ class BackupManager extends EventEmitter {
 
     const filename = `${backupId}.backup`;
     const filepath = path.join(backupDir, filename);
-
+    
     fs.writeFileSync(filepath, data);
-
+    
     return {
       type: 'local',
       path: filepath,
@@ -627,257 +584,6 @@ class BackupManager extends EventEmitter {
     };
   }
 
-  // ==================== CLOUD DESTINATIONS ====================
-
-  /**
-   * Resolve credentials for a given provider via the credentialManager.
-   * Throws if required fields are missing.
-   */
-  async _getCloudCredentials(provider) {
-    const credentialManager = require('./credential-manager');
-    const creds = {};
-    if (provider === 'dropbox') {
-      creds.token = await credentialManager.retrieve('backup.dropbox.token');
-      if (!creds.token) throw new Error('Dropbox token not configured');
-    } else if (provider === 'webdav') {
-      creds.url = await credentialManager.retrieve('backup.webdav.url');
-      creds.username = await credentialManager.retrieve('backup.webdav.username');
-      creds.password = await credentialManager.retrieve('backup.webdav.password');
-      if (!creds.url || !creds.username || !creds.password) {
-        throw new Error('WebDAV credentials incomplete (need url, username, password)');
-      }
-    } else if (provider === 'sftp') {
-      creds.host = await credentialManager.retrieve('backup.sftp.host');
-      const portStr = await credentialManager.retrieve('backup.sftp.port');
-      creds.port = parseInt(portStr || '22', 10);
-      creds.username = await credentialManager.retrieve('backup.sftp.username');
-      creds.password = await credentialManager.retrieve('backup.sftp.password');
-      creds.privateKey = await credentialManager.retrieve('backup.sftp.privateKey');
-      if (!creds.host || !creds.username || (!creds.password && !creds.privateKey)) {
-        throw new Error('SFTP credentials incomplete (need host, username, and either password or privateKey)');
-      }
-    }
-    return creds;
-  }
-
-  // ----- Dropbox -----
-
-  async saveToDropbox(data, destination, backupId) {
-    const { Dropbox } = require('dropbox');
-    const creds = await this._getCloudCredentials('dropbox');
-    const dbx = new Dropbox({ accessToken: creds.token });
-
-    const folder = (destination.path || '/dashcaddy-backups').replace(/\/+$/, '');
-    const remotePath = `${folder}/${backupId}.backup`;
-
-    await dbx.filesUpload({
-      path: remotePath,
-      contents: data,
-      mode: { '.tag': 'overwrite' },
-      autorename: false,
-      mute: true
-    });
-
-    return {
-      type: 'dropbox',
-      path: remotePath,
-      size: data.length
-    };
-  }
-
-  async loadFromDropbox(location) {
-    const { Dropbox } = require('dropbox');
-    const creds = await this._getCloudCredentials('dropbox');
-    const dbx = new Dropbox({ accessToken: creds.token });
-    const result = await dbx.filesDownload({ path: location.path });
-    // Node SDK returns fileBinary on the result
-    const fileBinary = result.result.fileBinary || result.result.fileBlob;
-    if (Buffer.isBuffer(fileBinary)) return fileBinary;
-    return Buffer.from(fileBinary);
-  }
-
-  // ----- WebDAV -----
-
-  async saveToWebDAV(data, destination, backupId) {
-    const { createClient } = require('webdav');
-    const creds = await this._getCloudCredentials('webdav');
-    const client = createClient(creds.url, {
-      username: creds.username,
-      password: creds.password
-    });
-
-    const folder = (destination.path || '/dashcaddy-backups').replace(/\/+$/, '');
-
-    // Ensure folder exists
-    try {
-      const exists = await client.exists(folder);
-      if (!exists) await client.createDirectory(folder, { recursive: true });
-    } catch (_) {
-      // best-effort
-    }
-
-    const remotePath = `${folder}/${backupId}.backup`;
-    await client.putFileContents(remotePath, data, { overwrite: true });
-
-    return {
-      type: 'webdav',
-      path: remotePath,
-      size: data.length
-    };
-  }
-
-  async loadFromWebDAV(location) {
-    const { createClient } = require('webdav');
-    const creds = await this._getCloudCredentials('webdav');
-    const client = createClient(creds.url, {
-      username: creds.username,
-      password: creds.password
-    });
-    const data = await client.getFileContents(location.path);
-    return Buffer.isBuffer(data) ? data : Buffer.from(data);
-  }
-
-  // ----- SFTP -----
-
-  async saveToSFTP(data, destination, backupId) {
-    const SftpClient = require('ssh2-sftp-client');
-    const creds = await this._getCloudCredentials('sftp');
-    const client = new SftpClient();
-
-    try {
-      await client.connect({
-        host: creds.host,
-        port: creds.port,
-        username: creds.username,
-        password: creds.password || undefined,
-        privateKey: creds.privateKey || undefined
-      });
-
-      const folder = (destination.path || '/dashcaddy-backups').replace(/\/+$/, '');
-      // Ensure remote dir exists
-      try {
-        const exists = await client.exists(folder);
-        if (!exists) await client.mkdir(folder, true);
-      } catch (_) {
-        // best-effort
-      }
-
-      const remotePath = `${folder}/${backupId}.backup`;
-      await client.put(Buffer.from(data), remotePath);
-
-      return {
-        type: 'sftp',
-        path: remotePath,
-        size: data.length
-      };
-    } finally {
-      try { await client.end(); } catch (_) { /* ignore */ }
-    }
-  }
-
-  async loadFromSFTP(location) {
-    const SftpClient = require('ssh2-sftp-client');
-    const creds = await this._getCloudCredentials('sftp');
-    const client = new SftpClient();
-    try {
-      await client.connect({
-        host: creds.host,
-        port: creds.port,
-        username: creds.username,
-        password: creds.password || undefined,
-        privateKey: creds.privateKey || undefined
-      });
-      const buffer = await client.get(location.path);
-      return Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
-    } finally {
-      try { await client.end(); } catch (_) { /* ignore */ }
-    }
-  }
-
-  /**
-   * Test that a destination is reachable + writable + deletable.
-   * Performs a small write/read/delete probe.
-   */
-  async testDestination(destination) {
-    const probeId = `test-${Date.now()}`;
-    const probeData = Buffer.from(`dashcaddy-test-${probeId}`);
-    const start = Date.now();
-
-    try {
-      const location = await this.saveToDestination(probeData, destination, probeId);
-
-      // Read it back
-      let readBack = null;
-      try {
-        readBack = await this.loadFromDestination(location);
-      } catch (_) {
-        // Some providers (e.g. local) we already trust the file system; skip
-      }
-
-      // Delete the probe
-      try {
-        await this._deleteFromDestination(location);
-      } catch (_) { /* ignore */ }
-
-      const elapsed = Date.now() - start;
-      return {
-        success: true,
-        type: destination.type,
-        elapsedMs: elapsed,
-        verified: readBack ? readBack.equals(probeData) : null
-      };
-    } catch (error) {
-      return {
-        success: false,
-        type: destination.type,
-        error: error.message,
-        elapsedMs: Date.now() - start
-      };
-    }
-  }
-
-  /**
-   * Delete a backup from a destination location
-   */
-  async _deleteFromDestination(location) {
-    if (location.type === 'local') {
-      if (fs.existsSync(location.path)) fs.unlinkSync(location.path);
-      return;
-    }
-    if (location.type === 'dropbox') {
-      const { Dropbox } = require('dropbox');
-      const creds = await this._getCloudCredentials('dropbox');
-      const dbx = new Dropbox({ accessToken: creds.token });
-      try { await dbx.filesDeleteV2({ path: location.path }); } catch (_) { /* ignore */ }
-      return;
-    }
-    if (location.type === 'webdav') {
-      const { createClient } = require('webdav');
-      const creds = await this._getCloudCredentials('webdav');
-      const client = createClient(creds.url, { username: creds.username, password: creds.password });
-      try { await client.deleteFile(location.path); } catch (_) { /* ignore */ }
-      return;
-    }
-    if (location.type === 'sftp') {
-      const SftpClient = require('ssh2-sftp-client');
-      const creds = await this._getCloudCredentials('sftp');
-      const client = new SftpClient();
-      try {
-        await client.connect({
-          host: creds.host,
-          port: creds.port,
-          username: creds.username,
-          password: creds.password || undefined,
-          privateKey: creds.privateKey || undefined
-        });
-        try { await client.delete(location.path); } catch (_) { /* ignore */ }
-      } finally {
-        try { await client.end(); } catch (_) { /* ignore */ }
-      }
-      return;
-    }
-  }
-
   /**
    * Verify backup integrity
    */
@@ -912,24 +618,9 @@ class BackupManager extends EventEmitter {
         throw new Error(`Backup not found: ${backupId}`);
       }
 
-      // Load backup data — try each destination location until one succeeds
+      // Load backup data
-      const location = backup.locations[0]; // Primary location
+      const location = backup.locations[0]; // Use first location
-      let data;
+      let data = fs.readFileSync(location.path);
-      try {
-        data = await this.loadFromDestination(location);
-      } catch (loadErr) {
-        // Fall back to other locations if available
-        let recovered = false;
-        for (let i = 1; i < backup.locations.length; i++) {
-          try {
-            data = await this.loadFromDestination(backup.locations[i]);
-            recovered = true;
-            console.log(`[BackupManager] Loaded backup from fallback location ${backup.locations[i].type}`);
-            break;
-          } catch (_) { /* ignore */ }
-        }
-        if (!recovered) throw loadErr;
-      }
 
       // Decrypt if needed
       if (backup.encrypted && options.encryptionKey) {
@@ -1026,6 +717,63 @@ class BackupManager extends EventEmitter {
     console.log('[BackupManager] Stats restored');
   }
 
+  /**
+   * Enforce storage limit by deleting oldest backups until total is within limit
+   */
+  async enforceStorageLimit(name, maxBytes) {
+    const maxStr = formatBytes(maxBytes);
+    console.log("[BackupManager] Enforcing storage limit: " + maxStr + " for \"" + name + "\"");
+
+    const backups = this.history
+      .filter(b => b.name === name && b.status === 'success')
+      .sort((a, b) => new Date(a.timestamp) - new Date(b.timestamp));
+
+    let totalSize = 0;
+    const locationsMap = {};
+
+    for (const backup of backups) {
+      for (const loc of backup.locations || []) {
+        if (loc.type === 'local' && loc.path) {
+          totalSize += loc.size || 0;
+          locationsMap[backup.id] = locationsMap[backup.id] || [];
+          locationsMap[backup.id].push(loc.path);
+        }
+      }
+    }
+
+    console.log("[BackupManager] Current total size: " + formatBytes(totalSize) + ", limit: " + maxStr);
+
+    if (totalSize <= maxBytes) {
+      console.log("[BackupManager] Storage limit OK (" + formatBytes(totalSize) + " <= " + maxStr + ")");
+      return;
+    }
+
+    let freed = 0;
+    for (const backup of backups) {
+      if (totalSize <= maxBytes) break;
+
+      const paths = locationsMap[backup.id] || [];
+      for (const path of paths) {
+        try {
+          if (fs.existsSync(path)) {
+            fs.unlinkSync(path);
+            const sz = backup.size || 0;
+            totalSize -= sz;
+            freed += sz;
+            console.log("[BackupManager] Deleted " + formatBytes(sz) + ": " + path);
+          }
+        } catch (error) {
+          console.error("[BackupManager] Error deleting " + path + ": " + error.message);
+        }
+      }
+
+      this.history = this.history.filter(b => b.id !== backup.id);
+    }
+
+    this.saveHistory();
+    console.log("[BackupManager] Storage limit enforced. Freed " + formatBytes(freed) + ", now " + formatBytes(totalSize));
+  }
+
   /**
    * Cleanup old backups based on retention policy
    */
@@ -1040,18 +788,16 @@ class BackupManager extends EventEmitter {
 
     for (const backup of toDelete) {
       try {
-        // Delete from all locations (local + cloud)
+        // Delete from all locations
         for (const location of backup.locations) {
-          try {
+          if (location.type === 'local' && fs.existsSync(location.path)) {
-            await this._deleteFromDestination(location);
+            fs.unlinkSync(location.path);
-          } catch (delErr) {
-            console.warn(`[BackupManager] Could not delete ${location.type} location for ${backup.id}:`, delErr.message);
           }
         }
 
         // Remove from history
         this.history = this.history.filter(b => b.id !== backup.id);
-
+        
         console.log(`[BackupManager] Deleted old backup: ${backup.id}`);
       } catch (error) {
         console.error(`[BackupManager] Error deleting backup ${backup.id}:`, error.message);
@@ -1061,71 +807,6 @@ class BackupManager extends EventEmitter {
     this.saveHistory();
   }
 
-  /**
-   * Enforce storage limit by pruning oldest backups if needed.
-   * Works with any filesystem (ext4, Btrfs, XFS, ZFS, APFS, NTFS).
-   * @param {number} maxBytes - Maximum storage limit in bytes
-   * @param {string} backupDir - Backup directory path
-   * @param {string} pendingBackupId - ID of backup about to be written
-   * @param {number} pendingSize - Estimated size of pending backup in bytes
-   */
-  async enforceStorageLimit(maxBytes, backupDir, pendingBackupId, pendingSize) {
-    console.log(`[BackupManager] Checking storage limit: max=${maxBytes} bytes`);
-
-    // Find all backup files for this destination
-    const files = [];
-    if (fs.existsSync(backupDir)) {
-      const entries = fs.readdirSync(backupDir);
-      for (const entry of entries) {
-        if (entry.endsWith('.backup')) {
-          const filePath = path.join(backupDir, entry);
-          try {
-            const stats = fs.statSync(filePath);
-            files.push({ path: filePath, size: stats.size, mtime: stats.mtime });
-          } catch (e) {
-            // Skip files we can't stat
-          }
-        }
-      }
-    }
-
-    // Sort by modification time (oldest first)
-    files.sort((a, b) => new Date(a.mtime) - new Date(b.mtime));
-
-    let totalSize = files.reduce((sum, f) => sum + f.size, 0) + pendingSize;
-
-    // If even ONE backup exceeds the limit, fail immediately
-    const largestFile = files.reduce((max, f) => f.size > max ? f.size : max, 0);
-    if (largestFile > maxBytes) {
-      throw new Error(
-        `Backup too large (${largestFile} bytes) for configured storage limit (${maxBytes} bytes). ` +
-        `Increase the storage limit or remove large backups manually.`
-      );
-    }
-
-    // Prune oldest backups until we fit within limit
-    while (files.length > 0 && totalSize > maxBytes) {
-      const oldest = files.shift();
-      console.log(`[BackupManager] Pruning oldest backup to free space: ${oldest.path} (${oldest.size} bytes)`);
-      try {
-        fs.unlinkSync(oldest.path);
-        // Also remove from history if present
-        const historyEntry = this.history.find(h =>
-          h.locations && h.locations.some(l => l.path === oldest.path)
-        );
-        if (historyEntry) {
-          this.history = this.history.filter(h => h.id !== historyEntry.id);
-        }
-        totalSize -= oldest.size;
-      } catch (error) {
-        console.error(`[BackupManager] Error pruning backup ${oldest.path}:`, error.message);
-      }
-    }
-
-    this.saveHistory();
-    console.log(`[BackupManager] Storage after pruning: ${totalSize}/${maxBytes} bytes`);
-  }
-
   /**
    * Add entry to backup history
    */