Improved backup-upload.php

automation/backup-upload.json.dist

@@ -0,0 +1,34 @@
+{
+    "storageType": "sftp",
+    "sftp": {
+        "host": "your_sftp_host",
+        "username": "your_username",
+        "password": "your_password",
+        "privateKey": "/path/to/my/private_key",
+        "passphrase": "passphrase",
+        "port": 22,
+        "useAgent": true,
+        "timeout": 30,
+        "maxTries": 10,
+        "fingerprint": "fingerprint-string",
+        "basePath": "/upload"
+    },
+    "dropbox": {
+        "accessToken": "your_dropbox_access_token"
+    },
+    "googleDrive": {
+        "clientId": "your_client_id",
+        "clientSecret": "your_client_secret",
+        "refreshToken": "your_refresh_token",
+        "folderId": "your_folder_id"
+    },
+    "upload": {
+        "retries": 3,
+        "delay": 5
+    },
+    "directory": "/srv/",
+    "patterns": [
+        "^database-{dateHour}.*\\.sql\\.bz2$",
+        "^files-{dateHour}.*\\.sql\\.bz2$"
+    ]
+}

automation/backup-upload.php

@@ -1,7 +1,6 @@
 <?php
 
 require __DIR__ . '/vendor/autoload.php';
-
 require_once 'helpers.php';
 
 use League\Flysystem\Filesystem;
@@ -11,31 +10,44 @@ use League\Flysystem\UnixVisibility\PortableVisibilityConverter;
 use Spatie\FlysystemDropbox\DropboxAdapter;
 use Hypweb\Flysystem\GoogleDrive\GoogleDriveAdapter;
 use Monolog\Logger;
-use Monolog\Handler\StreamHandler;
-use League\Flysystem\AdapterInterface;
 
+// Setup logger
 $logFilePath = '/var/log/namingo/backup_upload.log';
 $log = setupLogger($logFilePath, 'Backup_Upload');
 $log->info('job started.');
 
-// Storage type: 'sftp', 'dropbox', or 'google_drive'
+// Load configuration from JSON
-$storageType = 'sftp'; // Set this to your preferred storage
+$configPath = __DIR__ . '/backup-upload.json';
+if (!file_exists($configPath)) {
+    $log = setupLogger($logFilePath, 'Backup_Upload');
+    $log->error("Configuration file not found: $configPath");
+    exit();
+}
+
+$config = json_decode(file_get_contents($configPath), true);
+if ($config === null) {
+    $log->error("Invalid JSON format in configuration file: $configPath");
+    exit();
+}
+
+// Get storage type from config
+$storageType = $config['storageType'];
 
 // Setup the filesystem based on the storage type
 switch ($storageType) {
     case 'sftp':
+        $sftpSettings = $config['sftp'];
         $sftpProvider = new SftpConnectionProvider(
-            'your_sftp_host', // host
+            $sftpSettings['host'],
-            'your_username',  // username
+            $sftpSettings['username'],
-            'your_password',  // password
+            $sftpSettings['password'],
-            '/path/to/my/private_key', // private key
+            $sftpSettings['privateKey'],
-            'passphrase', // passphrase
+            $sftpSettings['passphrase'],
-            22, // port
+            $sftpSettings['port'],
-            true, // use agent
+            $sftpSettings['useAgent'],
-            30, // timeout
+            $sftpSettings['timeout'],
-            10, // max tries
+            $sftpSettings['maxTries'],
-            'fingerprint-string' // host fingerprint
+            $sftpSettings['fingerprint']
-            // connectivity checker (optional)
         );
 
         $visibilityConverter = PortableVisibilityConverter::fromArray([
@@ -49,19 +61,21 @@ switch ($storageType) {
             ],
         ]);
 
-        $adapter = new SftpAdapter($sftpProvider, '/upload', $visibilityConverter);
+        $adapter = new SftpAdapter($sftpProvider, $sftpSettings['basePath'], $visibilityConverter);
         break;
     case 'dropbox':
-        $client = new \Spatie\Dropbox\Client('your_dropbox_access_token');
+        $dropboxSettings = $config['dropbox'];
+        $client = new \Spatie\Dropbox\Client($dropboxSettings['accessToken']);
         $adapter = new DropboxAdapter($client);
         break;
     case 'google_drive':
+        $googleDriveSettings = $config['googleDrive'];
         $client = new \Google\Client();
-        $client->setClientId('your_client_id');
+        $client->setClientId($googleDriveSettings['clientId']);
-        $client->setClientSecret('your_client_secret');
+        $client->setClientSecret($googleDriveSettings['clientSecret']);
-        $client->refreshToken('your_refresh_token');
+        $client->refreshToken($googleDriveSettings['refreshToken']);
         $service = new \Google\Service\Drive($client);
-        $adapter = new GoogleDriveAdapter($service, 'your_folder_id');
+        $adapter = new GoogleDriveAdapter($service, $googleDriveSettings['folderId']);
         break;
     default:
         $log->error("Invalid storage type");
@@ -70,21 +84,34 @@ switch ($storageType) {
 
 $filesystem = new Filesystem($adapter);
 
-// Function to upload a file with try-catch for error handling
+// Function to upload a file with retry mechanism
-function uploadFile($filesystem, $localPath, $remotePath, $logger) {
+function uploadFile($filesystem, $localPath, $remotePath, $logger, $retries, $delay) {
-    try {
+    $attempt = 0;
-        if (file_exists($localPath)) {
+    while ($attempt < $retries) {
-            $stream = fopen($localPath, 'r+');
+        try {
-            $filesystem->writeStream($remotePath, $stream);
+            $attempt++;
-            if (is_resource($stream)) {
+            if (file_exists($localPath)) {
-                fclose($stream);
+                $stream = fopen($localPath, 'r+');
+                $filesystem->writeStream($remotePath, $stream);
+                if (is_resource($stream)) {
+                    fclose($stream);
+                }
+                $logger->info("Uploaded: $localPath to $remotePath on attempt $attempt");
+                return true; // Upload succeeded, exit function
+            } else {
+                $logger->warning("File not found: $localPath");
+                return false; // File not found, no need to retry
+            }
+        } catch (Exception $e) {
+            $logger->error("Error uploading $localPath on attempt $attempt: " . $e->getMessage());
+            if ($attempt < $retries) {
+                $logger->info("Retrying in $delay seconds...");
+                sleep($delay); // Wait before retrying
+            } else {
+                $logger->error("All $retries attempts failed for $localPath");
+                return false; // All attempts failed
             }
-            $logger->info("Uploaded: $localPath to $remotePath");
-        } else {
-            $logger->warning("File not found: $localPath");
         }
-    } catch (Exception $e) {
-        $logger->error("Error uploading $localPath: " . $e->getMessage());
     }
 }
 
@@ -92,18 +119,29 @@ function uploadFile($filesystem, $localPath, $remotePath, $logger) {
 $currentDateHour = date('Ymd-H'); // Format: YYYYMMDD-HH
 
 // Directory to check
-$directory = '/srv/';
+$directory = $config['directory'];
 
-// Pattern to match files
+// Load patterns from config
-$pattern = "/^database-$currentDateHour.*\.sql\.bz2$/";
+$patterns = array_map(function ($pattern) use ($currentDateHour) {
-$pattern2 = "/^files-$currentDateHour.*\.sql\.bz2$/";
+    return str_replace('{dateHour}', $currentDateHour, $pattern);
+}, $config['patterns']);
 
 // Scan directory for matching files
 $files = scandir($directory);
+$filesFound = false; // Flag to track if any files are found
+
 foreach ($files as $file) {
-    if (preg_match($pattern, $file) || preg_match($pattern2, $file)) {
+    foreach ($patterns as $pattern) {
-        $localPath = $directory . $file;
+        if (preg_match("/$pattern/", $file)) {
-        $remoteFileName = basename($file);
+            $filesFound = true;
-        uploadFile($filesystem, $localPath, $remoteFileName, $log);
+            $localPath = $directory . $file;
+            $remoteFileName = basename($file);
+            uploadFile($filesystem, $localPath, $remoteFileName, $log, $config['upload']['retries'], $config['upload']['delay']);
+        }
     }
 }
+
+// Log if no files were found
+if (!$filesFound) {
+    $log->info("No matching files found in directory: $directory for patterns: " . implode(', ', $patterns));
+}

docs/configuration.md

@@ -133,9 +133,11 @@ This will initialize and configure the audit trail functionality. This process e
 
 #### 1.4.5. Setup Backup
 
-To ensure the safety and availability of your data in Namingo, it's crucial to set up and verify automated backups. Begin by editing the ```backup.json``` file in the automation directory, where you'll input your database details. Ensure that the details for the database are accurately entered in two specified locations within the ```backup.json``` file.
+To set up backups in Namingo:
 
-Additionally, check that the cronjob for PHPBU is correctly scheduled on your server, as this automates the backup process. You can verify this by reviewing your server's cronjob list. These steps are vital to maintain regular, secure backups of your system, safeguarding against data loss and ensuring business continuity. 
+1. Rename `/opt/registry/automation/backup.json.dist` and `/opt/registry/automation/backup-upload.json.dist` to `backup.json` and `backup-upload.json`, respectively. Edit both files to include the correct database and other required details.
+
+2. Enable the backup functionality in `cron.php` and ensure cronjobs are configured and active on your server. Check the server's cronjob list to verify.
 
 #### 1.4.6. RDE (Registry data escrow) configuration