Caching and other improvements

automation/abusemonitor.php

@@ -28,102 +28,59 @@ $pool = new Swoole\Database\PDOPool(
 
 Swoole\Runtime::enableCoroutine();
 
-// Filesystem Cache setup for URLAbuse
+// Filesystem Cache setup
-$cachePath = __DIR__ . '/../cache';
+$cachePath = __DIR__ . '/cache';
+
+// Check if the 'cache' directory exists
+if (!is_dir($cachePath)) {
+    // Attempt to create the directory
+    if (!mkdir($cachePath, 0755, true)) {
+        $log->error("Unable to create cache directory at $cachePath. Please check permissions.");
+        throw new Exception("Unable to create cache directory at $cachePath. Please check permissions.");
+    }
+}
+
 $adapter = new LocalFilesystemAdapter($cachePath, null, LOCK_EX);
 $filesystem = new Filesystem($adapter);
 $cache = new Pool(new ScrapbookFlysystem($filesystem));
 
 // Cache key and file URL for URLAbuse data
-$cacheKey = 'urlabuse_blacklist';
+$urlAbuseCacheKey = 'urlabuse_blacklist';
-$fileUrl = 'https://urlabuse.com/public/data/data.txt';
+$urlAbuseUrl = 'https://urlabuse.com/public/data/data.txt';
+
+// Cache key and URL for URLHaus data
+$urlhausCacheKey = 'urlhaus_blacklist';
+$urlhausUrl = 'https://urlhaus.abuse.ch/downloads/json_recent/';
 
 // Creating first coroutine
-Coroutine::create(function () use ($pool, $log, $cache, $cacheKey, $fileUrl) {
+Coroutine::create(function () use ($pool, $log, $cache, $urlAbuseCacheKey, $urlAbuseUrl, $urlhausCacheKey, $urlhausUrl) {
     try {
         $pdo = $pool->get();
         $stmt = $pdo->query('SELECT name, clid FROM domain');
 
-        // Retrieve URLAbuse data with caching
+        // Retrieve cached URLAbuse data
-        $urlAbuseData = getUrlAbuseData($cache, $cacheKey, $fileUrl);
+        $urlAbuseData = getUrlAbuseData($cache, $urlAbuseCacheKey, $urlAbuseUrl);
-
+        
-        // Get URLhaus data
+        // Retrieve cached URLHaus data
-        $urlhausData = getUrlhausData();
+        $urlhausData = getUrlhausData($cache, $urlhausCacheKey, $urlhausUrl);
 
         while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
             $domain = $row['name'];
 
             // Check URLAbuse
             if (checkUrlAbuse($domain, $urlAbuseData)) {
-                $userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?');
+                processAbuseDetection($pdo, $domain, $row['clid'], 'URLAbuse', 'https://urlabuse.com/public/data/data.txt', $log);
-                $userStmt->execute([$row['clid']]);
-                $userData = $userStmt->fetch(PDO::FETCH_ASSOC);
-
-                if ($userData) {
-                    // Prepare INSERT statement to add a ticket
-                    $insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))');
-
-                    // Execute the prepared statement with appropriate values
-                    $insertStmt->execute([
-                        $userData['user_id'], // user_id
-                        "Abuse Report for $domain", // subject
-                        "Abuse detected for domain $domain.", // message
-                        $domain, // reported_domain
-                        "Link to URLAbuse", // evidence
-                        "https://urlabuse.com/public/data/data.txt", // relevant_urls
-                        date('Y-m-d H:i:s') // date_of_incident
-                    ]);
-                }
             }
 
             // Check URLhaus
-            $urlhausResult = checkUrlhaus($domain, $urlhausData);
+            if (checkUrlhaus($domain, $urlhausData)) {
-
+                processAbuseDetection($pdo, $domain, $row['clid'], 'URLHaus', 'https://urlhaus.abuse.ch/downloads/json_recent/', $log);
-            if ($urlhausResult) {
-                $userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?');
-                $userStmt->execute([$row['clid']]);
-                $userData = $userStmt->fetch(PDO::FETCH_ASSOC);
-
-                if ($userData) {
-                    // Prepare INSERT statement to add a ticket
-                    $insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))');
-
-                    // Execute the prepared statement with appropriate values
-                    $insertStmt->execute([
-                        $userData['user_id'], // user_id
-                        "Abuse Report for $domain", // subject
-                        "Abuse detected for domain $domain.", // message
-                        $domain, // reported_domain
-                        "Link to URLhaus", // evidence
-                        "https://urlhaus.abuse.ch/downloads/json_recent/", // relevant_urls
-                        date('Y-m-d H:i:s') // date_of_incident
-                    ]);
-                }
             }
 
             // Check Spamhaus
             if (checkSpamhaus($domain)) {
-                $userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?');
+                processAbuseDetection($pdo, $domain, $row['clid'], 'Spamhaus', 'http://www.spamhaus.org/query/domain/'.$domain, $log);
-                $userStmt->execute([$row['clid']]);
-                $userData = $userStmt->fetch(PDO::FETCH_ASSOC);
-
-                if ($userData) {
-                    // Prepare INSERT statement to add a ticket
-                    $insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))');
-
-                    // Execute the prepared statement with appropriate values
-                    $insertStmt->execute([
-                        $userData['user_id'], // user_id
-                        "Abuse Report for $domain", // subject
-                        "Abuse detected for domain $domain.", // message
-                        $domain, // reported_domain
-                        "Link to Spamhaus", // evidence
-                        "http://www.spamhaus.org/query/domain/$domain", // relevant_urls
-                        date('Y-m-d H:i:s') // date_of_incident
-                    ]);
-                }
             }
-            
         }
         $log->info('job finished successfully.');
     } catch (PDOException $e) {

automation/helpers.php

@@ -68,23 +68,27 @@ function checkSpamhaus($domain) {
     return checkdnsrr($queryDomain, "A");
 }
 
-function getUrlhausData() {
+function getUrlhausData($cache, $cacheKey, $urlhausUrl) {
-    $urlhausUrl = 'https://urlhaus.abuse.ch/downloads/json_recent/';
+    // Check if data is cached
-    $data = [];
+    $cachedFile = $cache->getItem($cacheKey);
 
-    Coroutine::create(function () use ($urlhausUrl, &$data) {
+    if (!$cachedFile->isHit()) {
-        $client = new Client('urlhaus.abuse.ch', 443, true); // SSL
+        // Data is not cached, download it
-        $client->set(['timeout' => 5]); // 5 seconds timeout
+        $httpClient = new Client();
-        $client->get('/downloads/json_recent/');
+        $response = $httpClient->get($urlhausUrl);
+        $fileContent = $response->getBody()->getContents();
 
-        if ($client->statusCode == 200) {
+        // Cache the file content
-            $data = json_decode($client->body, true);
+        $cachedFile->set($fileContent);
-        }
+        $cachedFile->expiresAfter(86400 * 7); // Cache for 7 days
+        $cache->save($cachedFile);
 
-        $client->close();
+        return processUrlhausData($fileContent);
-    });
+    } else {
-
+        // Retrieve data from cache
-    return processUrlhausData($data);
+        $fileContent = $cachedFile->get();
+        return processUrlhausData($fileContent);
+    }
 }
 
 function processUrlhausData($data) {
@@ -104,6 +108,30 @@ function checkUrlhaus($domain, Map $urlhausData) {
     return $urlhausData->get($domain, false);
 }
 
+function processAbuseDetection($pdo, $domain, $clid, $abuseType, $evidenceLink, $log) {
+    $userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?');
+    $userStmt->execute([$clid]);
+    $userData = $userStmt->fetch(PDO::FETCH_ASSOC);
+
+    if ($userData) {
+        // Prepare INSERT statement to add a ticket
+        $insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))');
+
+        // Execute the prepared statement with appropriate values
+        $insertStmt->execute([
+            $userData['user_id'], // user_id
+            "Abuse Report for $domain ($abuseType)", // subject
+            "Abuse detected for domain $domain via $abuseType.", // message
+            $domain, // reported_domain
+            "Link to $abuseType", // evidence
+            $evidenceLink, // relevant_urls
+            date('Y-m-d H:i:s') // date_of_incident
+        ]);
+
+        $log->info("Abuse detected for domain $domain using $abuseType.");
+    }
+}
+
 function getDomainPrice($pdo, $domain_name, $tld_id, $date_add = 12, $command = 'create', $registrar_id = null) {
     // Check if the domain is a premium domain
     $stmt = $pdo->prepare("