From bc220010d844b85c03e43108674e1c20254c6b23 Mon Sep 17 00:00:00 2001 From: Pinga <121483313+getpinga@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:00:57 +0300 Subject: [PATCH] Caching and other improvements --- automation/abusemonitor.php | 99 +++++++++++-------------------------- automation/helpers.php | 56 +++++++++++++++------ 2 files changed, 70 insertions(+), 85 deletions(-) diff --git a/automation/abusemonitor.php b/automation/abusemonitor.php index 0206481..a49288a 100644 --- a/automation/abusemonitor.php +++ b/automation/abusemonitor.php @@ -28,102 +28,59 @@ $pool = new Swoole\Database\PDOPool( Swoole\Runtime::enableCoroutine(); -// Filesystem Cache setup for URLAbuse -$cachePath = __DIR__ . '/../cache'; +// Filesystem Cache setup +$cachePath = __DIR__ . '/cache'; + +// Check if the 'cache' directory exists +if (!is_dir($cachePath)) { + // Attempt to create the directory + if (!mkdir($cachePath, 0755, true)) { + $log->error("Unable to create cache directory at $cachePath. Please check permissions."); + throw new Exception("Unable to create cache directory at $cachePath. Please check permissions."); + } +} + $adapter = new LocalFilesystemAdapter($cachePath, null, LOCK_EX); $filesystem = new Filesystem($adapter); $cache = new Pool(new ScrapbookFlysystem($filesystem)); // Cache key and file URL for URLAbuse data -$cacheKey = 'urlabuse_blacklist'; -$fileUrl = 'https://urlabuse.com/public/data/data.txt'; +$urlAbuseCacheKey = 'urlabuse_blacklist'; +$urlAbuseUrl = 'https://urlabuse.com/public/data/data.txt'; + +// Cache key and URL for URLHaus data +$urlhausCacheKey = 'urlhaus_blacklist'; +$urlhausUrl = 'https://urlhaus.abuse.ch/downloads/json_recent/'; // Creating first coroutine -Coroutine::create(function () use ($pool, $log, $cache, $cacheKey, $fileUrl) { +Coroutine::create(function () use ($pool, $log, $cache, $urlAbuseCacheKey, $urlAbuseUrl, $urlhausCacheKey, $urlhausUrl) { try { $pdo = $pool->get(); $stmt = $pdo->query('SELECT name, clid FROM domain'); - // Retrieve URLAbuse data with caching - $urlAbuseData = getUrlAbuseData($cache, $cacheKey, $fileUrl); - - // Get URLhaus data - $urlhausData = getUrlhausData(); + // Retrieve cached URLAbuse data + $urlAbuseData = getUrlAbuseData($cache, $urlAbuseCacheKey, $urlAbuseUrl); + + // Retrieve cached URLHaus data + $urlhausData = getUrlhausData($cache, $urlhausCacheKey, $urlhausUrl); while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) { $domain = $row['name']; // Check URLAbuse if (checkUrlAbuse($domain, $urlAbuseData)) { - $userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?'); - $userStmt->execute([$row['clid']]); - $userData = $userStmt->fetch(PDO::FETCH_ASSOC); - - if ($userData) { - // Prepare INSERT statement to add a ticket - $insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))'); - - // Execute the prepared statement with appropriate values - $insertStmt->execute([ - $userData['user_id'], // user_id - "Abuse Report for $domain", // subject - "Abuse detected for domain $domain.", // message - $domain, // reported_domain - "Link to URLAbuse", // evidence - "https://urlabuse.com/public/data/data.txt", // relevant_urls - date('Y-m-d H:i:s') // date_of_incident - ]); - } + processAbuseDetection($pdo, $domain, $row['clid'], 'URLAbuse', 'https://urlabuse.com/public/data/data.txt', $log); } // Check URLhaus - $urlhausResult = checkUrlhaus($domain, $urlhausData); - - if ($urlhausResult) { - $userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?'); - $userStmt->execute([$row['clid']]); - $userData = $userStmt->fetch(PDO::FETCH_ASSOC); - - if ($userData) { - // Prepare INSERT statement to add a ticket - $insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))'); - - // Execute the prepared statement with appropriate values - $insertStmt->execute([ - $userData['user_id'], // user_id - "Abuse Report for $domain", // subject - "Abuse detected for domain $domain.", // message - $domain, // reported_domain - "Link to URLhaus", // evidence - "https://urlhaus.abuse.ch/downloads/json_recent/", // relevant_urls - date('Y-m-d H:i:s') // date_of_incident - ]); - } + if (checkUrlhaus($domain, $urlhausData)) { + processAbuseDetection($pdo, $domain, $row['clid'], 'URLHaus', 'https://urlhaus.abuse.ch/downloads/json_recent/', $log); } // Check Spamhaus if (checkSpamhaus($domain)) { - $userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?'); - $userStmt->execute([$row['clid']]); - $userData = $userStmt->fetch(PDO::FETCH_ASSOC); - - if ($userData) { - // Prepare INSERT statement to add a ticket - $insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))'); - - // Execute the prepared statement with appropriate values - $insertStmt->execute([ - $userData['user_id'], // user_id - "Abuse Report for $domain", // subject - "Abuse detected for domain $domain.", // message - $domain, // reported_domain - "Link to Spamhaus", // evidence - "http://www.spamhaus.org/query/domain/$domain", // relevant_urls - date('Y-m-d H:i:s') // date_of_incident - ]); - } + processAbuseDetection($pdo, $domain, $row['clid'], 'Spamhaus', 'http://www.spamhaus.org/query/domain/'.$domain, $log); } - } $log->info('job finished successfully.'); } catch (PDOException $e) { diff --git a/automation/helpers.php b/automation/helpers.php index 8ae5b21..71db1ab 100644 --- a/automation/helpers.php +++ b/automation/helpers.php @@ -68,23 +68,27 @@ function checkSpamhaus($domain) { return checkdnsrr($queryDomain, "A"); } -function getUrlhausData() { - $urlhausUrl = 'https://urlhaus.abuse.ch/downloads/json_recent/'; - $data = []; +function getUrlhausData($cache, $cacheKey, $urlhausUrl) { + // Check if data is cached + $cachedFile = $cache->getItem($cacheKey); - Coroutine::create(function () use ($urlhausUrl, &$data) { - $client = new Client('urlhaus.abuse.ch', 443, true); // SSL - $client->set(['timeout' => 5]); // 5 seconds timeout - $client->get('/downloads/json_recent/'); + if (!$cachedFile->isHit()) { + // Data is not cached, download it + $httpClient = new Client(); + $response = $httpClient->get($urlhausUrl); + $fileContent = $response->getBody()->getContents(); - if ($client->statusCode == 200) { - $data = json_decode($client->body, true); - } + // Cache the file content + $cachedFile->set($fileContent); + $cachedFile->expiresAfter(86400 * 7); // Cache for 7 days + $cache->save($cachedFile); - $client->close(); - }); - - return processUrlhausData($data); + return processUrlhausData($fileContent); + } else { + // Retrieve data from cache + $fileContent = $cachedFile->get(); + return processUrlhausData($fileContent); + } } function processUrlhausData($data) { @@ -104,6 +108,30 @@ function checkUrlhaus($domain, Map $urlhausData) { return $urlhausData->get($domain, false); } +function processAbuseDetection($pdo, $domain, $clid, $abuseType, $evidenceLink, $log) { + $userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?'); + $userStmt->execute([$clid]); + $userData = $userStmt->fetch(PDO::FETCH_ASSOC); + + if ($userData) { + // Prepare INSERT statement to add a ticket + $insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))'); + + // Execute the prepared statement with appropriate values + $insertStmt->execute([ + $userData['user_id'], // user_id + "Abuse Report for $domain ($abuseType)", // subject + "Abuse detected for domain $domain via $abuseType.", // message + $domain, // reported_domain + "Link to $abuseType", // evidence + $evidenceLink, // relevant_urls + date('Y-m-d H:i:s') // date_of_incident + ]); + + $log->info("Abuse detected for domain $domain using $abuseType."); + } +} + function getDomainPrice($pdo, $domain_name, $tld_id, $date_add = 12, $command = 'create', $registrar_id = null) { // Check if the domain is a premium domain $stmt = $pdo->prepare("