mirror of
https://github.com/getnamingo/registry.git
synced 2025-07-22 18:45:59 +02:00
Caching and other improvements
This commit is contained in:
Pinga
parent
6b9bc5e7cd
commit
bc220010d8
2 changed files with 70 additions and 85 deletions
|
|
@ -28,103 +28,60 @@ $pool = new Swoole\Database\PDOPool(
|
|||
|
||||
Swoole\Runtime::enableCoroutine();
|
||||
|
||||
// Filesystem Cache setup for URLAbuse
|
||||
$cachePath = __DIR__ . '/../cache';
|
||||
// Filesystem Cache setup
|
||||
$cachePath = __DIR__ . '/cache';
|
||||
|
||||
// Check if the 'cache' directory exists
|
||||
if (!is_dir($cachePath)) {
|
||||
// Attempt to create the directory
|
||||
if (!mkdir($cachePath, 0755, true)) {
|
||||
$log->error("Unable to create cache directory at $cachePath. Please check permissions.");
|
||||
throw new Exception("Unable to create cache directory at $cachePath. Please check permissions.");
|
||||
}
|
||||
}
|
||||
|
||||
$adapter = new LocalFilesystemAdapter($cachePath, null, LOCK_EX);
|
||||
$filesystem = new Filesystem($adapter);
|
||||
$cache = new Pool(new ScrapbookFlysystem($filesystem));
|
||||
|
||||
// Cache key and file URL for URLAbuse data
|
||||
$cacheKey = 'urlabuse_blacklist';
|
||||
$fileUrl = 'https://urlabuse.com/public/data/data.txt';
|
||||
$urlAbuseCacheKey = 'urlabuse_blacklist';
|
||||
$urlAbuseUrl = 'https://urlabuse.com/public/data/data.txt';
|
||||
|
||||
// Cache key and URL for URLHaus data
|
||||
$urlhausCacheKey = 'urlhaus_blacklist';
|
||||
$urlhausUrl = 'https://urlhaus.abuse.ch/downloads/json_recent/';
|
||||
|
||||
// Creating first coroutine
|
||||
Coroutine::create(function () use ($pool, $log, $cache, $cacheKey, $fileUrl) {
|
||||
Coroutine::create(function () use ($pool, $log, $cache, $urlAbuseCacheKey, $urlAbuseUrl, $urlhausCacheKey, $urlhausUrl) {
|
||||
try {
|
||||
$pdo = $pool->get();
|
||||
$stmt = $pdo->query('SELECT name, clid FROM domain');
|
||||
|
||||
// Retrieve URLAbuse data with caching
|
||||
$urlAbuseData = getUrlAbuseData($cache, $cacheKey, $fileUrl);
|
||||
// Retrieve cached URLAbuse data
|
||||
$urlAbuseData = getUrlAbuseData($cache, $urlAbuseCacheKey, $urlAbuseUrl);
|
||||
|
||||
// Get URLhaus data
|
||||
$urlhausData = getUrlhausData();
|
||||
// Retrieve cached URLHaus data
|
||||
$urlhausData = getUrlhausData($cache, $urlhausCacheKey, $urlhausUrl);
|
||||
|
||||
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
|
||||
$domain = $row['name'];
|
||||
|
||||
// Check URLAbuse
|
||||
if (checkUrlAbuse($domain, $urlAbuseData)) {
|
||||
$userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?');
|
||||
$userStmt->execute([$row['clid']]);
|
||||
$userData = $userStmt->fetch(PDO::FETCH_ASSOC);
|
||||
|
||||
if ($userData) {
|
||||
// Prepare INSERT statement to add a ticket
|
||||
$insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))');
|
||||
|
||||
// Execute the prepared statement with appropriate values
|
||||
$insertStmt->execute([
|
||||
$userData['user_id'], // user_id
|
||||
"Abuse Report for $domain", // subject
|
||||
"Abuse detected for domain $domain.", // message
|
||||
$domain, // reported_domain
|
||||
"Link to URLAbuse", // evidence
|
||||
"https://urlabuse.com/public/data/data.txt", // relevant_urls
|
||||
date('Y-m-d H:i:s') // date_of_incident
|
||||
]);
|
||||
}
|
||||
processAbuseDetection($pdo, $domain, $row['clid'], 'URLAbuse', 'https://urlabuse.com/public/data/data.txt', $log);
|
||||
}
|
||||
|
||||
// Check URLhaus
|
||||
$urlhausResult = checkUrlhaus($domain, $urlhausData);
|
||||
|
||||
if ($urlhausResult) {
|
||||
$userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?');
|
||||
$userStmt->execute([$row['clid']]);
|
||||
$userData = $userStmt->fetch(PDO::FETCH_ASSOC);
|
||||
|
||||
if ($userData) {
|
||||
// Prepare INSERT statement to add a ticket
|
||||
$insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))');
|
||||
|
||||
// Execute the prepared statement with appropriate values
|
||||
$insertStmt->execute([
|
||||
$userData['user_id'], // user_id
|
||||
"Abuse Report for $domain", // subject
|
||||
"Abuse detected for domain $domain.", // message
|
||||
$domain, // reported_domain
|
||||
"Link to URLhaus", // evidence
|
||||
"https://urlhaus.abuse.ch/downloads/json_recent/", // relevant_urls
|
||||
date('Y-m-d H:i:s') // date_of_incident
|
||||
]);
|
||||
}
|
||||
if (checkUrlhaus($domain, $urlhausData)) {
|
||||
processAbuseDetection($pdo, $domain, $row['clid'], 'URLHaus', 'https://urlhaus.abuse.ch/downloads/json_recent/', $log);
|
||||
}
|
||||
|
||||
// Check Spamhaus
|
||||
if (checkSpamhaus($domain)) {
|
||||
$userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?');
|
||||
$userStmt->execute([$row['clid']]);
|
||||
$userData = $userStmt->fetch(PDO::FETCH_ASSOC);
|
||||
|
||||
if ($userData) {
|
||||
// Prepare INSERT statement to add a ticket
|
||||
$insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))');
|
||||
|
||||
// Execute the prepared statement with appropriate values
|
||||
$insertStmt->execute([
|
||||
$userData['user_id'], // user_id
|
||||
"Abuse Report for $domain", // subject
|
||||
"Abuse detected for domain $domain.", // message
|
||||
$domain, // reported_domain
|
||||
"Link to Spamhaus", // evidence
|
||||
"http://www.spamhaus.org/query/domain/$domain", // relevant_urls
|
||||
date('Y-m-d H:i:s') // date_of_incident
|
||||
]);
|
||||
processAbuseDetection($pdo, $domain, $row['clid'], 'Spamhaus', 'http://www.spamhaus.org/query/domain/'.$domain, $log);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
$log->info('job finished successfully.');
|
||||
} catch (PDOException $e) {
|
||||
$log->error('Database error: ' . $e->getMessage());
|
||||
|
|
|
|||
|
|
@ -68,23 +68,27 @@ function checkSpamhaus($domain) {
|
|||
return checkdnsrr($queryDomain, "A");
|
||||
}
|
||||
|
||||
function getUrlhausData() {
|
||||
$urlhausUrl = 'https://urlhaus.abuse.ch/downloads/json_recent/';
|
||||
$data = [];
|
||||
function getUrlhausData($cache, $cacheKey, $urlhausUrl) {
|
||||
// Check if data is cached
|
||||
$cachedFile = $cache->getItem($cacheKey);
|
||||
|
||||
Coroutine::create(function () use ($urlhausUrl, &$data) {
|
||||
$client = new Client('urlhaus.abuse.ch', 443, true); // SSL
|
||||
$client->set(['timeout' => 5]); // 5 seconds timeout
|
||||
$client->get('/downloads/json_recent/');
|
||||
if (!$cachedFile->isHit()) {
|
||||
// Data is not cached, download it
|
||||
$httpClient = new Client();
|
||||
$response = $httpClient->get($urlhausUrl);
|
||||
$fileContent = $response->getBody()->getContents();
|
||||
|
||||
if ($client->statusCode == 200) {
|
||||
$data = json_decode($client->body, true);
|
||||
// Cache the file content
|
||||
$cachedFile->set($fileContent);
|
||||
$cachedFile->expiresAfter(86400 * 7); // Cache for 7 days
|
||||
$cache->save($cachedFile);
|
||||
|
||||
return processUrlhausData($fileContent);
|
||||
} else {
|
||||
// Retrieve data from cache
|
||||
$fileContent = $cachedFile->get();
|
||||
return processUrlhausData($fileContent);
|
||||
}
|
||||
|
||||
$client->close();
|
||||
});
|
||||
|
||||
return processUrlhausData($data);
|
||||
}
|
||||
|
||||
function processUrlhausData($data) {
|
||||
|
|
@ -104,6 +108,30 @@ function checkUrlhaus($domain, Map $urlhausData) {
|
|||
return $urlhausData->get($domain, false);
|
||||
}
|
||||
|
||||
function processAbuseDetection($pdo, $domain, $clid, $abuseType, $evidenceLink, $log) {
|
||||
$userStmt = $pdo->prepare('SELECT user_id FROM registrar_users WHERE registrar_id = ?');
|
||||
$userStmt->execute([$clid]);
|
||||
$userData = $userStmt->fetch(PDO::FETCH_ASSOC);
|
||||
|
||||
if ($userData) {
|
||||
// Prepare INSERT statement to add a ticket
|
||||
$insertStmt = $pdo->prepare('INSERT INTO support_tickets (id, user_id, category_id, subject, message, status, priority, reported_domain, nature_of_abuse, evidence, relevant_urls, date_of_incident, date_created, last_updated) VALUES (NULL, ?, 8, ?, ?, "Open", "High", ?, "Abuse", ?, ?, ?, CURRENT_TIMESTAMP(3), CURRENT_TIMESTAMP(3))');
|
||||
|
||||
// Execute the prepared statement with appropriate values
|
||||
$insertStmt->execute([
|
||||
$userData['user_id'], // user_id
|
||||
"Abuse Report for $domain ($abuseType)", // subject
|
||||
"Abuse detected for domain $domain via $abuseType.", // message
|
||||
$domain, // reported_domain
|
||||
"Link to $abuseType", // evidence
|
||||
$evidenceLink, // relevant_urls
|
||||
date('Y-m-d H:i:s') // date_of_incident
|
||||
]);
|
||||
|
||||
$log->info("Abuse detected for domain $domain using $abuseType.");
|
||||
}
|
||||
}
|
||||
|
||||
function getDomainPrice($pdo, $domain_name, $tld_id, $date_add = 12, $command = 'create', $registrar_id = null) {
|
||||
// Check if the domain is a premium domain
|
||||
$stmt = $pdo->prepare("
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue