diff --git a/automation/config.php.dist b/automation/config.php.dist index c4d6e76..b7f072b 100644 --- a/automation/config.php.dist +++ b/automation/config.php.dist @@ -63,4 +63,11 @@ return [ 'mailer_sms' => 'twilio', // telesign, plivo, vonage, clickatell are also available 'mailer_sms_account' => 'YOUR_ACCOUNT_SID/USERNAME', 'mailer_sms_auth' => 'YOUR_AUTH_TOKEN/PASSWORD', + + // TMCH Configuration + 'tmch_path' => '/tmp/', + 'tmch_smdrl_user' => 'your_username', + 'tmch_smdrl_pass' => 'your_password', + 'tmch_dnl_user' => 'your_username', + 'tmch_dnl_pass' => 'your_password', ]; \ No newline at end of file diff --git a/automation/crontab.example b/automation/crontab.example index 7a770c8..758b21a 100644 --- a/automation/crontab.example +++ b/automation/crontab.example @@ -27,8 +27,8 @@ # run auto-clean-unused-contact-and-host.php every day 5 0 * * * root /usr/bin/php8.2 /opt/registry/automation/auto-clean-unused-contact-and-host.php -# run tmch.php every day -5 0 * * * root /usr/bin/php8.2 /opt/registry/automation/tmch.php +# run tmch.php twice a day +0 0,12 * * * root /usr/bin/php8.2 /opt/registry/automation/tmch.php # run escrow.php every day 5 0 * * * root /usr/bin/php8.2 /opt/registry/automation/escrow.php diff --git a/automation/tmch.php b/automation/tmch.php index 5a33484..672bfa4 100644 --- a/automation/tmch.php +++ b/automation/tmch.php @@ -1,48 +1,159 @@ info('job started.'); + +try { + $dbh = new PDO($dsn, $c['db_username'], $c['db_password']); + $dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); +} catch (PDOException $e) { + $log->error('DB Connection failed: ' . $e->getMessage()); +} + +$savePath = $c['tmch_path']; $files = [ - 'smdrl' => 'https://ry.marksdb.org/smdrl/smdrl-latest.csv', - 'surl' => 'https://test.ry.marksdb.org/dnl/surl-latest.csv', + 'smdrl' => 'https://test.ry.marksdb.org/smdrl/smdrl-latest.csv', 'dnl' => 'https://test.ry.marksdb.org/dnl/dnl-latest.csv', 'tmch' => 'http://crl.icann.org/tmch.crl' ]; // Configure the username and password for each URL. $credentials = [ - 'smdrl' => ['user' => 'username1', 'pass' => 'password1'], - 'surl' => ['user' => 'username2', 'pass' => 'password2'], - 'dnl' => ['user' => 'username3', 'pass' => 'password3'] - // 'tmch' is not listed here since it doesn't require authentication. + 'smdrl' => ['user' => $c['tmch_smdrl_user'], 'pass' => $c['tmch_smdrl_pass']], + 'dnl' => ['user' => $c['tmch_dnl_user'], 'pass' => $c['tmch_dnl_pass']] ]; -foreach ($files as $key => $url) { - $ch = curl_init($url); +try { + foreach ($files as $key => $url) { + $ch = curl_init($url); - // Check if credentials exist for this URL and set them. - if (isset($credentials[$key])) { - curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); - curl_setopt($ch, CURLOPT_USERPWD, $credentials[$key]['user'] . ":" . $credentials[$key]['pass']); - } + // Check if credentials exist for this URL and set them. + if (isset($credentials[$key])) { + curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($ch, CURLOPT_USERPWD, $credentials[$key]['user'] . ":" . $credentials[$key]['pass']); + } - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_MAXREDIRS, 10); - curl_setopt($ch, CURLOPT_TIMEOUT, 30); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_MAXREDIRS, 10); + curl_setopt($ch, CURLOPT_TIMEOUT, 30); - $data = curl_exec($ch); + $data = curl_exec($ch); - if ($data === false) { - echo "Failed to download $url. Error: " . curl_error($ch) . "\n"; - } else { - $filePath = $savePath . basename($url); - if (file_put_contents($filePath, $data)) { - echo "Successfully downloaded $url to $filePath\n"; + if ($data === false) { + $log->error("Failed to download $url. Error: " . curl_error($ch)); } else { - echo "Failed to save the downloaded file to $filePath\n"; + $timestamp = date('YmdHis'); + $filePath = $savePath . $timestamp . '_' . basename($url); + if (file_put_contents($filePath, $data)) { + $log->info("Successfully downloaded $url to $filePath"); + } else { + $log->info("Failed to save the downloaded file to $filePath"); + } + } + + curl_close($ch); + + // Clear existing table data + clearTableData($dbh, $key); + + // Depending on the file type, call different function to handle and insert data + if ($key === 'smdrl') { + insertSmdrlData($dbh, $filePath, $log); + } elseif ($key === 'dnl') { + insertDnlData($dbh, $filePath); + } elseif ($key === 'tmch') { + insertTmchData($dbh, $filePath); } } + $log->info('job finished successfully.'); +} catch (PDOException $e) { + $log->error('Database error: ' . $e->getMessage()); +} catch (Throwable $e) { + $log->error('Error: ' . $e->getMessage()); +} - curl_close($ch); +function clearTableData($dbh, $key) { + $tableMap = [ + 'smdrl' => 'tmch_revocation', + 'dnl' => 'tmch_claims', + 'tmch' => 'tmch_crl' + ]; + + if (isset($tableMap[$key])) { + $dbh->exec("TRUNCATE TABLE " . $tableMap[$key]); + } +} + +function insertSmdrlData($dbh, $filePath, $log) { + if (($handle = fopen($filePath, "r")) !== FALSE) { + // Skip the first two header rows + for ($i = 0; $i < 2; $i++) { + fgetcsv($handle); + } + + while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { + if (count($data) >= 2) { + // Convert ISO 8601 format to MySQL DATETIME(3) format + try { + $date = new DateTime($data[1]); + $formattedDate = $date->format('Y-m-d H:i:s.v'); // Format for MySQL DATETIME(3) + } catch (Exception $e) { + $log->info('Date conversion error:' . $e->getMessage()); + continue; // Skip this record or handle it as needed + } + + // Assume data format: smd_id,insertion_datetime + $stmt = $dbh->prepare("INSERT INTO tmch_revocation (smd_id, revocation_time) VALUES (?, ?)"); + $stmt->execute([$data[0], $formattedDate]); + } + } + fclose($handle); + } +} + +function insertDnlData($dbh, $filePath) { + if (($handle = fopen($filePath, "r")) !== FALSE) { + // Skip the first two header rows + for ($i = 0; $i < 2; $i++) { + fgetcsv($handle); + } + + while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { + if (count($data) >= 3) { + // Convert ISO 8601 format to MySQL DATETIME(3) format + try { + $date = new DateTime($data[2]); + $formattedDate = $date->format('Y-m-d H:i:s.v'); // Format for MySQL DATETIME(3) + } catch (Exception $e) { + $log->info('Date conversion error:' . $e->getMessage()); + continue; // Skip this record or handle it as needed + } + + // Assuming data format + $stmt = $dbh->prepare("INSERT INTO tmch_claims (domain_label, claim_key, insert_time) VALUES (?, ?, ?)"); + $stmt->execute([$data[0], $data[1], $formattedDate]); + } + } + fclose($handle); + } +} + +function insertTmchData($dbh, $filePath) { + $content = file_get_contents($filePath); + if ($content !== false) { + // Get the current date-time in the format accepted by both MySQL and PostgreSQL + $currentDateTime = (new DateTime())->format('Y-m-d H:i:s'); + + // Insert the whole content into the database + $stmt = $dbh->prepare("INSERT INTO tmch_crl (content, url, update_timestamp) VALUES (?, ?, ?)"); + $stmt->execute([$content, $filePath, $currentDateTime]); + } } \ No newline at end of file diff --git a/database/registry.mariadb.sql b/database/registry.mariadb.sql index c1d7f5c..ce22206 100644 --- a/database/registry.mariadb.sql +++ b/database/registry.mariadb.sql @@ -672,6 +672,28 @@ CREATE TABLE IF NOT EXISTS `registry`.`ticket_responses` ( FOREIGN KEY (ticket_id) REFERENCES support_tickets(id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Ticket Responses'; +CREATE TABLE IF NOT EXISTS `registry`.`tmch_claims` ( + `id` INT(11) UNSIGNED AUTO_INCREMENT PRIMARY KEY, + `domain_label` VARCHAR(100) NOT NULL, + `claim_key` VARCHAR(200) NOT NULL, + `insert_time` datetime(3) NOT NULL, + UNIQUE KEY `tmch_claims_1` (`domain_label`,`claim_key`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='TMCH Claims'; + +CREATE TABLE IF NOT EXISTS `registry`.`tmch_revocation` ( + `id` INT(11) UNSIGNED AUTO_INCREMENT PRIMARY KEY, + `smd_id` VARCHAR(100) NOT NULL, + `revocation_time` datetime(3) NOT NULL, + UNIQUE KEY `tmch_revocation_1` (`smd_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='TMCH Revocation'; + +CREATE TABLE IF NOT EXISTS `registry`.`tmch_crl` ( + `id` INT(11) UNSIGNED AUTO_INCREMENT PRIMARY KEY, + `content` TEXT NOT NULL, + `url` VARCHAR(255) NOT NULL, + `update_timestamp` datetime(3) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='TMCH Crl'; + INSERT INTO `registry`.`domain_tld` VALUES('1','.TEST','/^(?!-)(?!.*--)[A-Z0-9-]{1,63}(?