diff --git a/Gemfile b/Gemfile index 6f04e08d8..5bae52e19 100644 --- a/Gemfile +++ b/Gemfile @@ -113,3 +113,6 @@ gem 'net-ftp' # https://stackoverflow.com/questions/79360526/uninitialized-constant-activesupportloggerthreadsafelevellogger-nameerror gem 'concurrent-ruby', '1.3.4' + +# gives you access to stdin, stdout, and stderr when running other programs +gem 'open3' \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index aaa2d9a94..9c7de730f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -394,6 +394,7 @@ GEM omniauth-rails_csrf_protection (0.1.2) actionpack (>= 4.2) omniauth (>= 1.3.1) + open3 (0.2.1) openid_connect (1.4.2) activemodel attr_required (>= 1.0.0) @@ -636,6 +637,7 @@ DEPENDENCIES nokogiri (~> 1.16.0) omniauth-rails_csrf_protection omniauth-tara! + open3 openssl paper_trail (~> 14.0) pdfkit diff --git a/app/controllers/admin/tools/wordcloud_controller.rb b/app/controllers/admin/tools/wordcloud_controller.rb new file mode 100644 index 000000000..7a9bde243 --- /dev/null +++ b/app/controllers/admin/tools/wordcloud_controller.rb @@ -0,0 +1,174 @@ +# frozen_string_literal: true + +module Admin + module Tools + # Controller for the admin wordcloud generator tool that creates visual representations + # of the most common words used in domain names + class WordcloudController < BaseController # rubocop:disable Metrics/ClassLength + WORDCLOUD_DIR = Rails.root.join('public', 'wordcloud') + WORDCLOUD_IMAGE_PATH = WORDCLOUD_DIR.join('wordcloud.png') + WORDCLOUD_CONFIG_PATH = WORDCLOUD_DIR.join('config.json') + TOP_WORDS_PATH = WORDCLOUD_DIR.join('top_words.txt') + + before_action :authorize_admin + before_action :clear_cache, only: :create + before_action :ensure_wordcloud_dir, only: :create + + def index + # Load configuration + @config = load_wordcloud_config + + # Setup wordcloud data if image exists + if File.exist?(WORDCLOUD_IMAGE_PATH) + setup_wordcloud_data + else + @wordcloud_url = nil + end + end + + def create + # Validate domains file + if params[:domains_file].present? + domains_file_path = process_uploaded_file(params[:domains_file]) + return redirect_to admin_tools_wordcloud_path if domains_file_path.nil? + else + flash[:alert] = I18n.t('admin.tools.wordcloud_no_file') + return redirect_to admin_tools_wordcloud_path + end + + # Collect and save configuration + config = build_config_from_params + File.write(WORDCLOUD_CONFIG_PATH, config.to_json) + + # Start the background job + GenerateWordCloudJob.perform_later(domains_file_path.to_s, current_admin_user.id, config) + redirect_to progress_admin_tools_wordcloud_path + + rescue StandardError => e + logger.error "Error starting wordcloud generation: #{e.message}" + flash[:alert] = "#{I18n.t('admin.tools.wordcloud_error')}: #{e.message}" + redirect_to admin_tools_wordcloud_path + end + + # GET /admin/tools/wordcloud/progress + def progress + @progress_key = "wordcloud_progress:#{current_admin_user.id}" + @progress_data = Rails.cache.fetch(@progress_key) || { status: 'not_started', progress: 0 } + end + + # GET /admin/tools/wordcloud/status + def status + progress_key = "wordcloud_progress:#{current_admin_user.id}" + progress_data = Rails.cache.fetch(progress_key) || { status: 'not_started', progress: 0 } + + render json: progress_data + end + + private + + def ensure_wordcloud_dir + FileUtils.mkdir_p(WORDCLOUD_DIR) unless Dir.exist?(WORDCLOUD_DIR) + end + + def process_uploaded_file(uploaded_file) + # Create a persistent copy of the uploaded file + persistent_file_path = Rails.root.join('tmp', "domains_#{Time.now.to_i}.csv") + + # Copy the file content to a persistent location + FileUtils.cp(uploaded_file.tempfile.path, persistent_file_path) + + # Validate file has content + if File.size(persistent_file_path).zero? + File.delete(persistent_file_path) + flash[:alert] = I18n.t('admin.tools.wordcloud_empty_file') + return nil + end + + persistent_file_path + end + + def build_config_from_params + # Base configuration + config = { + width: params[:width].presence || 800, + height: params[:height].presence || 800, + max_words: params[:max_words].presence || 500, + background_color: params[:background_color].presence || 'white', + min_word_length: params[:min_word_length].presence || 2, + include_numbers: params[:include_numbers] == '1', + batch_size: params[:batch_size].presence || 500 + } + + # Process additional stopwords + if params[:additional_stopwords].present? + stopwords = params[:additional_stopwords].downcase.split(/[\s,]+/).reject(&:empty?) + config[:additional_stopwords] = stopwords if stopwords.any? + end + + # Process special terms + if params[:special_terms].present? + special_terms = params[:special_terms].split(/[\s,]+/).reject(&:empty?) + config[:special_terms] = special_terms if special_terms.any? + end + + config + end + + def load_wordcloud_config + if File.exist?(WORDCLOUD_CONFIG_PATH) + begin + JSON.parse(File.read(WORDCLOUD_CONFIG_PATH)) + rescue JSON::ParserError + default_wordcloud_config + end + else + default_wordcloud_config + end + end + + def setup_wordcloud_data + # Add timestamp to prevent caching + @wordcloud_url = "/wordcloud/wordcloud.png?t=#{File.mtime(WORDCLOUD_IMAGE_PATH).to_i}" + + # Get the file's modification time and convert to application timezone + @wordcloud_generated_at = File.mtime(WORDCLOUD_IMAGE_PATH).in_time_zone(Time.zone) + + # Load top words + load_top_words + end + + def load_top_words + return unless File.exist?(TOP_WORDS_PATH) + + @top_words = [] + File.readlines(TOP_WORDS_PATH).each do |line| + if line =~ /^\d+\.\s+(\w+):\s+(\d+)$/ + @top_words << [$1, $2.to_i] + end + end + end + + def default_wordcloud_config + { + 'width' => 800, + 'height' => 800, + 'max_words' => 500, + 'background_color' => 'white', + 'additional_stopwords' => [], + 'include_numbers' => true, + 'min_word_length' => 2, + 'special_terms' => ['e-', 'i-', '2-', '3-', '4-', '.com', 'tr.ee', 'ai', 'web'], + 'batch_size' => 500 + } + end + + def authorize_admin + authorize! :access, :tools + end + + def clear_cache + Rails.cache.delete("wordcloud_progress:#{current_admin_user.id}") + end + end + end +end diff --git a/app/controllers/admin/tools_controller.rb b/app/controllers/admin/tools_controller.rb new file mode 100644 index 000000000..afc2891ac --- /dev/null +++ b/app/controllers/admin/tools_controller.rb @@ -0,0 +1,14 @@ +module Admin + class ToolsController < BaseController + before_action :authorize_admin + + # GET /admin/tools + def index; end + + private + + def authorize_admin + authorize! :access, :tools + end + end +end diff --git a/app/jobs/generate_word_cloud_job.rb b/app/jobs/generate_word_cloud_job.rb new file mode 100644 index 000000000..159a1eaa3 --- /dev/null +++ b/app/jobs/generate_word_cloud_job.rb @@ -0,0 +1,86 @@ +# Use Open3 to capture output in real-time +require 'open3' + +class GenerateWordCloudJob < ApplicationJob + def perform(domains_file_path, user_id, config = {}) + # Set up progress tracking + progress_key = "wordcloud_progress:#{user_id}" + Rails.cache.write(progress_key, { status: 'processing', progress: 0 }) + + begin + # Ensure the wordcloud directory exists + wordcloud_dir = Rails.root.join('public', 'wordcloud') + FileUtils.mkdir_p(wordcloud_dir) unless Dir.exist?(wordcloud_dir) + + # Setup Python environment + python_executable = ENV.fetch('PYTHON_EXECUTABLE', 'python3') + script_path = Rails.root.join('lib', 'wordcloud', 'generate_wordcloud.py') + + # Create a config file for the Python script + config_file_path = Rails.root.join(wordcloud_dir, "wordcloud_config_#{Time.now.to_i}.json") + File.write(config_file_path, config.to_json) + + # Set environment variables to ensure proper encoding + env = { 'PYTHONIOENCODING' => 'utf-8', 'PYTHONUNBUFFERED' => '1' } + + # Debug information + # Rails.logger.info("Python executable: #{python_executable}") + # Rails.logger.info("Script path: #{script_path}") + # Rails.logger.info("Domains file: #{domains_file_path}") + # Rails.logger.info("Output directory: #{wordcloud_dir}") + + # Check if files exist + # Rails.logger.info("Script exists: #{File.exist?(script_path)}") + # Rails.logger.info("Domains file exists: #{File.exist?(domains_file_path)}") + + # Make script executable + FileUtils.chmod('+x', script_path) unless File.executable?(script_path) + + Open3.popen2e(env, python_executable, script_path.to_s, domains_file_path, wordcloud_dir.to_s, config_file_path.to_s) do |stdin, stdout_err, wait_thr| + # Close stdin since we don't need it + stdin.close + + # Process output line by line + while line = stdout_err.gets + # Parse progress from Python script output + if line =~ /Processing batch (\d+)\/(\d+)/ + current = $1.to_i + total = $2.to_i + progress = ((current.to_f / total) * 80).round + Rails.cache.write(progress_key, { status: 'processing', progress: progress }) + elsif line =~ /Total estimated cost/ + # Update when word extraction is complete + Rails.cache.write(progress_key, { status: 'processing', progress: 80 }) + elsif line =~ /Generating word cloud/ + # Update when word cloud generation starts + Rails.cache.write(progress_key, { status: 'processing', progress: 90 }) + end + + # Log output for debugging + Rails.logger.info("WordCloud: #{line.strip}") + end + + # Check if the process was successful + exit_status = wait_thr.value + if exit_status.success? + Rails.cache.write(progress_key, { status: 'completed', progress: 100 }) + else + Rails.cache.write(progress_key, { + status: 'failed', + progress: 0, + error: "Process failed with status #{exit_status.exitstatus}" + }) + end + end + + rescue => e + Rails.logger.error("Error in WordCloud job: #{e.message}") + Rails.logger.error(e.backtrace.join("\n")) + Rails.cache.write(progress_key, { status: 'failed', progress: 0, error: e.message }) + ensure + # Clean up the config file + File.delete(config_file_path) if File.exist?(config_file_path) + end + end +end + diff --git a/app/models/ability.rb b/app/models/ability.rb index ba6971dcf..eeec470f2 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -121,6 +121,7 @@ class Ability can :destroy, :pending can :create, :zonefile can :access, :settings_menu + can :access, :tools can :manage, :mass_actions can :manage, BouncedMailAddress end diff --git a/app/views/admin/base/_menu.haml b/app/views/admin/base/_menu.haml index f1e855742..73b4daf77 100644 --- a/app/views/admin/base/_menu.haml +++ b/app/views/admin/base/_menu.haml @@ -6,6 +6,8 @@ %li= link_to t(:contacts), admin_contacts_path - if can? :show, Registrar %li= link_to t(:registrars), admin_registrars_path + - if can?(:access, :tools) + %li= link_to t(:tools), admin_tools_path - if can?(:access, :settings_menu) %li.dropdown %a.dropdown-toggle{"data-toggle" => "dropdown", href: "#"} diff --git a/app/views/admin/tools/index.html.erb b/app/views/admin/tools/index.html.erb new file mode 100644 index 000000000..1b111132a --- /dev/null +++ b/app/views/admin/tools/index.html.erb @@ -0,0 +1,23 @@ +<%= render "shared/title", name: t('admin.tools.title') %> + +
+
+
+
+

<%= t('admin.tools.available_tools') %>

+
+
+
+
+
+

<%= t('admin.tools.wordcloud_generator') %>

+

<%= t('admin.tools.wordcloud_generator_description') %>

+ <%= link_to t('admin.tools.generate_wordcloud'), admin_tools_wordcloud_path, class: 'btn btn-primary' %> +
+
+ +
+
+
+
+
diff --git a/app/views/admin/tools/wordcloud/index.html.erb b/app/views/admin/tools/wordcloud/index.html.erb new file mode 100644 index 000000000..37b65db8e --- /dev/null +++ b/app/views/admin/tools/wordcloud/index.html.erb @@ -0,0 +1,198 @@ +<% content_for :actions do %> + <%= link_to t('back'), admin_tools_path, class: 'btn btn-default' %> +<% end %> +<%= render "shared/title", name: t('admin.tools.wordcloud_title') %> + + + +
+
+
+
+
+
+

<%= t('admin.tools.wordcloud_title') %>

+
+
+ <% if @wordcloud_url %> +
+ <%= link_to @wordcloud_url, target: "_blank", title: t('admin.tools.view_full_size') do %> + <%= image_tag @wordcloud_url, class: 'img-responsive', alt: t('admin.tools.wordcloud_title') %> +
+ <%= t('admin.tools.click_to_enlarge') %> +
+ <% end %> + + <% if @wordcloud_generated_at %> +
+ <%= t('admin.tools.generated_at', time: l(@wordcloud_generated_at, format: :long)) %> +
+ <% end %> +
+ <% end %> +
+

<%= t('admin.tools.wordcloud_instructions') %>

+
+ <%= form_tag admin_tools_wordcloud_path, method: :post, multipart: true do %> +
+
+
+
+

<%= t('admin.tools.custom_file_description') %>

+ <%= file_field_tag :domains_file, accept: '.csv', class: 'form-control' %> +
+
+
+
+ +
+
+

+ + <%= t('admin.tools.advanced_options') %> + +

+
+
+
+
+
+
+ <%= label_tag :width, t('admin.tools.wordcloud_width') %> + <%= number_field_tag :width, @config['width'], min: 400, max: 2000, step: 100, class: 'form-control' %> +
+
+
+
+ <%= label_tag :height, t('admin.tools.wordcloud_height') %> + <%= number_field_tag :height, @config['height'], min: 400, max: 2000, step: 100, class: 'form-control' %> +
+
+
+
+ <%= label_tag :max_words, t('admin.tools.wordcloud_max_words') %> + <%= number_field_tag :max_words, @config['max_words'], min: 100, max: 1000, step: 50, class: 'form-control' %> +
+
+
+ +
+
+
+ <%= label_tag :batch_size, t('admin.tools.batch_size') %> + <%= number_field_tag :batch_size, @config['batch_size'], min: 100, max: 1000, step: 50, class: 'form-control' %> + <%= t('admin.tools.batch_size_help') %> +
+
+
+
+ <%= label_tag :background_color, t('admin.tools.wordcloud_background') %> + <%= select_tag :background_color, + options_for_select([ + ['White', 'white'], + ['Black', 'black'], + ['Transparent', 'transparent'], + ['Light Gray', '#f0f0f0'] + ], @config['background_color']), + class: 'form-control' %> +
+
+
+ +
+
+
+ <%= label_tag :min_word_length, t('admin.tools.min_word_length') %> + <%= number_field_tag :min_word_length, @config['min_word_length'], min: 1, max: 5, class: 'form-control' %> +
+
+
+
+
+ +
+
+
+
+ +
+ <%= label_tag :special_terms, t('admin.tools.special_terms') %> + <%= text_field_tag :special_terms, @config['special_terms'].is_a?(Array) ? @config['special_terms'].join(', ') : '', + class: 'form-control', + placeholder: t('admin.tools.special_terms_placeholder') %> + <%= t('admin.tools.special_terms_help') %> +
+ +
+ <%= label_tag :additional_stopwords, t('admin.tools.additional_stopwords') %> + <%= text_area_tag :additional_stopwords, @config['additional_stopwords'].is_a?(Array) ? @config['additional_stopwords'].join(', ') : '', + rows: 3, + placeholder: t('admin.tools.stopwords_placeholder'), + class: 'form-control' %> + <%= t('admin.tools.stopwords_help') %> +
+
+
+
+ + <%= submit_tag t('admin.tools.generate_wordcloud'), class: 'btn btn-primary btn-lg mt-3' %> + <% end %> +
+
+
+ +
+
+
+

<%= t('admin.tools.top_words') %>

+
+
+ <% if @top_words && @top_words.any? %> +
    + <% @top_words.each do |word, count| %> +
  1. <%= word %>: <%= count %>
    2. + <% end %> +
+ <% else %> +

<%= t('admin.tools.top_words_empty') %>

+ <% end %> +
+
+
+
+
+
\ No newline at end of file diff --git a/app/views/admin/tools/wordcloud/progress.html.erb b/app/views/admin/tools/wordcloud/progress.html.erb new file mode 100644 index 000000000..2849b1794 --- /dev/null +++ b/app/views/admin/tools/wordcloud/progress.html.erb @@ -0,0 +1,93 @@ +
+
+
+

WordCloud Generation Progress

+ +
+
+
+
+ <%= @progress_data[:progress] %>% +
+
+ +
+ <% case @progress_data[:status] %> + <% when 'not_started' %> +
Waiting to start processing...
+ <% when 'processing' %> +
Processing in progress...
+ <% when 'completed' %> +
+ WordCloud generation completed! + <%= link_to "View WordCloud", admin_tools_wordcloud_path, class: "btn btn-primary" %> +
+ <% when 'failed' %> +
+ Error: <%= @progress_data[:error] || "Unknown error occurred" %> +
+ <% end %> +
+ +
+ <%= link_to "Back to Tools", admin_tools_path, class: "btn btn-secondary" %> +
+
+
+
+
+
+ + \ No newline at end of file diff --git a/config/environments/production.rb b/config/environments/production.rb index 2825185d0..07f89003c 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -54,6 +54,10 @@ Rails.application.configure do # Use a different cache store in production. # config.cache_store = :mem_cache_store + config.cache_store = :redis_cache_store, { + url: "#{ENV.fetch('REDIS_URL', 'redis://localhost:6379')}/1', + expires_in: 300.seconds + } # Use a real queuing backend for Active Job (and separate queues per environment) config.active_job.queue_adapter = :sidekiq diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index b2d067a2f..33392bcb8 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -2,7 +2,7 @@ require 'sidekiq/web' # Require at the top of the initializer Sidekiq.configure_server do |config| config.logger.level = Logger::INFO - + # Custom job logging format Sidekiq.logger.formatter = proc do |severity, datetime, progname, msg| thread_id = Thread.current.object_id.to_s(36) diff --git a/config/locales/admin/tools.en.yml b/config/locales/admin/tools.en.yml new file mode 100644 index 000000000..55452f632 --- /dev/null +++ b/config/locales/admin/tools.en.yml @@ -0,0 +1,40 @@ +en: + admin: + tools: + title: "Administrative Tools" + available_tools: "Available Tools" + wordcloud_generator: "Domain Name Wordcloud Generator" + wordcloud_generator_description: "Generate a visual wordcloud from domain names in the registry" + generate_wordcloud: "Generate Wordcloud" + regenerate_wordcloud: "Regenerate Wordcloud" + regenerate_description: "Click below to regenerate the wordcloud with the latest domain data" + wordcloud_title: "Domain Name Wordcloud" + wordcloud_success: "Wordcloud generated successfully" + wordcloud_error: "Error generating wordcloud" + wordcloud_processing: "Processing domain names. This may take a few minutes..." + wordcloud_instructions: "Generate a visual representation of the most common words in domain names. Click the button below to create the wordcloud." + top_words: "Top Words" + top_words_empty: "Generate a wordcloud to see the most frequent words." + click_to_enlarge: "Click to enlarge" + view_full_size: "View full size wordcloud image" + use_custom_domains: "Use custom domain list" + custom_file_description: "Upload a CSV file with one domain name per line" + file_upload_error: "Error processing uploaded file" + file_optional: "If no file is uploaded, all active domains in the registry will be used" + generated_at: "Generated at %{time}" + wordcloud_no_file: "No domain names found" + wordcloud_width: "Width" + wordcloud_height: "Height" + wordcloud_max_words: "Max Words" + wordcloud_background: "Background" + additional_stopwords: "Stopwords" + stopwords_placeholder: "Enter additional stopwords, one per line" + stopwords_help: "Stopwords are words that will not be included in the wordcloud" + advanced_options: "Advanced Options" + min_word_length: "Min Word Length" + include_numbers: "Include Numbers" + special_terms: "Special Terms" + special_terms_placeholder: "e.g., e-, i-, .com, ai, web" + special_terms_help: "These terms will be preserved in the word cloud even if they would normally be filtered out" + batch_size: "Batch Size" + batch_size_help: "Number of domains to process in each API call." diff --git a/config/routes.rb b/config/routes.rb index 50e72511b..aca21c4f2 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -250,6 +250,17 @@ Rails.application.routes.draw do end # post 'admi/upload_spreadsheet', to: 'customers#upload_spreadsheet', as: :customers_upload_spreadsheet + resources :tools, only: %i[index] + + namespace :tools do + resource :wordcloud, controller: 'wordcloud', only: %i[create] do + collection do + get '', to: 'wordcloud#index' + get 'progress', to: 'wordcloud#progress', as: :progress + get 'status', to: 'wordcloud#status', as: :status + end + end + end resources :bank_statements do resources :bank_transactions diff --git a/lib/wordcloud/fonts/Pacifico-Regular.ttf b/lib/wordcloud/fonts/Pacifico-Regular.ttf new file mode 100644 index 000000000..e7def95d3 Binary files /dev/null and b/lib/wordcloud/fonts/Pacifico-Regular.ttf differ diff --git a/lib/wordcloud/generate_wordcloud.py b/lib/wordcloud/generate_wordcloud.py new file mode 100644 index 000000000..175bc6100 --- /dev/null +++ b/lib/wordcloud/generate_wordcloud.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 + +import os +import re +import sys +import json +import random +import numpy as np +from PIL import Image +from os import path +from wordcloud import WordCloud, STOPWORDS +import openai +import matplotlib.pyplot as plt +from dotenv import load_dotenv +load_dotenv() + +d = path.dirname(__file__) if "__file__" in locals() else os.getcwd() + +output_dir = sys.argv[2] if len(sys.argv) > 2 else d + +# Load configuration if provided +config = {} +if len(sys.argv) > 3 and sys.argv[3]: + config_file = sys.argv[3] + if path.exists(config_file): + with open(config_file, 'r') as f: + config = json.load(f) + print(f"Loaded configuration: {config}") + +# Check if domains file path is provided and exists +if len(sys.argv) > 1 and sys.argv[1]: + domains_file = sys.argv[1] + if not path.exists(domains_file): + print(f"Error: Provided domains file {domains_file} not found") + sys.exit(1) +else: + print(f"Error: Domains file not found") + sys.exit(1) + +# Read domain names from the file +with open(domains_file, 'r', encoding='utf-8') as f: + domain_names = [line.strip().lower() for line in f if line.strip()] + +if not domain_names: + print("Error: No domain names found in the provided file") + sys.exit(1) + +# Get special terms from config or use defaults +SPECIAL_TERMS = config.get('special_terms', ['e-', 'i-', '2-', '3-', '4-', '.com', 'tr.ee', 'ai', 'web']) +print(f"Using special terms: {SPECIAL_TERMS}") + +# Get batch size from config or use default +BATCH_SIZE = int(config.get('batch_size', 500)) +print(f"Using batch size: {BATCH_SIZE}") + +# Function to extract words using OpenAI API +def extract_words_with_openai(domain_names, special_terms, batch_size=BATCH_SIZE): + # Get API key from environment variable + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise ValueError("OpenAI API key not found. Set the OPENAI_API_KEY environment variable.") + + # Get model and temperature from environment variables + model = os.environ.get("OPENAI_MODEL", "gpt-4.1-2025-04-14") + temperature = float(os.environ.get("OPENAI_TEMPERATURE", "0.3")) + max_tokens = int(os.environ.get("OPENAI_MAX_TOKENS", "2000")) + + # Process domains in batches + all_words = [] + total_prompt_tokens = 0 + total_completion_tokens = 0 + total_cost = 0 + + # Calculate number of batches + num_batches = (len(domain_names) + batch_size - 1) // batch_size + + for i in range(0, len(domain_names), batch_size): + batch = domain_names[i:i+batch_size] + print(f"Processing batch {i//batch_size + 1}/{num_batches} ({len(batch)} domains)...") + sys.stdout.flush() + + # Prepare the prompt with domain names and special terms + domains_text = "\n".join(batch) + special_terms_text = ", ".join([f"`{term}`" for term in special_terms]) + + prompt = f"""You are a bilingual Estonian-English linguist and word segmentation expert. I will give you a list of .ee domain names. + +Your task is to extract a clean list of words for word cloud generation. + +Follow these rules strictly: +1. Before doing anything else, always extract and separate these predefined special terms if they appear as prefixes or parts of the domain name: {special_terms_text}. Keep symbols and numbers as they are. For example, if the domain name is `e-robot.ee`, the output should be `e- robot`. Remove extensions from the special terms. +2. If a word contains a number (e.g., `auto24`), separate the number and the word: `auto`, `24`. +3. If the domain name is a compound of 2+ Estonian or English words (e.g., `virtuaalabiline` or `doorkeeper`), intelligently split them into individual meaningful components. Prioritize Estonian words over English words. +4. Keep all resulting words in lowercase and remove the `.ee` extension from all the words +5. Try to find the most common words and phrases in the domain names. +6. Return ONLY a space-separated list of words and numberswith no explanations, no formatting, no introductions, and no additional text. + +Example output format: +word1 word2 word3 word4 word5 + +Here are the domain names: +{domains_text} +""" + + # Make the API call + try: + print(f"Using model: {model} with temperature: {temperature}") + response = openai.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": "You are a helpful assistant that extracts words from domain names. You ONLY output the extracted words with no additional text."}, + {"role": "user", "content": prompt} + ], + temperature=temperature, + max_tokens=max_tokens + ) + + # Track token usage + prompt_tokens = response.usage.prompt_tokens + completion_tokens = response.usage.completion_tokens + total_tokens = response.usage.total_tokens + + total_prompt_tokens += prompt_tokens + total_completion_tokens += completion_tokens + + print(f"Token usage - Prompt: {prompt_tokens}, Completion: {completion_tokens}, Total: {total_tokens}") + + # Calculate cost (approximate, based on current pricing) + if "gpt-4.1" in model: + prompt_cost = (prompt_tokens / 1000000) * 2.00 # $2.00 per 1M tokens for GPT-4.1 input + completion_cost = (completion_tokens / 1000000) * 8.00 # $8.00 per 1M tokens for GPT-4.1 output + else: + prompt_cost = 0 + completion_cost = 0 + + batch_cost = prompt_cost + completion_cost + total_cost += batch_cost + print(f"Estimated batch cost: ${batch_cost:.6f}") + + # Extract the words from the response + words_text = response.choices[0].message.content.strip() + + # Process the response to get a clean list of words + batch_words = [] + for line in words_text.split('\n'): + line = line.strip() + if line and not line.startswith('```') and not line.endswith('```'): + # Remove any list markers like "1. ", "- ", etc. + cleaned_line = re.sub(r'^[\d\-\*\•\.\s]+', '', line) + if cleaned_line: + batch_words.extend(cleaned_line.split()) + + all_words.extend(batch_words) + print(f"Extracted {len(batch_words)} words from this batch") + + except Exception as e: + print(f"Error calling OpenAI API for batch: {e}") + + print(f"Total token usage - Prompt: {total_prompt_tokens}, Completion: {total_completion_tokens}") + print(f"Total estimated cost: ${total_cost:.6f}") + + return all_words + +# Process domain names using OpenAI +print("Extracting words from domain names using OpenAI...") +extracted_words = extract_words_with_openai(domain_names, SPECIAL_TERMS) +print(f"Extracted {len(extracted_words)} words") +# print("Sample of extracted words:", extracted_words) + +# Join the extracted words for the word cloud +processed_text = ' '.join(extracted_words) +# print("Processed text sample:", processed_text) + +def custom_color_func(word, font_size, position, orientation, random_state=None, + **kwargs): + return "hsl(215, 100%%, %d%%)" % random.randint(15, 80) + +mask = np.array(Image.open(path.join(d, 'mask.png'))) + +# Get configuration values with defaults +width = int(config.get('width', 800)) +height = int(config.get('height', 800)) +max_words = int(config.get('max_words', 500)) +background_color = config.get('background_color', 'white') +min_word_length = int(config.get('min_word_length', 2)) +include_numbers = config.get('include_numbers', True) + +# Handle transparent background +if background_color == 'transparent': + background_color = None + +# Get additional stopwords +additional_stopwords = config.get('additional_stopwords', []) + +stopwords = set(STOPWORDS) +stopwords = { + 'ja', 'ning', 'et', 'kui', 'aga', 'ka', 'ei', 'see', 'on', 'ole', + 'oma', 'seda', 'siis', 'või', 'mis', 'nii', 'veel', 'kes', 'üle', + 'välja', 'olema', 'kus', 'nagu', 'kuid', 'selle', 'pole', 'ära', + 'vaid', 'sest', 'juba', 'meie', 'mida', 'need', 'olid', 'minu', + 'tema', 'pärast', 'mingi', 'palju', 'kõik', 'seal', 'olen', 'oled', + 'oli', 'olnud', 'ongi', 'poolt', 'meil', 'teda', 'just', 'kuna', + 'läbi', 'küll', + 'the', 'and', 'a', 'to', 'of', 'in', 'is', 'that', 'it', 'for', + 'with', 'as', 'be', 'on', 'not', 'this', 'but', 'by', 'from', 'are', + 'or', 'an', 'at', 'was', 'have', 'has', 'had', 'were', 'will', 'would', + 'should', 'can', 'could', 'may', 'might', 'must', 'do', 'does', 'did', + 'doing', 'done', 'their', 'they', 'them', 'there', 'these', 'those', + 'which', 'who', 'whom', 'whose', 'what', 'when', 'where', 'why', 'how' +} + +stopwords.update(stopwords) +stopwords.update(additional_stopwords) + +font_path = path.join(d, 'fonts', 'Pacifico-Regular.ttf') +# Alternative: use a system font +# font_path = fm.findfont(fm.FontProperties(family='Arial')) + +print("Generating word cloud...") +wc = WordCloud(width=width, height=height, + mask=mask, + stopwords=stopwords, + background_color=background_color, + max_words=max_words, + include_numbers=include_numbers, + collocations=False, + min_word_length=min_word_length, + regexp=r"[A-Za-zÕÄÖÜõäöü0-9][\w\-'ÕÄÖÜõäöü]*(?