zydronium/internetee-registry
1
0
Fork 0
mirror of https://github.com/internetee/registry.git synced 2025-08-16 22:43:50 +02:00
Code Issues Projects Releases Packages Wiki Activity

Added additional prompt setting

Browse source
This commit is contained in:
tsoganov 2025-05-15 09:35:35 +03:00
parent 67a9a2ac45
commit 5b6888eb43
9 changed files with 176 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

3
app/controllers/admin/tools/wordcloud_controller.rb
View file

@ -96,7 +96,8 @@ module Admin
background_color: params[:background_color].presence || 'white', background_color: params[:background_color].presence || 'white',
min_word_length: params[:min_word_length].presence || 2, min_word_length: params[:min_word_length].presence || 2,
include_numbers: params[:include_numbers] == '1', include_numbers: params[:include_numbers] == '1',
batch_size: params[:batch_size].presence || 500 batch_size: params[:batch_size].presence || 500,
additional_prompt: params[:additional_prompt].presence || nil
} }
# Process additional stopwords # Process additional stopwords

114
app/views/admin/tools/wordcloud/_form.html.erb Normal file
View file

@ -0,0 +1,114 @@
<%= form_tag admin_tools_wordcloud_path, method: :post, multipart: true do %>
<div class="row">
<div class="col-md-12">
<div class="form-group">
<div class="custom-file-upload">
<p class="text-muted"><%= t('admin.tools.wordcloud.custom_file_description') %></p>
<%= file_field_tag :domains_file, accept: '.csv', class: 'form-control' %>
</div>
</div>
</div>
</div>
<div class="panel panel-default mt-3">
<div class="panel-heading">
<h4 class="panel-title">
<a data-toggle="collapse" href="#advancedOptions">
<i class="fa fa-cog"></i> <%= t('admin.tools.wordcloud.advanced_options') %>
</a>
</h4>
</div>
<div id="advancedOptions" class="panel-collapse collapse">
<div class="panel-body">
<div class="row">
<div class="col-md-4">
<div class="form-group">
<%= label_tag :width, t('admin.tools.wordcloud.width') %>
<%= number_field_tag :width, @config['width'], min: 400, max: 2000, step: 100, class: 'form-control' %>
</div>
</div>
<div class="col-md-4">
<div class="form-group">
<%= label_tag :height, t('admin.tools.wordcloud.height') %>
<%= number_field_tag :height, @config['height'], min: 400, max: 2000, step: 100, class: 'form-control' %>
</div>
</div>
<div class="col-md-4">
<div class="form-group">
<%= label_tag :max_words, t('admin.tools.wordcloud.max_words') %>
<%= number_field_tag :max_words, @config['max_words'], min: 100, max: 1000, step: 50, class: 'form-control' %>
</div>
</div>
</div>
<div class="row">
<div class="col-md-6">
<div class="form-group">
<%= label_tag :batch_size, t('admin.tools.wordcloud.batch_size') %>
<%= number_field_tag :batch_size, @config['batch_size'], min: 100, max: 1000, step: 50, class: 'form-control' %>
<small class="text-muted"><%= t('admin.tools.wordcloud.batch_size_help') %></small>
</div>
</div>
<div class="col-md-6">
<div class="form-group">
<%= label_tag :background_color, t('admin.tools.wordcloud.background') %>
<%= select_tag :background_color,
options_for_select([
['White', 'white'],
['Black', 'black'],
['Transparent', 'transparent'],
['Light Gray', '#f0f0f0']
], @config['background_color']),
class: 'form-control' %>
</div>
</div>
</div>
<div class="row">
<div class="col-md-6">
<div class="form-group">
<%= label_tag :min_word_length, t('admin.tools.wordcloud.min_word_length') %>
<%= number_field_tag :min_word_length, @config['min_word_length'], min: 1, max: 5, class: 'form-control' %>
</div>
</div>
<div class="col-md-6">
<div class="form-group">
<div class="checkbox" style="margin-top: 30px;">
<label>
<%= check_box_tag :include_numbers, '1', @config['include_numbers'] %>
<%= t('admin.tools.wordcloud.include_numbers') %>
</label>
</div>
</div>
</div>
</div>
<div class="form-group">
<%= label_tag :special_terms, t('admin.tools.wordcloud.special_terms') %>
<%= text_field_tag :special_terms, @config['special_terms'].is_a?(Array) ? @config['special_terms'].join(', ') : '',
class: 'form-control',
placeholder: t('admin.tools.wordcloud.special_terms_placeholder') %>
<small class="text-muted"><%= t('admin.tools.wordcloud.special_terms_help') %></small>
</div>
<div class="form-group">
<%= label_tag :additional_stopwords, t('admin.tools.wordcloud.additional_stopwords') %>
<%= text_area_tag :additional_stopwords, @config['additional_stopwords'].is_a?(Array) ? @config['additional_stopwords'].join(', ') : '',
rows: 3,
placeholder: t('admin.tools.wordcloud.stopwords_placeholder'),
class: 'form-control' %>
<small class="text-muted"><%= t('admin.tools.wordcloud.stopwords_help') %></small>
</div>
<div class="form-group">
<%= label_tag :additional_prompt, t('admin.tools.wordcloud.additional_prompt') %>
<%= text_area_tag :additional_prompt, @config['additional_prompt'], class: "form-control", rows: 3,
placeholder: t('admin.tools.wordcloud.additional_prompt_placeholder') %>
<small class="form-text text-muted"><%= t('admin.tools.wordcloud.additional_prompt_help') %></small>
</div>
</div>
</div>
</div>
<%= submit_tag t('admin.tools.generate_wordcloud'), class: 'btn btn-primary btn-lg mt-3' %>
<% end %>

126
app/views/admin/tools/wordcloud/index.html.erb
View file

@ -1,7 +1,7 @@
<% content_for :actions do %> <% content_for :actions do %>
<%= link_to t('back'), admin_tools_path, class: 'btn btn-default' %> <%= link_to t('back'), admin_tools_path, class: 'btn btn-default' %>
<% end %> <% end %>
<%= render "shared/title", name: t('admin.tools.wordcloud_title') %> <%= render "shared/title", name: t('admin.tools.wordcloud.title') %>
<style> <style>
.wordcloud-container { .wordcloud-container {
@ -42,135 +42,29 @@
<div class="col-md-8"> <div class="col-md-8">
<div class="panel panel-default"> <div class="panel panel-default">
<div class="panel-heading"> <div class="panel-heading">
<h3 class="panel-title"><%= t('admin.tools.wordcloud_title') %></h3> <h3 class="panel-title"><%= t('admin.tools.wordcloud.title') %></h3>
</div> </div>
<div class="panel-body text-center"> <div class="panel-body text-center">
<% if @wordcloud_url %> <% if @wordcloud_url %>
<div class="wordcloud-container"> <div class="wordcloud-container">
<%= link_to @wordcloud_url, target: "_blank", title: t('admin.tools.view_full_size') do %> <%= link_to @wordcloud_url, target: "_blank", title: t('admin.tools.wordcloud.view_full_size') do %>
<%= image_tag @wordcloud_url, class: 'img-responsive', alt: t('admin.tools.wordcloud_title') %> <%= image_tag @wordcloud_url, class: 'img-responsive', alt: t('admin.tools.wordcloud.title') %>
<div class="text-center mt-2"> <div class="text-center mt-2">
<small><i class="fa fa-search-plus"></i> <%= t('admin.tools.click_to_enlarge') %></small> <small><i class="fa fa-search-plus"></i> <%= t('admin.tools.wordcloud.click_to_enlarge') %></small>
</div> </div>
<% end %> <% end %>
<% if @wordcloud_generated_at %> <% if @wordcloud_generated_at %>
<div class="text-muted mt-2"> <div class="text-muted mt-2">
<small><i class="fa fa-clock-o"></i> <%= t('admin.tools.generated_at', time: l(@wordcloud_generated_at, format: :long)) %></small> <small><i class="fa fa-clock-o"></i> <%= t('admin.tools.wordcloud.generated_at', time: l(@wordcloud_generated_at, format: :long)) %></small>
</div> </div>
<% end %> <% end %>
</div> </div>
<% end %> <% end %>
<div class="instructions"> <div class="instructions">
<p><%= t('admin.tools.wordcloud_instructions') %></p> <p><%= t('admin.tools.wordcloud.instructions') %></p>
</div> </div>
<%= form_tag admin_tools_wordcloud_path, method: :post, multipart: true do %> <%= render 'admin/tools/wordcloud/form' %>
<div class="row">
<div class="col-md-12">
<div class="form-group">
<div class="custom-file-upload">
<p class="text-muted"><%= t('admin.tools.custom_file_description') %></p>
<%= file_field_tag :domains_file, accept: '.csv', class: 'form-control' %>
</div>
</div>
</div>
</div>
<div class="panel panel-default mt-3">
<div class="panel-heading">
<h4 class="panel-title">
<a data-toggle="collapse" href="#advancedOptions">
<i class="fa fa-cog"></i> <%= t('admin.tools.advanced_options') %>
</a>
</h4>
</div>
<div id="advancedOptions" class="panel-collapse collapse">
<div class="panel-body">
<div class="row">
<div class="col-md-4">
<div class="form-group">
<%= label_tag :width, t('admin.tools.wordcloud_width') %>
<%= number_field_tag :width, @config['width'], min: 400, max: 2000, step: 100, class: 'form-control' %>
</div>
</div>
<div class="col-md-4">
<div class="form-group">
<%= label_tag :height, t('admin.tools.wordcloud_height') %>
<%= number_field_tag :height, @config['height'], min: 400, max: 2000, step: 100, class: 'form-control' %>
</div>
</div>
<div class="col-md-4">
<div class="form-group">
<%= label_tag :max_words, t('admin.tools.wordcloud_max_words') %>
<%= number_field_tag :max_words, @config['max_words'], min: 100, max: 1000, step: 50, class: 'form-control' %>
</div>
</div>
</div>
<div class="row">
<div class="col-md-6">
<div class="form-group">
<%= label_tag :batch_size, t('admin.tools.batch_size') %>
<%= number_field_tag :batch_size, @config['batch_size'], min: 100, max: 1000, step: 50, class: 'form-control' %>
<small class="text-muted"><%= t('admin.tools.batch_size_help') %></small>
</div>
</div>
<div class="col-md-6">
<div class="form-group">
<%= label_tag :background_color, t('admin.tools.wordcloud_background') %>
<%= select_tag :background_color,
options_for_select([
['White', 'white'],
['Black', 'black'],
['Transparent', 'transparent'],
['Light Gray', '#f0f0f0']
], @config['background_color']),
class: 'form-control' %>
</div>
</div>
</div>
<div class="row">
<div class="col-md-6">
<div class="form-group">
<%= label_tag :min_word_length, t('admin.tools.min_word_length') %>
<%= number_field_tag :min_word_length, @config['min_word_length'], min: 1, max: 5, class: 'form-control' %>
</div>
</div>
<div class="col-md-6">
<div class="form-group">
<div class="checkbox" style="margin-top: 30px;">
<label>
<%= check_box_tag :include_numbers, '1', @config['include_numbers'] %>
<%= t('admin.tools.include_numbers') %>
</label>
</div>
</div>
</div>
</div>
<div class="form-group">
<%= label_tag :special_terms, t('admin.tools.special_terms') %>
<%= text_field_tag :special_terms, @config['special_terms'].is_a?(Array) ? @config['special_terms'].join(', ') : '',
class: 'form-control',
placeholder: t('admin.tools.special_terms_placeholder') %>
<small class="text-muted"><%= t('admin.tools.special_terms_help') %></small>
</div>
<div class="form-group">
<%= label_tag :additional_stopwords, t('admin.tools.additional_stopwords') %>
<%= text_area_tag :additional_stopwords, @config['additional_stopwords'].is_a?(Array) ? @config['additional_stopwords'].join(', ') : '',
rows: 3,
placeholder: t('admin.tools.stopwords_placeholder'),
class: 'form-control' %>
<small class="text-muted"><%= t('admin.tools.stopwords_help') %></small>
</div>
</div>
</div>
</div>
<%= submit_tag t('admin.tools.generate_wordcloud'), class: 'btn btn-primary btn-lg mt-3' %>
<% end %>
</div> </div>
</div> </div>
</div> </div>
@ -178,7 +72,7 @@
<div class="col-md-4"> <div class="col-md-4">
<div class="panel panel-default"> <div class="panel panel-default">
<div class="panel-heading"> <div class="panel-heading">
<h3 class="panel-title"><%= t('admin.tools.top_words') %></h3> <h3 class="panel-title"><%= t('admin.tools.wordcloud.top_words') %></h3>
</div> </div>
<div class="panel-body"> <div class="panel-body">
<% if @top_words && @top_words.any? %> <% if @top_words && @top_words.any? %>
@ -188,7 +82,7 @@
<% end %> <% end %>
</ol> </ol>
<% else %> <% else %>
<p class="text-muted"><%= t('admin.tools.top_words_empty') %></p> <p class="text-muted"><%= t('admin.tools.wordcloud.top_words_empty') %></p>
<% end %> <% end %>
</div> </div>
</div> </div>

2
config/environments/production.rb
View file

@ -55,7 +55,7 @@ Rails.application.configure do
# Use a different cache store in production. # Use a different cache store in production.
# config.cache_store = :mem_cache_store # config.cache_store = :mem_cache_store
config.cache_store = :redis_cache_store, { config.cache_store = :redis_cache_store, {
url: "#{ENV.fetch('REDIS_URL', 'redis://localhost:6379')}/1', url: "#{ENV.fetch('REDIS_URL', 'redis://localhost:6379')}/1",
expires_in: 300.seconds expires_in: 300.seconds
} }

66
config/locales/admin/tools.en.yml
View file

@ -6,35 +6,37 @@ en:
wordcloud_generator: "Domain Name Wordcloud Generator" wordcloud_generator: "Domain Name Wordcloud Generator"
wordcloud_generator_description: "Generate a visual wordcloud from domain names in the registry" wordcloud_generator_description: "Generate a visual wordcloud from domain names in the registry"
generate_wordcloud: "Generate Wordcloud" generate_wordcloud: "Generate Wordcloud"
regenerate_wordcloud: "Regenerate Wordcloud" wordcloud:
regenerate_description: "Click below to regenerate the wordcloud with the latest domain data" title: "Domain Name Wordcloud"
wordcloud_title: "Domain Name Wordcloud" success: "Wordcloud generated successfully"
wordcloud_success: "Wordcloud generated successfully" error: "Error generating wordcloud"
wordcloud_error: "Error generating wordcloud" processing: "Processing domain names. This may take a few minutes..."
wordcloud_processing: "Processing domain names. This may take a few minutes..." instructions: "Generate a visual representation of the most common words in domain names. Click the button below to create the wordcloud."
wordcloud_instructions: "Generate a visual representation of the most common words in domain names. Click the button below to create the wordcloud." top_words: "Top Words"
top_words: "Top Words" top_words_empty: "Generate a wordcloud to see the most frequent words."
top_words_empty: "Generate a wordcloud to see the most frequent words." click_to_enlarge: "Click to enlarge"
click_to_enlarge: "Click to enlarge" view_full_size: "View full size wordcloud image"
view_full_size: "View full size wordcloud image" use_custom_domains: "Use custom domain list"
use_custom_domains: "Use custom domain list" custom_file_description: "Upload a CSV file with one domain name per line"
custom_file_description: "Upload a CSV file with one domain name per line" file_upload_error: "Error processing uploaded file"
file_upload_error: "Error processing uploaded file" file_optional: "If no file is uploaded, all active domains in the registry will be used"
file_optional: "If no file is uploaded, all active domains in the registry will be used" generated_at: "Generated at %{time}"
generated_at: "Generated at %{time}" no_file: "No domain names found"
wordcloud_no_file: "No domain names found" width: "Width"
wordcloud_width: "Width" height: "Height"
wordcloud_height: "Height" max_words: "Max Words"
wordcloud_max_words: "Max Words" background: "Background"
wordcloud_background: "Background" additional_stopwords: "Stopwords"
additional_stopwords: "Stopwords" stopwords_placeholder: "Enter additional stopwords, one per line"
stopwords_placeholder: "Enter additional stopwords, one per line" stopwords_help: "Stopwords are words that will not be included in the wordcloud"
stopwords_help: "Stopwords are words that will not be included in the wordcloud" advanced_options: "Advanced Options"
advanced_options: "Advanced Options" min_word_length: "Min Word Length"
min_word_length: "Min Word Length" include_numbers: "Include Numbers"
include_numbers: "Include Numbers" special_terms: "Special Terms"
special_terms: "Special Terms" special_terms_placeholder: "e.g., e-, i-, .com, ai, web"
special_terms_placeholder: "e.g., e-, i-, .com, ai, web" special_terms_help: "These terms will be preserved in the word cloud even if they would normally be filtered out"
special_terms_help: "These terms will be preserved in the word cloud even if they would normally be filtered out" batch_size: "Batch Size"
batch_size: "Batch Size" batch_size_help: "Number of domains to process in each API call."
batch_size_help: "Number of domains to process in each API call." additional_prompt: "Additional Prompt Text"
additional_prompt_placeholder: "Add any additional instructions for the word cloud generation here..."
additional_prompt_help: "Optional text that will be used as additional context during word cloud generation."

8
lib/wordcloud/generate_wordcloud.py
View file

@ -53,8 +53,12 @@ print(f"Using special terms: {SPECIAL_TERMS}")
BATCH_SIZE = int(config.get('batch_size', 500)) BATCH_SIZE = int(config.get('batch_size', 500))
print(f"Using batch size: {BATCH_SIZE}") print(f"Using batch size: {BATCH_SIZE}")
# Get additional prompt from config or use default
ADDITIONAL_PROMPT = config.get('additional_prompt', None)
print(f"Using additional prompt: {ADDITIONAL_PROMPT}")
# Function to extract words using OpenAI API # Function to extract words using OpenAI API
def extract_words_with_openai(domain_names, special_terms, batch_size=BATCH_SIZE): def extract_words_with_openai(domain_names, special_terms, batch_size=BATCH_SIZE, additional_prompt=ADDITIONAL_PROMPT):
# Get API key from environment variable # Get API key from environment variable
api_key = os.environ.get("OPENAI_API_KEY") api_key = os.environ.get("OPENAI_API_KEY")
if not api_key: if not api_key:
@ -95,6 +99,8 @@ Follow these rules strictly:
5. Try to find the most common words and phrases in the domain names. 5. Try to find the most common words and phrases in the domain names.
6. Return ONLY a space-separated list of words and numberswith no explanations, no formatting, no introductions, and no additional text. 6. Return ONLY a space-separated list of words and numberswith no explanations, no formatting, no introductions, and no additional text.
{additional_prompt}
Example output format: Example output format:
word1 word2 word3 word4 word5 word1 word2 word3 word4 word5

2
public/wordcloud/config.json
View file

@ -1 +1 @@
{"width":"800","height":"800","max_words":"500","background_color":"white","min_word_length":"2","include_numbers":true,"batch_size":"500","special_terms":["e-","i-","2-","3-","4-",".com","tr.ee","ai","web"]} {"width":"800","height":"800","max_words":"500","background_color":"white","min_word_length":"2","include_numbers":true,"batch_size":"500","additional_prompt":null,"special_terms":["e-","i-","2-","3-","4-",".com","tr.ee","ai","web"]}

14
public/wordcloud/top_words.txt
View file

@ -1,11 +1,11 @@
Top 10 most frequent words: Top 10 most frequent words:
1. tr: 4 1. tr: 4
2. 2-: 4 2. auto: 4
3. faktor: 4 3. 2-: 4
4. auto: 3 4. faktor: 4
5. e-: 2 5. e-: 2
6. i-: 2 6. i-: 2
7. car: 2 7. digi: 2
8. pood: 2 8. car: 2
9. ai: 1 9. pood: 2
10. robot: 1 10. ai: 1

BIN
public/wordcloud/wordcloud.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 358 KiB

After

Width:  |  Height:  |  Size: 351 KiB

Before After
Before After