zydronium/internetee-registry
1
0
Fork 0
mirror of https://github.com/internetee/registry.git synced 2025-07-27 21:16:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

Added indexes to log_domains table

Browse source
This commit is contained in:
Sergei Tsoganov 2022-12-07 11:21:30 +02:00
parent 10fb39980b
commit a682681625
7 changed files with 364 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Gemfile
View file

@ -78,6 +78,8 @@ gem 'wkhtmltopdf-binary', '~> 0.12.5.1'
gem 'directo', github: 'internetee/directo', branch: 'master'
gem 'strong_migrations'
group :development, :test do
gem 'pry', '0.14.1'
end

5
Gemfile.lock
View file

@ -474,6 +474,8 @@ GEM
activesupport (>= 4.0)
sprockets (>= 3.0.0)
spy (1.0.1)
strong_migrations (1.4.0)
activerecord (>= 5.2)
swd (1.3.0)
activesupport (>= 3)
attr_required (>= 0.0.5)
@ -588,6 +590,7 @@ DEPENDENCIES
simplecov (= 0.17.1)
simpleidn (= 0.2.1)
spy
strong_migrations
truemail (~> 2.4)
uglifier
validates_email_format_of (= 1.7.2)
@ -597,4 +600,4 @@ DEPENDENCIES
wkhtmltopdf-binary (~> 0.12.5.1)
BUNDLED WITH
2.3.25
2.3.26

159
app/controllers/repp/v1/stats_controller.rb
View file

@ -1,68 +1,16 @@
module Repp
module V1
class StatsController < BaseController
class StatsController < BaseController # rubocop:disable Metrics/ClassLength
before_action :set_date_params
api :get, '/repp/v1/stats/market_share_distribution'
desc 'Get market share and distribution of registrars'
param :q, Hash, required: true, desc: 'Period parameters for data' do
param :end_date, String, required: true, desc: 'Period end date'
end
def market_share_distribution
date_to = to_date(search_params[:end_date]).end_of_month
date_from = to_date(search_params[:start_date] || '01.05')
log_domains_del = ::Version::DomainVersion.where(event: 'destroy')
.where('created_at > ?', date_to)
.where("object ->> 'created_at' <= ?", date_to)
.where("object ->> 'created_at' >= ?", date_from)
.select("DISTINCT ON (object ->> 'name') object, created_at")
.order(Arel.sql("object ->> 'name', created_at desc"))
# p "log_domains_del"
# log_domains_del.each do |d|
# p d.attributes
# end
# log_domains_del_grouped = log_domains_del.group("object ->> 'registrar_id'").count
log_domains_del_grouped = log_domains_del.group_by { |ld| ld.object['registrar_id'].to_s }
.transform_values(&:count)
# p log_domains_del_grouped
log_domains_trans = ::Version::DomainVersion.where(event: 'update')
.where('created_at > ?', date_to)
.where("object ->> 'created_at' <= ?", date_to)
.where("object ->> 'created_at' >= ?", date_from)
.where("object_changes ->> 'registrar_id' IS NOT NULL")
.select("DISTINCT ON (object ->> 'name') object, created_at")
.order(Arel.sql("object ->> 'name', created_at desc"))
# p "log_domains_trans"
# p log_domains_trans.map{|d| [d.object['name'], d.created_at] }
log_domains_trans_grouped = log_domains_trans.group_by { |ld| ld.object['registrar_id'].to_s }
.transform_values(&:count)
# p log_domains_trans_grouped
updated = log_domains_trans.map { |ld| ld.object['name'] } + log_domains_del.map { |ld| ld.object['name'] }
domains = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from)
.where.not(name: updated.uniq)
domains_grouped = domains.group(:registrar_id).count.stringify_keys
# p "domains"
# p domains_grouped
grouped = summarize([log_domains_del_grouped, log_domains_trans_grouped, domains_grouped])
registrar_names = ::Registrar.where(test_registrar: false)
.map { |r| { "#{r.id}": r.name }.with_indifferent_access }
.inject(:merge)
result = grouped.map do |key, value|
next unless registrar_names.key?(key)
name = registrar_names[key]
hash = { name: registrar_names[key], y: value }
hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name
hash
end
# p result
domains_by_rar = domains_by_registrar(@date_to, @date_from)
result = serialize_distribution_result(domains_by_rar)
render_success(data: result)
end
@ -71,25 +19,23 @@ module Repp
desc 'Get market share and growth rate of registrars'
param :q, Hash, required: true, desc: 'Period parameters for data' do
param :end_date, String, required: true, desc: 'Period end date'
param :compare_to_date, String, required: true, desc: 'Comparison date'
param :compare_to_end_date, String, required: true, desc: 'Comparison date'
end
def market_share_growth_rate
registrars = ::Registrar.where(test_registrar: false).joins(:domains)
domains_by_rar = registrars.where(from_condition).where(to_condition).group(:name).count
prev_domains_by_rar = registrars.where(compare_to_condition).group(:name).count
domains_by_rar = domains_by_registrar(@date_to, @date_from)
prev_domains_by_rar = domains_by_registrar(@date_compare_to, @date_compare_from)
set_zero_values!(domains_by_rar, prev_domains_by_rar)
market_share_by_rar = calculate_market_share(domains_by_rar)
prev_market_share_by_rar = calculate_market_share(prev_domains_by_rar)
result = { prev_data: { name: search_params[:compare_to_date],
domains: serialize(prev_domains_by_rar),
market_share: serialize(prev_market_share_by_rar) },
result = { prev_data: { name: search_params[:compare_to_end_date],
domains: serialize_growth_rate_result(prev_domains_by_rar),
market_share: serialize_growth_rate_result(prev_market_share_by_rar) },
data: { name: search_params[:end_date],
domains: serialize(domains_by_rar),
market_share: serialize(market_share_by_rar) } }
domains: serialize_growth_rate_result(domains_by_rar),
market_share: serialize_growth_rate_result(market_share_by_rar) } }
render_success(data: result)
end
# rubocop:enable Metrics/MethodLength
@ -97,26 +43,15 @@ module Repp
private
def search_params
params.permit(:q, q: %i[start_date end_date compare_to_date])
params.permit(:q, q: %i[start_date end_date compare_to_end_date compare_to_start_date])
.fetch(:q, {}) || {}
end
def from_condition
return unless search_params[:start_date]
"domains.created_at >= '#{to_date(search_params[:start_date])}'"
end
def to_condition
return unless search_params[:end_date]
"domains.created_at <= '#{to_date(search_params[:end_date]).end_of_month}'"
end
def compare_to_condition
return unless search_params[:compare_to_date]
"domains.created_at <= '#{to_date(search_params[:compare_to_date]).end_of_month}'"
def set_date_params
@date_to = to_date(search_params[:end_date]).end_of_month
@date_from = to_date(search_params[:start_date] || '01.05')
@date_compare_to = to_date(search_params[:compare_to_end_date]).end_of_month
@date_compare_from = to_date(search_params[:compare_to_start_date] || '01.05')
end
def to_date(date_param)
@ -125,10 +60,6 @@ module Repp
Date.strptime(date_param, '%m.%y')
end
def serialize(rars)
rars.map { |key, value| [key, value] }
end
def set_zero_values!(cur, prev)
cur_dup = cur.dup
cur_dup.each_key do |k|
@ -147,9 +78,61 @@ module Repp
end
end
def domains_by_registrar(date_to, date_from)
log_domains_del = log_domains(event: 'destroy', date_to: date_to, date_from: date_from)
log_domains_trans = log_domains(event: 'update', date_to: date_to, date_from: date_from)
logged_domains = log_domains_trans.map { |ld| ld.object['name'] } +
log_domains_del.map { |ld| ld.object['name'] }
domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from)
.where.not(name: logged_domains.uniq)
.group(:registrar_id).count.stringify_keys
summarize([group(log_domains_del), group(log_domains_trans), domains_grouped])
end
def summarize(arr)
arr.inject { |memo, el| memo.merge(el) { |_, old_v, new_v| old_v + new_v } }
end
def log_domains(event:, date_to:, date_from:)
domains = ::Version::DomainVersion.where(event: event)
domains.where!("object_changes ->> 'registrar_id' IS NOT NULL") if event == 'update'
domains.where('created_at > ?', date_to)
.where("object ->> 'created_at' <= ?", date_to)
.where("object ->> 'created_at' >= ?", date_from)
.select("DISTINCT ON (object ->> 'name') object, created_at")
.order(Arel.sql("object ->> 'name', created_at desc"))
end
def group(domains)
domains.group_by { |ld| ld.object['registrar_id'].to_s }
.transform_values(&:count)
end
def registrar_names
@registrar_names ||= ::Registrar.where(test_registrar: false)
.map { |r| { "#{r.id}": r.name }.with_indifferent_access }
.inject(:merge)
end
def serialize_distribution_result(grouped)
grouped.map do |key, value|
next unless registrar_names.key?(key)
name = registrar_names[key]
hash = { name: registrar_names[key], y: value }
hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name
hash
end
end
def serialize_growth_rate_result(grouped)
grouped.map do |key, value|
next unless registrar_names.key?(key)
name = registrar_names[key]
[name, value]
end
end
end
end
end

26
config/initializers/strong_migrations.rb Normal file
View file

@ -0,0 +1,26 @@
# Mark existing migrations as safe
StrongMigrations.start_after = 20221206081743
# Set timeouts for migrations
# If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user
StrongMigrations.lock_timeout = 10.seconds
StrongMigrations.statement_timeout = 1.hour
# Analyze tables after indexes are added
# Outdated statistics can sometimes hurt performance
StrongMigrations.auto_analyze = true
# Set the version of the production database
# so the right checks are run in development
# StrongMigrations.target_version = 10
# Add custom checks
# StrongMigrations.add_check do |method, args|
# if method == :add_index && args[0].to_s == "users"
# stop! "No more indexes on the users table"
# end
# end
# Make some operations safe by default
# See https://github.com/ankane/strong_migrations#safe-by-default
# StrongMigrations.safe_by_default = true

21
db/migrate/20221206090120_modify_log_domains_object_changes_data_type.rb Normal file
View file

@ -0,0 +1,21 @@
class ModifyLogDomainsObjectChangesDataType < ActiveRecord::Migration[6.1]
def up
add_column :log_domains, :object_changes_jsonb, :jsonb, default: '{}'
# Copy data from old column to the new one
Version::DomainVersion.update_all('object_changes_jsonb = object_changes::jsonb')
# Rename columns instead of modify their type, it's way faster
safety_assured do
rename_column :log_domains, :object_changes, :object_changes_json
rename_column :log_domains, :object_changes_jsonb, :object_changes
end
end
def down
safety_assured do
rename_column :log_domains, :object_changes, :object_changes_jsonb
rename_column :log_domains, :object_changes_json, :object_changes
end
end
end

15
db/migrate/20221206091556_add_indexes_to_log_domains.rb Normal file
View file

@ -0,0 +1,15 @@
class AddIndexesToLogDomains < ActiveRecord::Migration[6.1]
disable_ddl_transaction!
def up
enable_extension 'btree_gin'
add_index :log_domains, :event, algorithm: :concurrently
add_index :log_domains, :object, using: :gin, algorithm: :concurrently
add_index :log_domains, :object_changes, using: :gin, algorithm: :concurrently
end
def down
remove_index :log_domains, :event
remove_index :log_domains, :object
remove_index :log_domains, :object_changes
end
end

243
db/structure.sql
View file

@ -9,6 +9,27 @@ SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;
--
-- Name: public; Type: SCHEMA; Schema: -; Owner: -
--
-- *not* creating schema, since initdb creates it
--
-- Name: btree_gin; Type: EXTENSION; Schema: -; Owner: -
--
CREATE EXTENSION IF NOT EXISTS btree_gin WITH SCHEMA public;
--
-- Name: EXTENSION btree_gin; Type: COMMENT; Schema: -; Owner: -
--
COMMENT ON EXTENSION btree_gin IS 'support for indexing common datatypes in GIN';
--
-- Name: btree_gist; Type: EXTENSION; Schema: -; Owner: -
--
@ -956,15 +977,14 @@ CREATE TABLE public.domains (
pending_json jsonb,
force_delete_date date,
statuses character varying[],
statuses_before_force_delete character varying[] DEFAULT '{}'::character varying[],
status_notes public.hstore,
upid integer,
up_date timestamp without time zone,
uuid uuid DEFAULT public.gen_random_uuid() NOT NULL,
locked_by_registrant_at timestamp without time zone,
force_delete_start timestamp without time zone,
force_delete_data public.hstore,
json_statuses_history jsonb,
status_notes public.hstore
json_statuses_history jsonb
);
@ -1651,11 +1671,12 @@ CREATE TABLE public.log_domains (
event character varying NOT NULL,
whodunnit character varying,
object jsonb,
object_changes json,
object_changes_json json,
created_at timestamp without time zone,
session character varying,
children jsonb,
uuid character varying
uuid character varying,
object_changes jsonb DEFAULT '"{}"'::jsonb
);
@ -2287,6 +2308,74 @@ CREATE SEQUENCE public.payment_orders_id_seq
ALTER SEQUENCE public.payment_orders_id_seq OWNED BY public.payment_orders.id;
--
-- Name: pghero_query_stats; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.pghero_query_stats (
id bigint NOT NULL,
database text,
"user" text,
query text,
query_hash bigint,
total_time double precision,
calls bigint,
captured_at timestamp without time zone
);
--
-- Name: pghero_query_stats_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.pghero_query_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: pghero_query_stats_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.pghero_query_stats_id_seq OWNED BY public.pghero_query_stats.id;
--
-- Name: pghero_space_stats; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.pghero_space_stats (
id bigint NOT NULL,
database text,
schema text,
relation text,
size bigint,
captured_at timestamp without time zone
);
--
-- Name: pghero_space_stats_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.pghero_space_stats_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: pghero_space_stats_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.pghero_space_stats_id_seq OWNED BY public.pghero_space_stats.id;
--
-- Name: prices; Type: TABLE; Schema: public; Owner: -
--
@ -2325,6 +2414,48 @@ CREATE SEQUENCE public.prices_id_seq
ALTER SEQUENCE public.prices_id_seq OWNED BY public.prices.id;
--
-- Name: que_jobs; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.que_jobs (
priority smallint DEFAULT 100 NOT NULL,
run_at timestamp with time zone DEFAULT now() NOT NULL,
job_id bigint NOT NULL,
job_class text NOT NULL,
args json DEFAULT '[]'::json NOT NULL,
error_count integer DEFAULT 0 NOT NULL,
last_error text,
queue text DEFAULT ''::text NOT NULL
);
--
-- Name: TABLE que_jobs; Type: COMMENT; Schema: public; Owner: -
--
COMMENT ON TABLE public.que_jobs IS '3';
--
-- Name: que_jobs_job_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.que_jobs_job_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: que_jobs_job_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.que_jobs_job_id_seq OWNED BY public.que_jobs.job_id;
--
-- Name: registrant_verifications; Type: TABLE; Schema: public; Owner: -
--
@ -2605,8 +2736,7 @@ CREATE TABLE public.validation_events (
validation_eventable_type character varying,
validation_eventable_id bigint,
created_at timestamp(6) without time zone NOT NULL,
updated_at timestamp(6) without time zone NOT NULL,
event_type public.validation_type
updated_at timestamp(6) without time zone NOT NULL
);
@ -3148,6 +3278,20 @@ ALTER TABLE ONLY public.notifications ALTER COLUMN id SET DEFAULT nextval('publi
ALTER TABLE ONLY public.payment_orders ALTER COLUMN id SET DEFAULT nextval('public.payment_orders_id_seq'::regclass);
--
-- Name: pghero_query_stats id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.pghero_query_stats ALTER COLUMN id SET DEFAULT nextval('public.pghero_query_stats_id_seq'::regclass);
--
-- Name: pghero_space_stats id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.pghero_space_stats ALTER COLUMN id SET DEFAULT nextval('public.pghero_space_stats_id_seq'::regclass);
--
-- Name: prices id; Type: DEFAULT; Schema: public; Owner: -
--
@ -3155,6 +3299,13 @@ ALTER TABLE ONLY public.payment_orders ALTER COLUMN id SET DEFAULT nextval('publ
ALTER TABLE ONLY public.prices ALTER COLUMN id SET DEFAULT nextval('public.prices_id_seq'::regclass);
--
-- Name: que_jobs job_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.que_jobs ALTER COLUMN job_id SET DEFAULT nextval('public.que_jobs_job_id_seq'::regclass);
--
-- Name: registrant_verifications id; Type: DEFAULT; Schema: public; Owner: -
--
@ -3664,6 +3815,22 @@ ALTER TABLE ONLY public.payment_orders
ADD CONSTRAINT payment_orders_pkey PRIMARY KEY (id);
--
-- Name: pghero_query_stats pghero_query_stats_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.pghero_query_stats
ADD CONSTRAINT pghero_query_stats_pkey PRIMARY KEY (id);
--
-- Name: pghero_space_stats pghero_space_stats_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.pghero_space_stats
ADD CONSTRAINT pghero_space_stats_pkey PRIMARY KEY (id);
--
-- Name: prices prices_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@ -3672,6 +3839,14 @@ ALTER TABLE ONLY public.prices
ADD CONSTRAINT prices_pkey PRIMARY KEY (id);
--
-- Name: que_jobs que_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.que_jobs
ADD CONSTRAINT que_jobs_pkey PRIMARY KEY (queue, priority, run_at, job_id);
--
-- Name: registrant_verifications registrant_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@ -4251,6 +4426,13 @@ CREATE INDEX index_log_domain_contacts_on_item_type_and_item_id ON public.log_do
CREATE INDEX index_log_domain_contacts_on_whodunnit ON public.log_domain_contacts USING btree (whodunnit);
--
-- Name: index_log_domains_on_event; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_log_domains_on_event ON public.log_domains USING btree (event);
--
-- Name: index_log_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -
--
@ -4258,6 +4440,20 @@ CREATE INDEX index_log_domain_contacts_on_whodunnit ON public.log_domain_contact
CREATE INDEX index_log_domains_on_item_type_and_item_id ON public.log_domains USING btree (item_type, item_id);
--
-- Name: index_log_domains_on_object; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_log_domains_on_object ON public.log_domains USING gin (object);
--
-- Name: index_log_domains_on_object_changes; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_log_domains_on_object_changes ON public.log_domains USING gin (object_changes);
--
-- Name: index_log_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: -
--
@ -4426,6 +4622,20 @@ CREATE INDEX index_notifications_on_registrar_id ON public.notifications USING b
CREATE INDEX index_payment_orders_on_invoice_id ON public.payment_orders USING btree (invoice_id);
--
-- Name: index_pghero_query_stats_on_database_and_captured_at; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_pghero_query_stats_on_database_and_captured_at ON public.pghero_query_stats USING btree (database, captured_at);
--
-- Name: index_pghero_space_stats_on_database_and_captured_at; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_pghero_space_stats_on_database_and_captured_at ON public.pghero_space_stats USING btree (database, captured_at);
--
-- Name: index_prices_on_zone_id; Type: INDEX; Schema: public; Owner: -
--
@ -4482,13 +4692,6 @@ CREATE INDEX index_users_on_registrar_id ON public.users USING btree (registrar_
CREATE INDEX index_validation_events_on_event_data ON public.validation_events USING gin (event_data);
--
-- Name: index_validation_events_on_event_type; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_validation_events_on_event_type ON public.validation_events USING btree (event_type);
--
-- Name: index_validation_events_on_validation_eventable; Type: INDEX; Schema: public; Owner: -
--
@ -5234,9 +5437,11 @@ INSERT INTO "schema_migrations" (version) VALUES
('20210708131814'),
('20210729131100'),
('20210729134625'),
('20210827185249'),
('20211029073644'),
('20211028122103'),
('20211028125245'),
('20211029082225'),
('20211124071418'),
('20211124084308'),
('20211125181033'),
('20211125184334'),
('20211126085139'),
@ -5245,7 +5450,6 @@ INSERT INTO "schema_migrations" (version) VALUES
('20220113201642'),
('20220113220809'),
('20220124105717'),
('20220216113112'),
('20220228093211'),
('20220316140727'),
('20220406085500'),
@ -5258,6 +5462,9 @@ INSERT INTO "schema_migrations" (version) VALUES
('20220701113409'),
('20220715145808'),
('20220818075833'),
('20221011061840');
('20221011061840'),
('20221206075912'),
('20221206090120'),
('20221206091556');