From f1c2121ca7a545bc26ee038fb6bdacbd8c8ca3e2 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 3 Jul 2015 13:38:07 +0300 Subject: [PATCH 1/9] Updated que init doc #2557 --- doc/que/que-init-example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/que/que-init-example b/doc/que/que-init-example index 95a2f45f2..d9162e013 100644 --- a/doc/que/que-init-example +++ b/doc/que/que-init-example @@ -32,7 +32,7 @@ status) start) echo "$1 que monitor and server" for i in `seq 1 $QUE_INSTANCES`; do - cd $APP_ROOT && RAILS_ENV=$RAILS_ENV $RUBY_BUNDLE_PATH exec rake daemon:que:start + cd $APP_ROOT && QUE_WORKER_COUNT=1 RAILS_ENV=$RAILS_ENV $RUBY_BUNDLE_PATH exec rake daemon:que:start echo '.' done ;; From 10889e7a12a291f43bb23a4b684811e45405bfb4 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 3 Jul 2015 13:40:27 +0300 Subject: [PATCH 2/9] Rake db honors structure sql #2751 --- db/{schema.rb => schema-read-only.rb} | 16 ++++---- db/structure.sql | 4 ++ lib/tasks/db.rake | 53 ++++++++++++++++++++++++--- 3 files changed, 60 insertions(+), 13 deletions(-) rename db/{schema.rb => schema-read-only.rb} (98%) diff --git a/db/schema.rb b/db/schema-read-only.rb similarity index 98% rename from db/schema.rb rename to db/schema-read-only.rb index 19b75088b..7a0b84e08 100644 --- a/db/schema.rb +++ b/db/schema-read-only.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150701074344) do +ActiveRecord::Schema.define(version: 20150703090039) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -932,14 +932,14 @@ ActiveRecord::Schema.define(version: 20150701074344) do end create_table "que_jobs", id: false, force: :cascade do |t| - t.integer "priority", limit: 2, default: 100, null: false - t.datetime "run_at", default: '2015-06-29 12:38:58', null: false - t.integer "job_id", limit: 8, default: 0, null: false - t.text "job_class", null: false - t.json "args", default: [], null: false - t.integer "error_count", default: 0, null: false + t.integer "priority", limit: 2, default: 100, null: false + t.datetime "run_at", default: "now()", null: false + t.integer "job_id", limit: 8, default: "nextval('que_jobs_job_id_seq'::regclass)", null: false + t.text "job_class", null: false + t.json "args", default: [], null: false + t.integer "error_count", default: 0, null: false t.text "last_error" - t.text "queue", default: "", null: false + t.text "queue", default: "", null: false end create_table "registrant_verifications", force: :cascade do |t| diff --git a/db/structure.sql b/db/structure.sql index 271e8d8ae..825f82db0 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -4851,3 +4851,7 @@ INSERT INTO schema_migrations (version) VALUES ('20150612123111'); INSERT INTO schema_migrations (version) VALUES ('20150701074344'); +INSERT INTO schema_migrations (version) VALUES ('20150703074448'); + +INSERT INTO schema_migrations (version) VALUES ('20150703090039'); + diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index cd022bcb1..4e115b471 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -1,12 +1,38 @@ +Rake::Task["db:schema:load"].clear + +Rake::Task["db:migrate"].enhance do + if ActiveRecord::Base.schema_format == :sql + Rake::Task["db:schema:dump"].invoke + end +end + +Rake::Task["db:schema:dump"].enhance do + if ActiveRecord::Base.schema_format == :sql + File.rename('db/schema.rb', 'db/schema-read-only.rb') + Rake::Task["db:structure:dump"].invoke # for users who do manually db:schema:dump + end +end + namespace :db do + namespace :schema do + task load: [:environment, :load_config] do + puts 'Only rake db:structure:load is supported and invoked now. Otherwise zonefile generation does not work nor que.' + Rake::Task["db:structure:load"].invoke + end + end + def databases @db ||= [Rails.env, "api_log_#{Rails.env}", "whois_#{Rails.env}"] end + def other_databases + @other_dbs ||= ["api_log_#{Rails.env}", "whois_#{Rails.env}"] + end + def schema_file(db) case db when Rails.env - 'schema.rb' + 'structure.sql' # just in case when "api_log_#{Rails.env}" 'api_log_schema.rb' when "whois_#{Rails.env}" @@ -25,7 +51,7 @@ namespace :db do puts "\n---------------------------- Import seed ----------------------------------------\n" Rake::Task['db:seed'].invoke - Rake::Task['zonefile:replace_procedure'].invoke + # Rake::Task['zonefile:replace_procedure'].invoke # not needed any more puts "\n All done!\n\n" end @@ -73,7 +99,10 @@ namespace :db do namespace :schema do desc 'Schema load for all databases: registry, api_log and whois' task load: [:environment, :load_config] do - databases.each do |name| + puts "\n------------------------ #{Rails.env} structure loading -----------------------------\n" + Rake::Task['db:structure:load'].invoke + + other_databases.each do |name| begin puts "\n------------------------ #{name} schema loading -----------------------------\n" ActiveRecord::Base.clear_all_connections! @@ -89,9 +118,12 @@ namespace :db do end end - desc 'Schema load for all databases: registry, api_log and whois' + desc 'Schema dump for all databases: registry, api_log and whois' task dump: [:environment, :load_config] do - databases.each do |name| + puts "\n---------------------------- #{Rails.env} structure and schema dump--------------\n" + Rake::Task['db:schema:dump'].invoke # dumps both schema and structure + + other_databases.each do |name| begin puts "\n---------------------------- #{name} ----------------------------------------\n" filename = "#{Rails.root}/db/#{schema_file(name)}" @@ -104,6 +136,17 @@ namespace :db do end end end + # alias names + namespace :structure do + desc '(alias) Schema dump for all databases: registry, api_log and whois' + task :dump do + Rake::Task['db:all:schema:dump'].invoke + end + desc '(alias) Schema load for all databases: registry, api_log and whois' + task :load do + Rake::Task['db:all:schema:load'].invoke + end + end end end end From 7086ce1cfb50dfd0583375c2a6c530af2d60b034 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 3 Jul 2015 13:45:52 +0300 Subject: [PATCH 3/9] Updated robot tests to honor struture sql #2751 --- bin/robot | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/robot b/bin/robot index 0e77d4fba..a9255c78d 100755 --- a/bin/robot +++ b/bin/robot @@ -31,8 +31,6 @@ bundle install RAILS_ENV=test bundle exec rake db:all:drop RAILS_ENV=test bundle exec rake db:all:setup -RAILS_ENV=test bundle exec rake zonefile:replace_procedure -RAILS_ENV=test bundle exec rake assets:precompile echo "GIT_LAST_COMMITS" git log --pretty='%s (%cn, %cr)' --abbrev-commit --graph --decorate -n 20 --no-color @@ -44,7 +42,7 @@ RCODE=$? echo "END_OF_RUBOCOP_RESULTS" echo "TEST_RESULTS" -# basic test +# basic tests without EPP # ROBOT=true bundle exec rake # all tests with EPP From 9f9a330950b7f99e378df37ba77aef7497b65526 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 3 Jul 2015 13:50:56 +0300 Subject: [PATCH 4/9] rake db:rollback updates schema file as well #2751 --- lib/tasks/db.rake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index 4e115b471..9a91ddfcd 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -6,6 +6,12 @@ Rake::Task["db:migrate"].enhance do end end +Rake::Task["db:rollback"].enhance do + if ActiveRecord::Base.schema_format == :sql + Rake::Task["db:schema:dump"].invoke + end +end + Rake::Task["db:schema:dump"].enhance do if ActiveRecord::Base.schema_format == :sql File.rename('db/schema.rb', 'db/schema-read-only.rb') From a1cb5ad9e3a869f9ac9db6ce6ed52bb94da2145b Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 3 Jul 2015 13:54:34 +0300 Subject: [PATCH 5/9] Update structure file --- db/schema-read-only.rb | 10 ++++---- db/structure.sql | 55 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/db/schema-read-only.rb b/db/schema-read-only.rb index 7a0b84e08..e316c3816 100644 --- a/db/schema-read-only.rb +++ b/db/schema-read-only.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150703090039) do +ActiveRecord::Schema.define(version: 20150703105159) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -919,14 +919,14 @@ ActiveRecord::Schema.define(version: 20150703090039) do create_table "pricelists", force: :cascade do |t| t.string "desc" t.string "category" - t.decimal "price_cents", precision: 8, scale: 2, default: 0.0, null: false - t.string "price_currency", default: "EUR", null: false + t.decimal "price_cents", precision: 10, scale: 2, default: 0.0, null: false + t.string "price_currency", default: "EUR", null: false t.datetime "valid_from" t.datetime "valid_to" t.string "creator_str" t.string "updator_str" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false t.string "duration" t.string "operation_category" end diff --git a/db/structure.sql b/db/structure.sql index 70087236c..25c13a804 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -74,7 +74,7 @@ CREATE FUNCTION generate_zonefile(i_origin character varying) RETURNS text SELECT concat(d.name_puny, '. IN NS ', ns.hostname, '.') FROM domains d JOIN nameservers ns ON ns.domain_id = d.id - WHERE d.name LIKE include_filter AND d.name NOT LIKE exclude_filter + WHERE d.name LIKE include_filter AND d.name NOT LIKE exclude_filter OR d.name = i_origin ORDER BY d.name ), chr(10) @@ -2404,8 +2404,8 @@ ALTER SEQUENCE pricelists_id_seq OWNED BY pricelists.id; CREATE TABLE que_jobs ( priority smallint DEFAULT 100 NOT NULL, - run_at timestamp without time zone DEFAULT '2015-06-30 14:16:50.905537'::timestamp without time zone NOT NULL, - job_id bigint DEFAULT 0 NOT NULL, + run_at timestamp with time zone DEFAULT now() NOT NULL, + job_id bigint NOT NULL, job_class text NOT NULL, args json DEFAULT '[]'::json NOT NULL, error_count integer DEFAULT 0 NOT NULL, @@ -2414,6 +2414,32 @@ CREATE TABLE que_jobs ( ); +-- +-- Name: TABLE que_jobs; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE que_jobs IS '3'; + + +-- +-- Name: que_jobs_job_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE que_jobs_job_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: que_jobs_job_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE que_jobs_job_id_seq OWNED BY que_jobs.job_id; + + -- -- Name: registrant_verifications; Type: TABLE; Schema: public; Owner: -; Tablespace: -- @@ -3169,6 +3195,13 @@ ALTER TABLE ONLY people ALTER COLUMN id SET DEFAULT nextval('people_id_seq'::reg ALTER TABLE ONLY pricelists ALTER COLUMN id SET DEFAULT nextval('pricelists_id_seq'::regclass); +-- +-- Name: job_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY que_jobs ALTER COLUMN job_id SET DEFAULT nextval('que_jobs_job_id_seq'::regclass); + + -- -- Name: id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3688,6 +3721,14 @@ ALTER TABLE ONLY pricelists ADD CONSTRAINT pricelists_pkey PRIMARY KEY (id); +-- +-- Name: que_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- + +ALTER TABLE ONLY que_jobs + ADD CONSTRAINT que_jobs_pkey PRIMARY KEY (queue, priority, run_at, job_id); + + -- -- Name: registrant_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- @@ -4812,7 +4853,11 @@ INSERT INTO schema_migrations (version) VALUES ('20150701074344'); INSERT INTO schema_migrations (version) VALUES ('20150703074448'); -INSERT INTO schema_migrations (version) VALUES ('20150703090039'); - INSERT INTO schema_migrations (version) VALUES ('20150703084632'); +INSERT INTO schema_migrations (version) VALUES ('20150703090039'); + +INSERT INTO schema_migrations (version) VALUES ('20150703104149'); + +INSERT INTO schema_migrations (version) VALUES ('20150703105159'); + From 8946a476c2ff22114e965b67d86535b9d7b774a4 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 3 Jul 2015 13:55:45 +0300 Subject: [PATCH 6/9] Revert back to original spring watcher --- Gemfile | 7 ++++--- Gemfile.lock | 10 ---------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/Gemfile b/Gemfile index 15674d021..6e502b3c1 100644 --- a/Gemfile +++ b/Gemfile @@ -103,9 +103,10 @@ group :development do # dev tools gem 'spring', '~> 1.3.6' gem 'spring-commands-rspec', '~> 1.0.4' - gem 'spring-watcher-listen', # otherwise spring polls the filesystem on every 0.2 seconds - github: 'jonleighton/spring-watcher-listen', - ref: '7f6003e14f8f9ca178a5194f210c07f54cfb67ec' + # emits errors, needs more investigation + # gem 'spring-watcher-listen', # otherwise spring polls the filesystem on every 0.2 seconds + # github: 'jonleighton/spring-watcher-listen', + # ref: '7f6003e14f8f9ca178a5194f210c07f54cfb67ec' gem 'guard', '~> 2.12.6' # run tests automatically gem 'guard-rspec', '~> 4.5.2' gem 'guard-rails', '~> 0.7.1' # run EPP server automatically diff --git a/Gemfile.lock b/Gemfile.lock index 74a409af3..836ecc817 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,15 +24,6 @@ GIT hpricot libxml-ruby -GIT - remote: https://github.com/jonleighton/spring-watcher-listen.git - revision: 7f6003e14f8f9ca178a5194f210c07f54cfb67ec - ref: 7f6003e14f8f9ca178a5194f210c07f54cfb67ec - specs: - spring-watcher-listen (1.0.0) - listen (~> 2.7) - spring (~> 1.2) - GIT remote: https://github.com/rubysec/bundler-audit.git revision: f89ef7fae1090bbad825ea76812d56d72b417055 @@ -625,7 +616,6 @@ DEPENDENCIES simpleidn (~> 0.0.5) spring (~> 1.3.6) spring-commands-rspec (~> 1.0.4) - spring-watcher-listen! therubyracer (~> 0.12.2) traceroute (~> 0.5.0) turbolinks (~> 2.5.3) From 7ac6d6b362f45af30cb6214348304c7546582f79 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 3 Jul 2015 15:40:29 +0300 Subject: [PATCH 7/9] Rubocop update #2751 --- .rubocop.yml | 1 + app/models/bank_transaction.rb | 2 +- lib/daemons/daemons | 4 ++-- lib/daemons/que.rb | 9 +++++---- lib/tasks/db.rake | 3 ++- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 16450b1d3..ba9acf010 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -5,6 +5,7 @@ AllCops: - 'Guardfile' # stuff generated by AR and rails - 'db/schema.rb' + - 'db/schema-read-only.rb' - 'db/whois_schema.rb' - 'db/api_log_schema.rb' - 'db/migrate/*' diff --git a/app/models/bank_transaction.rb b/app/models/bank_transaction.rb index a5c8da94c..cd94b0220 100644 --- a/app/models/bank_transaction.rb +++ b/app/models/bank_transaction.rb @@ -3,7 +3,7 @@ class BankTransaction < ActiveRecord::Base belongs_to :bank_statement has_one :account_activity - scope :unbinded, -> { + scope :unbinded, lambda { where('id NOT IN (SELECT bank_transaction_id FROM account_activities where bank_transaction_id IS NOT NULL)') } diff --git a/lib/daemons/daemons b/lib/daemons/daemons index f021cee84..42722e47c 100755 --- a/lib/daemons/daemons +++ b/lib/daemons/daemons @@ -1,5 +1,5 @@ #!/usr/bin/env ruby results = [] -Dir[File.dirname(__FILE__) + "/*_ctl"].each {|f| results << `ruby #{f} #{ARGV.first}`} +Dir[File.dirname(__FILE__) + "/*_ctl"].each { |f| results << `ruby #{f} #{ARGV.first}` } results.delete_if { |result| result.nil? || result.empty? } -puts results.join unless results.empty? \ No newline at end of file +puts results.join unless results.empty? diff --git a/lib/daemons/que.rb b/lib/daemons/que.rb index 7586e65b8..c4ad0abf0 100755 --- a/lib/daemons/que.rb +++ b/lib/daemons/que.rb @@ -3,15 +3,16 @@ ENV["RAILS_ENV"] ||= "production" root = File.expand_path(File.dirname(__FILE__)) -root = File.dirname(root) until File.exists?(File.join(root, 'config')) +root = File.dirname(root) until File.exist?(File.join(root, 'config')) Dir.chdir(root) require File.join(root, "config", "environment") -$running = true +@running = true Signal.trap("TERM") do - $running = false + @running = false end -while($running) do +# rubocop: disable Style/WhileUntilDo +while @running do end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index 9a91ddfcd..3d80d0f3b 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -22,7 +22,8 @@ end namespace :db do namespace :schema do task load: [:environment, :load_config] do - puts 'Only rake db:structure:load is supported and invoked now. Otherwise zonefile generation does not work nor que.' + puts 'Only rake db:structure:load is supported and invoked. ' \ + 'Otherwise zonefile generation does not work nor que.' Rake::Task["db:structure:load"].invoke end end From 0d653a311fce2bb14afadb4dcd9bc91153298733 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 3 Jul 2015 15:44:43 +0300 Subject: [PATCH 8/9] Comment out keyrelay spec at the moment --- spec/epp/keyrelay_spec.rb | 47 ++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/spec/epp/keyrelay_spec.rb b/spec/epp/keyrelay_spec.rb index 75bb253ad..b952ab523 100644 --- a/spec/epp/keyrelay_spec.rb +++ b/spec/epp/keyrelay_spec.rb @@ -121,31 +121,32 @@ describe 'EPP Keyrelay', epp: true do @registrar2.messages.queued.count.should == msg_count end - it 'does not allow both relative and absolute' do - msg_count = @registrar2.messages.queued.count - xml = @epp_xml.keyrelay({ - name: { value: @domain.name }, - keyData: { - flags: { value: '256' }, - protocol: { value: '3' }, - alg: { value: '8' }, - pubKey: { value: 'cmlraXN0aGViZXN0' } - }, - authInfo: { - pw: { value: @domain.auth_info } - }, - expiry: { - relative: { value: 'P1D' }, - absolute: { value: '2014-12-23' } - } - }) + # keyrelay not enabled at the moment + # it 'does not allow both relative and absolute' do + # msg_count = @registrar2.messages.queued.count + # xml = @epp_xml.keyrelay({ + # name: { value: @domain.name }, + # keyData: { + # flags: { value: '256' }, + # protocol: { value: '3' }, + # alg: { value: '8' }, + # pubKey: { value: 'cmlraXN0aGViZXN0' } + # }, + # authInfo: { + # pw: { value: @domain.auth_info } + # }, + # expiry: { + # relative: { value: 'P1D' }, + # absolute: { value: '2014-12-23' } + # } + # }) - response = epp_plain_request(xml, :xml) - response[:msg].should == 'Exactly one parameter required: keyrelay > expiry > relative OR '\ - 'keyrelay > expiry > absolute' + # response = epp_plain_request(xml, :xml) + # response[:msg].should == 'Exactly one parameter required: keyrelay > expiry > relative OR '\ + # 'keyrelay > expiry > absolute' - @registrar2.messages.queued.count.should == msg_count - end + # @registrar2.messages.queued.count.should == msg_count + # end it 'saves legal document with keyrelay' do xml = @epp_xml.keyrelay({ From 41c9f998d384e0f10663206588ed8f0991e289c6 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 3 Jul 2015 15:48:38 +0300 Subject: [PATCH 9/9] Update travis about structured #2751 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2dd3a99b2..1025db403 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,6 @@ before_script: - cp config/database-travis.yml config/database.yml - RAILS_ENV=test bundle exec rake db:all:drop - RAILS_ENV=test bundle exec rake db:all:setup - - RAILS_ENV=test bundle exec rake zonefile:replace_procedure script: - RAILS_ENV=test bundle exec rake cache: bundler