Added que init script #2724

CHANGELOG.md

@@ -1,8 +1,13 @@
+01.07.2015
+
+* Added que init script example at doc/que directory, please setup que accornding to doc/que/README.md
+
 26.06.2015
 
 * Added new relic license key ta application-example.yml, please update application.yml
 
 22.06.2015
+
 * Update zonefile
 
 16.06.2015

Gemfile

@@ -88,6 +88,7 @@ gem 'uuidtools', '~> 2.1.4' # For unique IDs (used by the epp gem)
 # que
 gem 'que',           '~> 0.10.0'
 gem 'que-web',       '~> 0.4.0'
+gem 'daemons-rails', '~> 1.2.1'
 
 # for importing legacy db
 gem 'activerecord-import', '~> 0.7.0' # for inserting dummy data

Gemfile.lock

@@ -161,6 +161,10 @@ GEM
     crack (0.4.2)
       safe_yaml (~> 1.0.0)
     currencies (0.4.2)
+    daemons (1.2.3)
+    daemons-rails (1.2.1)
+      daemons
+      multi_json (~> 1.0)
     database_cleaner (1.4.1)
     debug_inspector (0.0.2)
     deep_cloneable (2.1.1)
@@ -192,7 +196,7 @@ GEM
     faker (1.4.3)
       i18n (~> 0.5)
     fastercsv (1.5.5)
-    ffi (1.9.8)
+    ffi (1.9.9)
     figaro (1.1.1)
       thor (~> 0.14)
     flay (2.4.0)
@@ -214,9 +218,9 @@ GEM
       rack-accept
       rack-mount
       virtus (>= 1.0.0)
-    guard (2.12.6)
+    guard (2.12.7)
       formatador (>= 0.2.4)
-      listen (~> 2.7)
+      listen (>= 2.7, <= 4.0)
       lumberjack (~> 1.0)
       nenv (~> 0.1)
       notiffany (~> 0.0)
@@ -255,7 +259,7 @@ GEM
       haml (~> 4.0.0)
       nokogiri (~> 1.6.0)
       ruby_parser (~> 3.5)
-    html5_validators (1.2.0)
+    html5_validators (1.2.1)
     httpclient (2.6.0.1)
     httpi (2.4.1)
       rack
@@ -283,7 +287,7 @@ GEM
     kgio (2.9.3)
     launchy (2.4.3)
       addressable (~> 2.3)
-    libv8 (3.16.14.7)
+    libv8 (3.16.14.9)
     libxml-ruby (2.8.0)
     listen (2.10.1)
       celluloid (~> 0.16.0)
@@ -494,7 +498,7 @@ GEM
       sprockets (>= 2.8, < 4.0)
     sys-uname (0.9.0)
       ffi (>= 1.0.0)
-    terminal-table (1.4.5)
+    terminal-table (1.5.2)
     therubyracer (0.12.2)
       libv8 (~> 3.16.14.0)
       ref
@@ -562,6 +566,7 @@ DEPENDENCIES
   coderay (~> 1.1.0)
   coffee-rails (~> 4.1.0)
   countries (~> 0.11.4)
+  daemons-rails (~> 1.2.1)
   data_migrate!
   database_cleaner (~> 1.4.1)
   deep_cloneable (~> 2.1.1)

README.md

@@ -398,7 +398,7 @@ sudo apt-get install libxext-dev libxrender1 fontconfig
 ### Deployment
 
 * [Application build and update](/doc/application_build_doc.md)
-
+* [Registry que server](/doc/que/README.md)
 
 ### Autotesting
 

bin/delayed_job

@@ -1,5 +0,0 @@
-#!/usr/bin/env ruby
-
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
-require 'delayed/command'
-Delayed::Command.new(ARGV).daemonize

config/daemons.yml

@@ -0,0 +1,7 @@
+dir_mode: script
+dir: ../../log
+multiple: true
+backtrace: true
+monitor: true
+ontop: false
+app_name: 'que'

config/environments/alpha.rb

@@ -81,3 +81,8 @@ Rails.application.configure do
   # Do not dump schema after migrations.
   config.active_record.dump_schema_after_migration = false
 end
+
+# In off mode, queueing a job will simply insert it into the database - 
+# the current process will make no effort to run it. 
+# You should use this if you want to use a dedicated process to work tasks
+Que.mode = :off 

config/environments/development.rb

@@ -47,3 +47,8 @@ Rails.application.configure do
     Bullet.unused_eager_loading_enable = false
   end
 end
+
+# In this mode, any jobs you queue will be run in the same thread, synchronously
+# (that is, MyJob.enqueue runs the job and won't return until it's completed).
+# This makes your application's behavior easier to test
+Que.mode = :sync 

config/environments/production.rb

@@ -81,3 +81,8 @@ Rails.application.configure do
   # Do not dump schema after migrations.
   config.active_record.dump_schema_after_migration = false
 end
+
+# In off mode, queueing a job will simply insert it into the database - 
+# the current process will make no effort to run it. 
+# You should use this if you want to use a dedicated process to work tasks
+Que.mode = :off 

config/environments/staging.rb

@@ -84,3 +84,8 @@ Rails.application.configure do
   # TODO: Change this:
   config.action_mailer.default_url_options = { host: 'localhost', port: 3000 }
 end
+
+# In off mode, queueing a job will simply insert it into the database - 
+# the current process will make no effort to run it. 
+# You should use this if you want to use a dedicated process to work tasks
+Que.mode = :off 

config/environments/test.rb

@@ -57,3 +57,8 @@ Rails.application.configure do
     Bullet.add_whitelist type: :n_plus_one_query, class_name: 'Contact', association: :registrar
   end
 end
+
+# In this mode, any jobs you queue will be run in the same thread, synchronously
+# (that is, MyJob.enqueue runs the job and won't return until it's completed).
+# This makes your application's behavior easier to test
+Que.mode = :sync 

config/initializers/que.rb

@@ -1 +0,0 @@
-Que.mode = :off

db/structure.sql

@@ -25,167 +25,6 @@ COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
 
 SET search_path = public, pg_catalog;
 
---
--- Name: generate_zonefile(character varying); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION generate_zonefile(i_origin character varying) RETURNS text
-    LANGUAGE plpgsql
-    AS $_$
-      DECLARE
-        zone_header text := concat('$ORIGIN ', i_origin, '.');
-        serial_num varchar;
-        include_filter varchar := '';
-        exclude_filter varchar := '';
-        tmp_var text;
-        ret text;
-      BEGIN
-        -- define filters
-        include_filter = '%.' || i_origin;
-
-        -- for %.%.%
-        IF i_origin ~ '\.' THEN
-          exclude_filter := '';
-        -- for %.%
-        ELSE
-          exclude_filter := '%.%.' || i_origin;
-        END IF;
-
-        SELECT ROUND(extract(epoch from now() at time zone 'utc')) INTO serial_num;
-
-        -- zonefile header
-        SELECT concat(
-          format('%-10s', '$ORIGIN .'), chr(10),
-          format('%-10s', '$TTL'), zf.ttl, chr(10), chr(10),
-          format('%-10s', i_origin || '.'), 'IN SOA ', zf.master_nameserver, '. ', zf.email, '. (', chr(10),
-          format('%-17s', ''), format('%-12s', serial_num), '; serial number', chr(10),
-          format('%-17s', ''), format('%-12s', zf.refresh), '; refresh, seconds', chr(10),
-          format('%-17s', ''), format('%-12s', zf.retry), '; retry, seconds', chr(10),
-          format('%-17s', ''), format('%-12s', zf.expire), '; expire, seconds', chr(10),
-          format('%-17s', ''), format('%-12s', zf.minimum_ttl), '; minimum TTL, seconds', chr(10),
-          format('%-17s', ''), ')'
-        ) FROM zonefile_settings zf WHERE i_origin = zf.origin INTO tmp_var;
-
-        ret = concat(tmp_var, chr(10), chr(10));
-
-        -- ns records
-        SELECT array_to_string(
-          array(
-            SELECT concat(d.name_puny, '. IN NS ', ns.hostname, '.')
-            FROM domains d
-            JOIN nameservers ns ON ns.domain_id = d.id
-            WHERE d.name LIKE include_filter AND d.name NOT LIKE exclude_filter
-            ORDER BY d.name
-          ),
-          chr(10)
-        ) INTO tmp_var;
-
-        ret := concat(ret, '; Zone NS Records', chr(10), tmp_var, chr(10), chr(10));
-
-        -- a glue records for origin nameservers
-        SELECT array_to_string(
-          array(
-            SELECT concat(ns.hostname, '. IN A ', ns.ipv4)
-            FROM nameservers ns
-            JOIN domains d ON d.id = ns.domain_id
-            WHERE d.name = i_origin
-            AND ns.hostname LIKE '%.' || d.name
-            AND ns.ipv4 IS NOT NULL AND ns.ipv4 <> ''
-          ), chr(10)
-        ) INTO tmp_var;
-
-        ret := concat(ret, '; Zone A Records', chr(10), tmp_var);
-
-        -- a glue records for other nameservers
-        SELECT array_to_string(
-          array(
-            SELECT concat(ns.hostname, '. IN A ', ns.ipv4)
-            FROM nameservers ns
-            JOIN domains d ON d.id = ns.domain_id
-            WHERE d.name LIKE include_filter AND d.name NOT LIKE exclude_filter
-            AND ns.hostname LIKE '%.' || d.name
-            AND d.name <> i_origin
-            AND ns.ipv4 IS NOT NULL AND ns.ipv4 <> ''
-            AND NOT EXISTS ( -- filter out glue records that already appeared in origin glue recrods
-              SELECT 1 FROM nameservers nsi
-              JOIN domains di ON nsi.domain_id = di.id
-              WHERE di.name = i_origin
-              AND nsi.hostname = ns.hostname
-            )
-          ), chr(10)
-        ) INTO tmp_var;
-
-        -- TODO This is a possible subtitition to the previous query, stress testing is needed to see which is faster
-
-        -- SELECT ns.*
-        -- FROM nameservers ns
-        -- JOIN domains d ON d.id = ns.domain_id
-        -- WHERE d.name LIKE '%ee' AND d.name NOT LIKE '%pri.ee'
-        -- AND ns.hostname LIKE '%.' || d.name
-        -- AND d.name <> 'ee'
-        -- AND ns.ipv4 IS NOT NULL AND ns.ipv4 <> ''
-        -- AND ns.hostname NOT IN (
-        --   SELECT ns.hostname FROM domains d JOIN nameservers ns ON d.id = ns.domain_id WHERE d.name = 'ee'
-        -- )
-
-        ret := concat(ret, chr(10), tmp_var, chr(10), chr(10));
-
-        -- aaaa glue records for origin nameservers
-        SELECT array_to_string(
-          array(
-            SELECT concat(ns.hostname, '. IN AAAA ', ns.ipv6)
-            FROM nameservers ns
-            JOIN domains d ON d.id = ns.domain_id
-            WHERE d.name = i_origin
-            AND ns.hostname LIKE '%.' || d.name
-            AND ns.ipv6 IS NOT NULL AND ns.ipv6 <> ''
-          ), chr(10)
-        ) INTO tmp_var;
-
-        ret := concat(ret, '; Zone AAAA Records', chr(10), tmp_var);
-
-        -- aaaa glue records for other nameservers
-        SELECT array_to_string(
-          array(
-            SELECT concat(ns.hostname, '. IN AAAA ', ns.ipv6)
-            FROM nameservers ns
-            JOIN domains d ON d.id = ns.domain_id
-            WHERE d.name LIKE include_filter AND d.name NOT LIKE exclude_filter
-            AND ns.hostname LIKE '%.' || d.name
-            AND d.name <> i_origin
-            AND ns.ipv6 IS NOT NULL AND ns.ipv6 <> ''
-            AND NOT EXISTS ( -- filter out glue records that already appeared in origin glue recrods
-              SELECT 1 FROM nameservers nsi
-              JOIN domains di ON nsi.domain_id = di.id
-              WHERE di.name = i_origin
-              AND nsi.hostname = ns.hostname
-            )
-          ), chr(10)
-        ) INTO tmp_var;
-
-        ret := concat(ret, chr(10), tmp_var, chr(10), chr(10));
-
-        -- ds records
-        SELECT array_to_string(
-          array(
-            SELECT concat(
-              d.name_puny, '. IN DS ', dk.ds_key_tag, ' ',
-              dk.ds_alg, ' ', dk.ds_digest_type, ' ( ', dk.ds_digest, ' )'
-            )
-            FROM domains d
-            JOIN dnskeys dk ON dk.domain_id = d.id
-            WHERE d.name LIKE include_filter AND d.name NOT LIKE exclude_filter AND dk.flags = 257
-            ),
-          chr(10)
-        ) INTO tmp_var;
-
-        ret := concat(ret, '; Zone DS Records', chr(10), tmp_var, chr(10));
-
-        RETURN ret;
-      END;
-      $_$;
-
-
 SET default_tablespace = '';
 
 SET default_with_oids = false;
@@ -472,39 +311,6 @@ CREATE SEQUENCE banklink_transactions_id_seq
 ALTER SEQUENCE banklink_transactions_id_seq OWNED BY banklink_transactions.id;
 
 
---
--- Name: blocked_domains; Type: TABLE; Schema: public; Owner: -; Tablespace: 
---
-
-CREATE TABLE blocked_domains (
-    id integer NOT NULL,
-    names character varying[],
-    created_at timestamp without time zone,
-    updated_at timestamp without time zone,
-    creator_str character varying,
-    updator_str character varying
-);
-
-
---
--- Name: blocked_domains_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE blocked_domains_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: blocked_domains_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE blocked_domains_id_seq OWNED BY blocked_domains.id;
-
-
 --
 -- Name: cached_nameservers; Type: TABLE; Schema: public; Owner: -; Tablespace: 
 --
@@ -1391,43 +1197,6 @@ CREATE SEQUENCE log_bank_transactions_id_seq
 ALTER SEQUENCE log_bank_transactions_id_seq OWNED BY log_bank_transactions.id;
 
 
---
--- Name: log_blocked_domains; Type: TABLE; Schema: public; Owner: -; Tablespace: 
---
-
-CREATE TABLE log_blocked_domains (
-    id integer NOT NULL,
-    item_type character varying NOT NULL,
-    item_id integer NOT NULL,
-    event character varying NOT NULL,
-    whodunnit character varying,
-    object json,
-    object_changes json,
-    created_at timestamp without time zone,
-    session character varying,
-    children json
-);
-
-
---
--- Name: log_blocked_domains_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE log_blocked_domains_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: log_blocked_domains_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE log_blocked_domains_id_seq OWNED BY log_blocked_domains.id;
-
-
 --
 -- Name: log_certificates; Type: TABLE; Schema: public; Owner: -; Tablespace: 
 --
@@ -2404,7 +2173,7 @@ ALTER SEQUENCE pricelists_id_seq OWNED BY pricelists.id;
 
 CREATE TABLE que_jobs (
     priority smallint DEFAULT 100 NOT NULL,
-    run_at timestamp without time zone DEFAULT '2015-06-30 14:16:50.905537'::timestamp without time zone NOT NULL,
+    run_at timestamp without time zone DEFAULT '2015-06-29 12:38:58.258132'::timestamp without time zone NOT NULL,
     job_id bigint DEFAULT 0 NOT NULL,
     job_class text NOT NULL,
     args json DEFAULT '[]'::json NOT NULL,
@@ -2819,13 +2588,6 @@ ALTER TABLE ONLY bank_transactions ALTER COLUMN id SET DEFAULT nextval('bank_tra
 ALTER TABLE ONLY banklink_transactions ALTER COLUMN id SET DEFAULT nextval('banklink_transactions_id_seq'::regclass);
 
 
---
--- Name: id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY blocked_domains ALTER COLUMN id SET DEFAULT nextval('blocked_domains_id_seq'::regclass);
-
-
 --
 -- Name: id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -2980,13 +2742,6 @@ ALTER TABLE ONLY log_bank_statements ALTER COLUMN id SET DEFAULT nextval('log_ba
 ALTER TABLE ONLY log_bank_transactions ALTER COLUMN id SET DEFAULT nextval('log_bank_transactions_id_seq'::regclass);
 
 
---
--- Name: id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY log_blocked_domains ALTER COLUMN id SET DEFAULT nextval('log_blocked_domains_id_seq'::regclass);
-
-
 --
 -- Name: id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -3288,14 +3043,6 @@ ALTER TABLE ONLY banklink_transactions
     ADD CONSTRAINT banklink_transactions_pkey PRIMARY KEY (id);
 
 
---
--- Name: blocked_domains_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: 
---
-
-ALTER TABLE ONLY blocked_domains
-    ADD CONSTRAINT blocked_domains_pkey PRIMARY KEY (id);
-
-
 --
 -- Name: certificates_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: 
 --
@@ -3472,14 +3219,6 @@ ALTER TABLE ONLY log_bank_transactions
     ADD CONSTRAINT log_bank_transactions_pkey PRIMARY KEY (id);
 
 
---
--- Name: log_blocked_domains_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: 
---
-
-ALTER TABLE ONLY log_blocked_domains
-    ADD CONSTRAINT log_blocked_domains_pkey PRIMARY KEY (id);
-
-
 --
 -- Name: log_certificates_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: 
 --
@@ -4082,20 +3821,6 @@ CREATE INDEX index_log_bank_transactions_on_item_type_and_item_id ON log_bank_tr
 CREATE INDEX index_log_bank_transactions_on_whodunnit ON log_bank_transactions USING btree (whodunnit);
 
 
---
--- Name: index_log_blocked_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: 
---
-
-CREATE INDEX index_log_blocked_domains_on_item_type_and_item_id ON log_blocked_domains USING btree (item_type, item_id);
-
-
---
--- Name: index_log_blocked_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: 
---
-
-CREATE INDEX index_log_blocked_domains_on_whodunnit ON log_blocked_domains USING btree (whodunnit);
-
-
 --
 -- Name: index_log_certificates_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: 
 --
@@ -4808,5 +4533,3 @@ INSERT INTO schema_migrations (version) VALUES ('20150611124920');
 
 INSERT INTO schema_migrations (version) VALUES ('20150612123111');
 
-INSERT INTO schema_migrations (version) VALUES ('20150701074344');
-

doc/que/README.md

@@ -0,0 +1,46 @@
+Registry que server
+===================
+
+Que server responsibilites:
+
+*  handle write type of communication between Registrant and Registry
+*  handle future jobs for Registry
+*  handle heavy load jobs for Registry
+
+Installation
+------------
+
+Que can deploy either separaetly or along to Registry server depends on real load situation.
+In both serarious que requires working Registry deployment and full access to Registry databases. 
+
+Installation at deployed server:
+
+    cd /home/registry/registry/current
+    sudo cp doc/que/que-init-example /etc/init.d/que # and edit it
+    sudo chmod +x /etc/init.d/que
+    sudo /etc/init.d/que          # for help and other commands
+    sudo /etc/init.d/que start    # for manual start 
+    sudo update-rc.d que defaults # for start in server boot
+
+# Debugging
+
+You can run que manually as well:
+
+    cd /home/registry/registry/current
+
+For all manual que tasks:
+
+    RAILS_ENV=production bundle exec rake -T que    # for all que tasks for manual control
+    rake que:clear           # Clear Que's job table
+    rake que:drop            # Drop Que's job table
+    rake que:migrate         # Migrate Que's job table to the most recent version (creating it if it doesn't exist)
+    rake que:work            # Process Que's jobs using a worker pool
+
+For all que daemon tasks what inist script uses
+
+    RAILS_ENV=production bundle exec rake -T daemon # for all que daemon tasks what init script uses
+    rake daemon:que          # Start que script
+    rake daemon:que:restart  # Restart que daemon
+    rake daemon:que:start    # Start que daemon
+    rake daemon:que:status   # Status que daemon
+    rake daemon:que:stop     # Stop que daemon

doc/que/que-init-example

@@ -0,0 +1,51 @@
+#!/bin/bash
+### BEGIN INIT INFO
+# Provides:          Registry que server
+# Required-Start:    $all
+# Required-Stop:     $network $local_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Enable Registry que server
+### END INIT INFO
+
+set -u
+set -e
+
+#
+# Change these to match your server:
+#
+# Make sure that all paths are correct.
+#
+APP_HOME="/home/registry/registry"
+APP_ROOT="$APP_HOME/current"
+QUE_USER=registry # or use some other unprivileged system user
+RAILS_ENV=production
+RUBY_BUNDLE_PATH=/home/$QUE_USER/.rbenv/shims/bundle         
+QUE_INSTANCES=1 # or as many really needed based real load
+
+cd $APP_ROOT || exit 1
+
+case ${1-help} in
+status)
+        cd $APP_ROOT && RAILS_ENV=$RAILS_ENV $RUBY_BUNDLE_PATH exec rake daemon:que:status         
+        ;;
+start)
+        echo "$1 que monitor and server"
+        for i in `seq 1 $QUE_INSTANCES`; do
+          cd $APP_ROOT && RAILS_ENV=$RAILS_ENV $RUBY_BUNDLE_PATH exec rake daemon:que:start         
+          echo '.'
+        done
+        ;;
+stop)
+        echo "$1 que monitor and server"
+        cd $APP_ROOT && RAILS_ENV=$RAILS_ENV $RUBY_BUNDLE_PATH exec rake daemon:que:stop         
+        ;;
+restart)
+        echo "$1 que monitor and server"
+        cd $APP_ROOT && RAILS_ENV=$RAILS_ENV $RUBY_BUNDLE_PATH exec rake daemon:que:restart         
+        ;;
+*)
+        echo >&2 "Usage: $0 <status|start|stop|restart>"
+        exit 1
+        ;;
+esac

lib/daemons/daemons

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+results = []
+Dir[File.dirname(__FILE__) + "/*_ctl"].each {|f| results << `ruby #{f} #{ARGV.first}`}
+results.delete_if { |result| result.nil? || result.empty? }
+puts results.join unless results.empty?

lib/daemons/que.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+ENV["RAILS_ENV"] ||= "production"
+
+root = File.expand_path(File.dirname(__FILE__))
+root = File.dirname(root) until File.exists?(File.join(root, 'config'))
+Dir.chdir(root)
+
+require File.join(root, "config", "environment")
+
+$running = true
+Signal.trap("TERM") do 
+  $running = false
+end
+
+while($running) do
+end

lib/daemons/que_ctl

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'daemons/rails/config'
+
+config = Daemons::Rails::Config.for_controller(File.expand_path(__FILE__))
+Daemons::Rails.run config[:script], config.to_hash