Merge branch 'master' into 347-update_paperclip

Conflicts:
	Gemfile.lock

Gemfile

@@ -18,7 +18,9 @@ gem 'paperclip', '~> 4.1.0'
 gem 'resque', '~> 1.24'
 gem 'resque-status', '~> 0.3.3'
 gem 'resque_mailer', '~> 2.2'
-gem 'perform_later', '~> 1.3.0' # should be after resque_mailer
+gem 'resque-scheduler'
+# gem 'perform_later', '~> 1.3.0' # should be after resque_mailer
+gem 'perform_later', git: 'git://github.com/KensoDev/perform_later.git' # should be after resque_mailer
 gem 'russian', '~> 0.6.0'
 gem 'highline', '~> 1.6.11'
 gem 'state_machine'

Gemfile.lock

@@ -1,3 +1,13 @@
+GIT
+  remote: git://github.com/KensoDev/perform_later.git
+  revision: afd40794aa7101c8e73e9f105a064dd9bec47e31
+  specs:
+    perform_later (1.3.0)
+      rails (>= 3.0)
+      redis
+      resque (> 1.25)
+      resque-scheduler
+
 GIT
   remote: git://github.com/rosa-abf/grack.git
   revision: 020be3fef3fb308b9d214252522aa5945bf6584a
@@ -261,10 +271,6 @@ GEM
       activesupport (>= 3.0.0)
       cocaine (~> 0.5.3)
       mime-types
-    perform_later (1.3.0)
-      rails (~> 3.0)
-      redis
-      resque
     pg (0.14.1)
     polyglot (0.3.3)
     posix-spawn (0.3.8)
@@ -337,6 +343,10 @@ GEM
       redis-namespace (~> 1.2)
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
+    resque-scheduler (2.5.3)
+      redis (~> 3.0.4)
+      resque (~> 1.25.1)
+      rufus-scheduler (~> 2.0.24)
     resque-status (0.3.3)
       redisk (>= 0.2.1)
       resque (~> 1.19)
@@ -366,6 +376,8 @@ GEM
     rubypython (0.5.3)
       blankslate (>= 2.1.2.3)
       ffi (~> 1.0.7)
+    rufus-scheduler (2.0.24)
+      tzinfo (>= 0.3.22)
     russian (0.6.0)
       i18n (>= 0.5.0)
     rvm-capistrano (1.5.1)
@@ -495,7 +507,7 @@ DEPENDENCIES
   omniauth-github
   omniauth-google-oauth2
   paperclip (~> 4.1.0)
-  perform_later (~> 1.3.0)
+  perform_later!
   pg (~> 0.14.0)
   puma
   rack-throttle
@@ -508,6 +520,7 @@ DEPENDENCIES
   redhillonrails_core!
   redis-rails
   resque (~> 1.24)
+  resque-scheduler
   resque-status (~> 0.3.3)
   resque_mailer (~> 2.2)
   rest-client (~> 1.6.6)

Rakefile

@@ -4,6 +4,7 @@
 require File.expand_path('../config/application', __FILE__)
 require 'rake'
 require 'resque/tasks'
+require 'resque_scheduler/tasks'
 
 # This fixes connection fail with Postgres server on new fork:
 task "resque:setup" => :environment do

app/controllers/api/v1/jobs_controller.rb

@@ -16,6 +16,7 @@ class Api::V1::JobsController < Api::V1::BaseController
     if current_user.system?
       if task = (Resque.pop('rpm_worker_default') || Resque.pop('rpm_worker'))
         @build_list = BuildList.where(id: task['args'][0]['id']).first
+        @build_list.delayed_add_job_to_abf_worker_queue
       end
     end
 

app/models/build_list.rb

@@ -152,8 +152,9 @@ class BuildList < ActiveRecord::Base
       end
     end
 
-    after_transition on: :place_build, do: :add_job_to_abf_worker_queue,
-      :if  => lambda { |build_list| build_list.external_nodes.blank? }
+    after_transition(on: :place_build) do |build_list, transition|
+      build_list.add_job_to_abf_worker_queue if build_list.external_nodes.blank?
+    end
     after_transition on: :published,
       do: [:set_version_and_tag, :actualize_packages]
     after_transition on: :publish, do: :set_publisher
@@ -255,7 +256,7 @@ class BuildList < ActiveRecord::Base
   end
 
   later :publish, queue: :clone_build
-
+  later :add_job_to_abf_worker_queue, queue: :clone_build
 
   HUMAN_CONTAINER_STATUSES = { WAITING_FOR_RESPONSE => :waiting_for_publish,
                                BUILD_PUBLISHED => :container_published,
@@ -522,6 +523,11 @@ class BuildList < ActiveRecord::Base
     )
   end
 
+  def delayed_add_job_to_abf_worker_queue(*args)
+    restart_job if status == BUILD_PENDING
+  end
+  later :delayed_add_job_to_abf_worker_queue, delay: 60, queue: :clone_build
+
   protected
 
   def create_container

config/initializers/resque.rb

@@ -1,5 +1,7 @@
 require 'resque/server'
 require 'resque/status_server'
+require 'resque_scheduler'
+require 'resque_scheduler/server'
 
 ::ResqueServer = ::Resque::Server # need for CanCan
 

config/puma/production.rb

@@ -4,7 +4,7 @@ state_path File.join(base_path, 'shared', 'pids', 'puma.state')
 bind 'unix:///tmp/rosa_build_unicorn.sock'
 
 environment ENV['RAILS_ENV'] || 'production'
-threads *(ENV['PUMA_THREADS'] || '1,4').split(',')
+threads *(ENV['PUMA_THREADS'] || '1,3').split(',')
 workers ENV['PUMA_WORKERS'] || 7
 
 
@@ -14,7 +14,14 @@ on_worker_boot do
   if defined?(ActiveRecord::Base)
     ActiveSupport.on_load(:active_record) do
       ActiveRecord::Base.connection.disconnect! rescue ActiveRecord::ConnectionNotEstablished
-      ActiveRecord::Base.establish_connection
+
+      config = Rails.application.config.database_configuration[Rails.env]
+      config['reaping_frequency'] = ENV['DB_REAP_FREQ'] || 10 # seconds
+      config['pool']              = ENV['DB_POOL']      || 7
+
+      ActiveRecord::Base.establish_connection(config)
+
+      Rails.logger.info "Connected to PG. Connection pool size #{config['pool']}, reaping frequency #{config['reaping_frequency']}"
     end
     # QC::Conn.connect
     Rails.logger.info('Connected to PG')
@@ -24,4 +31,4 @@ on_worker_boot do
   # Rails.logger.info('Connected to Redis')
 end
 
-activate_control_app 'unix:///tmp/rosa_build_pumactl.sock'
+activate_control_app 'unix:///tmp/rosa_build_pumactl.sock'

lib/abf_worker/model_helper.rb

@@ -29,8 +29,7 @@ module AbfWorker::ModelHelper
   end
 
   def restart_job
-    redis = Resque.redis
-    redis.lpush "queue:#{worker_queue_with_priority}",
+    Resque.redis.lpush "queue:#{worker_queue_with_priority}",
       Resque.encode({'class' => worker_queue_class, 'args' => [abf_worker_args]})
   end
 

lib/recipes/resque.rb

@@ -36,5 +36,50 @@ Capistrano::Configuration.instance(:must_exist).load do
       ].join(',')
       run "cd #{fetch :current_path} && COUNT=#{workers_count} QUEUE=#{queue} #{rails_env} BACKGROUND=yes bundle exec rake resque:workers"
     end
+
+    def remote_file_exists?(full_path)
+      'true' ==  capture("if [ -e #{full_path} ]; then echo 'true'; fi").strip
+    end
+
+    namespace :scheduler do
+
+      desc "See current scheduler status"
+      task :status do
+        pid = "#{fetch :current_path}/tmp/pids/scheduler.pid"
+        if remote_file_exists?(pid)
+          info capture(:ps, "-f -p $(cat #{pid}) | sed -n 2p")
+        end
+      end
+
+      desc "Starts resque scheduler with default configs"
+      task :start do
+        start_scheduler
+      end
+
+      desc "Stops resque scheduler"
+      task :stop do
+        stop_scheduler
+      end
+
+      task :restart do
+        stop_scheduler
+        start_scheduler
+      end
+
+      def start_scheduler
+        pid = "#{fetch :current_path}/tmp/pids/scheduler.pid"
+        run "cd #{fetch :current_path} && #{rails_env} PIDFILE=#{pid} BACKGROUND=yes VERBOSE=1 MUTE=1 bundle exec rake resque:scheduler"
+      end
+
+      def stop_scheduler
+        pid = "#{fetch :current_path}/tmp/pids/scheduler.pid"
+        if remote_file_exists?(pid)
+          run "cd #{fetch :current_path} && kill -s QUIT $(cat #{pid}); rm #{pid}"
+        end
+      end
+
+    end
+
   end
+
 end