From 8402728f4d096ec238c1b796a409cae428c3e14e Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Mon, 10 Feb 2014 19:17:09 +0400 Subject: [PATCH 1/7] Fixed: loss build_lists tasks --- app/models/build_list.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/build_list.rb b/app/models/build_list.rb index afaee0d10..d00131977 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -152,8 +152,9 @@ class BuildList < ActiveRecord::Base end end - after_transition on: :place_build, do: :add_job_to_abf_worker_queue, - :if => lambda { |build_list| build_list.external_nodes.blank? } + after_transition(on: :place_build) do |build_list, transition| + build_list.add_job_to_abf_worker_queue if build_list.external_nodes.blank? + end after_transition on: :published, do: [:set_version_and_tag, :actualize_packages] after_transition on: :publish, do: :set_publisher @@ -255,6 +256,7 @@ class BuildList < ActiveRecord::Base end later :publish, queue: :clone_build + later :add_job_to_abf_worker_queue, queue: :clone_build HUMAN_CONTAINER_STATUSES = { WAITING_FOR_RESPONSE => :waiting_for_publish, From e889f2366065fdca3a0f09c3364717fa8fa07c69 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Mon, 10 Feb 2014 23:36:10 +0400 Subject: [PATCH 2/7] Configure resque-scheduler gem --- Gemfile | 4 ++- Gemfile.lock | 25 +++++++++++---- Rakefile | 1 + app/controllers/api/v1/jobs_controller.rb | 1 + app/models/build_list.rb | 6 +++- config/initializers/resque.rb | 2 ++ lib/recipes/resque.rb | 39 +++++++++++++++++++++++ 7 files changed, 70 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index f4848b1d3..fe6643670 100644 --- a/Gemfile +++ b/Gemfile @@ -18,7 +18,9 @@ gem 'paperclip', '~> 3.3.1' gem 'resque', '~> 1.24' gem 'resque-status', '~> 0.3.3' gem 'resque_mailer', '~> 2.2' -gem 'perform_later', '~> 1.3.0' # should be after resque_mailer +gem 'resque-scheduler' +# gem 'perform_later', '~> 1.3.0' # should be after resque_mailer +gem 'perform_later', git: 'git://github.com/KensoDev/perform_later.git' # should be after resque_mailer gem 'russian', '~> 0.6.0' gem 'highline', '~> 1.6.11' gem 'state_machine' diff --git a/Gemfile.lock b/Gemfile.lock index 1217a7b9e..4fb742a89 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,13 @@ +GIT + remote: git://github.com/KensoDev/perform_later.git + revision: afd40794aa7101c8e73e9f105a064dd9bec47e31 + specs: + perform_later (1.3.0) + rails (>= 3.0) + redis + resque (> 1.25) + resque-scheduler + GIT remote: git://github.com/rosa-abf/grack.git revision: 020be3fef3fb308b9d214252522aa5945bf6584a @@ -259,10 +269,6 @@ GEM activesupport (>= 3.0.0) cocaine (~> 0.4.0) mime-types - perform_later (1.3.0) - rails (~> 3.0) - redis - resque pg (0.14.1) polyglot (0.3.3) posix-spawn (0.3.8) @@ -307,7 +313,7 @@ GEM rdoc (3.12.2) json (~> 1.4) redcarpet (2.2.2) - redis (3.0.6) + redis (3.0.7) redis-actionpack (3.2.4) actionpack (~> 3.2.0) redis-rack (~> 1.4.4) @@ -335,6 +341,10 @@ GEM redis-namespace (~> 1.2) sinatra (>= 0.9.2) vegas (~> 0.1.2) + resque-scheduler (2.5.1) + redis (>= 3.0.0) + resque (~> 1.25) + rufus-scheduler (~> 2.0) resque-status (0.3.3) redisk (>= 0.2.1) resque (~> 1.19) @@ -364,6 +374,8 @@ GEM rubypython (0.5.3) blankslate (>= 2.1.2.3) ffi (~> 1.0.7) + rufus-scheduler (2.0.24) + tzinfo (>= 0.3.22) russian (0.6.0) i18n (>= 0.5.0) rvm-capistrano (1.5.1) @@ -493,7 +505,7 @@ DEPENDENCIES omniauth-github omniauth-google-oauth2 paperclip (~> 3.3.1) - perform_later (~> 1.3.0) + perform_later! pg (~> 0.14.0) puma rack-throttle @@ -506,6 +518,7 @@ DEPENDENCIES redhillonrails_core! redis-rails resque (~> 1.24) + resque-scheduler resque-status (~> 0.3.3) resque_mailer (~> 2.2) rest-client (~> 1.6.6) diff --git a/Rakefile b/Rakefile index ae5e4f293..31c005551 100644 --- a/Rakefile +++ b/Rakefile @@ -4,6 +4,7 @@ require File.expand_path('../config/application', __FILE__) require 'rake' require 'resque/tasks' +require 'resque_scheduler/tasks' # This fixes connection fail with Postgres server on new fork: task "resque:setup" => :environment do diff --git a/app/controllers/api/v1/jobs_controller.rb b/app/controllers/api/v1/jobs_controller.rb index f3a9ef152..cd91a7469 100644 --- a/app/controllers/api/v1/jobs_controller.rb +++ b/app/controllers/api/v1/jobs_controller.rb @@ -16,6 +16,7 @@ class Api::V1::JobsController < Api::V1::BaseController if current_user.system? if task = (Resque.pop('rpm_worker_default') || Resque.pop('rpm_worker')) @build_list = BuildList.where(id: task['args'][0]['id']).first + @build_list.delayed_add_job_to_abf_worker_queue end end diff --git a/app/models/build_list.rb b/app/models/build_list.rb index d00131977..f1242996f 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -258,7 +258,6 @@ class BuildList < ActiveRecord::Base later :publish, queue: :clone_build later :add_job_to_abf_worker_queue, queue: :clone_build - HUMAN_CONTAINER_STATUSES = { WAITING_FOR_RESPONSE => :waiting_for_publish, BUILD_PUBLISHED => :container_published, BUILD_PUBLISH => :container_publish, @@ -524,6 +523,11 @@ class BuildList < ActiveRecord::Base ) end + def delayed_add_job_to_abf_worker_queue + now_add_job_to_abf_worker_queue if status == BUILD_PENDING + end + later :delayed_add_job_to_abf_worker_queue, delay: 60, queue: :clone_build + protected def create_container diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb index f3e8d36bb..f2c69e5c3 100644 --- a/config/initializers/resque.rb +++ b/config/initializers/resque.rb @@ -1,5 +1,7 @@ require 'resque/server' require 'resque/status_server' +require 'resque_scheduler' +require 'resque_scheduler/server' ::ResqueServer = ::Resque::Server # need for CanCan diff --git a/lib/recipes/resque.rb b/lib/recipes/resque.rb index 4e2952f54..97f03ca7d 100644 --- a/lib/recipes/resque.rb +++ b/lib/recipes/resque.rb @@ -3,15 +3,18 @@ Capistrano::Configuration.instance(:must_exist).load do namespace :resque do task :start do start_workers + invoke "resque:scheduler:start" end task :stop do stop_workers + invoke "resque:scheduler:stop" end task :restart do stop_workers start_workers + invoke "resque:scheduler:restart" end def rails_env @@ -36,5 +39,41 @@ Capistrano::Configuration.instance(:must_exist).load do ].join(',') run "cd #{fetch :current_path} && COUNT=#{workers_count} QUEUE=#{queue} #{rails_env} BACKGROUND=yes bundle exec rake resque:workers" end + + def remote_file_exists?(full_path) + 'true' == capture("if [ -e #{full_path} ]; then echo 'true'; fi").strip + end + + namespace :scheduler do + + desc "See current scheduler status" + task :status do + pid = "#{fetch :current_path}/tmp/pids/scheduler.pid" + if remote_file_exists?(pid) + info capture(:ps, "-f -p $(cat #{pid}) | sed -n 2p") + end + end + + desc "Starts resque scheduler with default configs" + task :start do + pid = "#{fetch :current_path}/tmp/pids/scheduler.pid" + run "cd #{fetch :current_path} && #{rails_env} PIDFILE=#{pid} BACKGROUND=yes VERBOSE=1 MUTE=1 bundle exec rake resque:scheduler" + end + + desc "Stops resque scheduler" + task :stop do + pid = "#{fetch :current_path}/tmp/pids/scheduler.pid" + if remote_file_exists?(pid) + run "cd #{fetch :current_path} && kill -s QUIT $(cat #{pid}); rm #{pid}" + end + end + + task :restart do + invoke "resque:scheduler:stop" + invoke "resque:scheduler:start" + end + end + end + end From e9520ab02cbb029fe3a057e3db85afdc892a574a Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Mon, 10 Feb 2014 23:59:03 +0400 Subject: [PATCH 3/7] Hotfix --- lib/recipes/resque.rb | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/recipes/resque.rb b/lib/recipes/resque.rb index 97f03ca7d..afae87b5a 100644 --- a/lib/recipes/resque.rb +++ b/lib/recipes/resque.rb @@ -3,18 +3,15 @@ Capistrano::Configuration.instance(:must_exist).load do namespace :resque do task :start do start_workers - invoke "resque:scheduler:start" end task :stop do stop_workers - invoke "resque:scheduler:stop" end task :restart do stop_workers start_workers - invoke "resque:scheduler:restart" end def rails_env @@ -56,22 +53,31 @@ Capistrano::Configuration.instance(:must_exist).load do desc "Starts resque scheduler with default configs" task :start do - pid = "#{fetch :current_path}/tmp/pids/scheduler.pid" - run "cd #{fetch :current_path} && #{rails_env} PIDFILE=#{pid} BACKGROUND=yes VERBOSE=1 MUTE=1 bundle exec rake resque:scheduler" + start_scheduler end desc "Stops resque scheduler" task :stop do + stop_scheduler + end + + task :restart do + stop_scheduler + start_scheduler + end + + def start_scheduler + pid = "#{fetch :current_path}/tmp/pids/scheduler.pid" + run "cd #{fetch :current_path} && #{rails_env} PIDFILE=#{pid} BACKGROUND=yes VERBOSE=1 MUTE=1 bundle exec rake resque:scheduler" + end + + def stop_scheduler pid = "#{fetch :current_path}/tmp/pids/scheduler.pid" if remote_file_exists?(pid) run "cd #{fetch :current_path} && kill -s QUIT $(cat #{pid}); rm #{pid}" end end - task :restart do - invoke "resque:scheduler:stop" - invoke "resque:scheduler:start" - end end end From b9e48664ecc78c6bfe47bb7140f9ec67fce5367a Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Tue, 11 Feb 2014 00:14:52 +0400 Subject: [PATCH 4/7] Fixed: wrong number of arguments (1 for 0) --- app/models/build_list.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/build_list.rb b/app/models/build_list.rb index f1242996f..ffcb58761 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -523,7 +523,7 @@ class BuildList < ActiveRecord::Base ) end - def delayed_add_job_to_abf_worker_queue + def delayed_add_job_to_abf_worker_queue(*args) now_add_job_to_abf_worker_queue if status == BUILD_PENDING end later :delayed_add_job_to_abf_worker_queue, delay: 60, queue: :clone_build From f662332be5d173e82c31ddf582da81481574fa54 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Tue, 11 Feb 2014 21:34:48 +0400 Subject: [PATCH 5/7] decrease count of puma threads, small refactoring --- app/models/build_list.rb | 2 +- config/puma/production.rb | 2 +- lib/abf_worker/model_helper.rb | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/models/build_list.rb b/app/models/build_list.rb index ffcb58761..3527a4d00 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -524,7 +524,7 @@ class BuildList < ActiveRecord::Base end def delayed_add_job_to_abf_worker_queue(*args) - now_add_job_to_abf_worker_queue if status == BUILD_PENDING + restart_job if status == BUILD_PENDING end later :delayed_add_job_to_abf_worker_queue, delay: 60, queue: :clone_build diff --git a/config/puma/production.rb b/config/puma/production.rb index 29b235ba6..0da7225bc 100644 --- a/config/puma/production.rb +++ b/config/puma/production.rb @@ -4,7 +4,7 @@ state_path File.join(base_path, 'shared', 'pids', 'puma.state') bind 'unix:///tmp/rosa_build_unicorn.sock' environment ENV['RAILS_ENV'] || 'production' -threads *(ENV['PUMA_THREADS'] || '1,4').split(',') +threads *(ENV['PUMA_THREADS'] || '1,3').split(',') workers ENV['PUMA_WORKERS'] || 7 diff --git a/lib/abf_worker/model_helper.rb b/lib/abf_worker/model_helper.rb index 5ee7b8193..d7a5c7692 100644 --- a/lib/abf_worker/model_helper.rb +++ b/lib/abf_worker/model_helper.rb @@ -29,8 +29,7 @@ module AbfWorker::ModelHelper end def restart_job - redis = Resque.redis - redis.lpush "queue:#{worker_queue_with_priority}", + Resque.redis.lpush "queue:#{worker_queue_with_priority}", Resque.encode({'class' => worker_queue_class, 'args' => [abf_worker_args]}) end From ec40018d9b4908cb0f8b784afabc5661f12f2994 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Tue, 11 Feb 2014 21:48:54 +0400 Subject: [PATCH 6/7] updated DB_POOL variable --- config/puma/production.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config/puma/production.rb b/config/puma/production.rb index 0da7225bc..e5f119b24 100644 --- a/config/puma/production.rb +++ b/config/puma/production.rb @@ -14,7 +14,14 @@ on_worker_boot do if defined?(ActiveRecord::Base) ActiveSupport.on_load(:active_record) do ActiveRecord::Base.connection.disconnect! rescue ActiveRecord::ConnectionNotEstablished - ActiveRecord::Base.establish_connection + + config = Rails.application.config.database_configuration[Rails.env] + config['reaping_frequency'] = ENV['DB_REAP_FREQ'] || 10 # seconds + config['pool'] = ENV['DB_POOL'] || 21 + + ActiveRecord::Base.establish_connection(config) + + Rails.logger.info "Connected to PG. Connection pool size #{config['pool']}, reaping frequency #{config['reaping_frequency']}" end # QC::Conn.connect Rails.logger.info('Connected to PG') @@ -24,4 +31,4 @@ on_worker_boot do # Rails.logger.info('Connected to Redis') end -activate_control_app 'unix:///tmp/rosa_build_pumactl.sock' \ No newline at end of file +activate_control_app 'unix:///tmp/rosa_build_pumactl.sock' From 98d91a761d6bf386577f0818b6c18b706065150b Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Wed, 12 Feb 2014 12:27:37 +0400 Subject: [PATCH 7/7] rollback changes --- config/puma/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/puma/production.rb b/config/puma/production.rb index e5f119b24..342720498 100644 --- a/config/puma/production.rb +++ b/config/puma/production.rb @@ -17,7 +17,7 @@ on_worker_boot do config = Rails.application.config.database_configuration[Rails.env] config['reaping_frequency'] = ENV['DB_REAP_FREQ'] || 10 # seconds - config['pool'] = ENV['DB_POOL'] || 21 + config['pool'] = ENV['DB_POOL'] || 7 ActiveRecord::Base.establish_connection(config)