From 44ac1056abd7bae00ebb4f3df6ba6930389cb088 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Tue, 15 Apr 2014 22:41:06 +0400 Subject: [PATCH] #376: updated priority of queues --- app/jobs/build_lists_publish_task_manager_job.rb | 2 +- app/jobs/build_lists_queues_monitoring_job.rb | 2 +- app/jobs/clean_api_defender_statistics_job.rb | 2 +- app/jobs/clean_rpm_build_node_job.rb | 2 +- app/jobs/restart_nodes_job.rb | 2 +- app/models/build_list.rb | 6 +++--- app/models/concerns/file_store_clean.rb | 4 ++-- app/models/mass_build.rb | 8 ++++---- app/models/platform.rb | 4 ++-- app/models/project.rb | 2 +- app/models/pull_request.rb | 2 +- app/models/repository.rb | 9 ++++----- config/resque_schedule.yml | 10 +++++----- lib/recipes/resque.rb | 4 +++- 14 files changed, 30 insertions(+), 29 deletions(-) diff --git a/app/jobs/build_lists_publish_task_manager_job.rb b/app/jobs/build_lists_publish_task_manager_job.rb index 1fc5dca83..4a615dfec 100644 --- a/app/jobs/build_lists_publish_task_manager_job.rb +++ b/app/jobs/build_lists_publish_task_manager_job.rb @@ -1,5 +1,5 @@ class BuildListsPublishTaskManagerJob - @queue = :hook + @queue = :middle def self.perform AbfWorker::BuildListsPublishTaskManager.new.run diff --git a/app/jobs/build_lists_queues_monitoring_job.rb b/app/jobs/build_lists_queues_monitoring_job.rb index 08b4db60a..ecea75cf0 100644 --- a/app/jobs/build_lists_queues_monitoring_job.rb +++ b/app/jobs/build_lists_queues_monitoring_job.rb @@ -1,5 +1,5 @@ class BuildListsQueuesMonitoringJob - @queue = :hook + @queue = :middle def self.perform Redis.current.smembers('resque:queues').each do |key| diff --git a/app/jobs/clean_api_defender_statistics_job.rb b/app/jobs/clean_api_defender_statistics_job.rb index 6fea6e786..9f8a6d976 100644 --- a/app/jobs/clean_api_defender_statistics_job.rb +++ b/app/jobs/clean_api_defender_statistics_job.rb @@ -1,5 +1,5 @@ class CleanApiDefenderStatisticsJob - @queue = :clone_build + @queue = :low def self.perform deadline = Date.today - 1.month diff --git a/app/jobs/clean_rpm_build_node_job.rb b/app/jobs/clean_rpm_build_node_job.rb index 67c2f212e..b31e4ec25 100644 --- a/app/jobs/clean_rpm_build_node_job.rb +++ b/app/jobs/clean_rpm_build_node_job.rb @@ -1,5 +1,5 @@ class CleanRpmBuildNodeJob - @queue = :hook + @queue = :middle def self.perform RpmBuildNode.all.each do |n| diff --git a/app/jobs/restart_nodes_job.rb b/app/jobs/restart_nodes_job.rb index e158cf3a0..020062162 100644 --- a/app/jobs/restart_nodes_job.rb +++ b/app/jobs/restart_nodes_job.rb @@ -1,5 +1,5 @@ class RestartNodesJob - @queue = :hook + @queue = :low def self.perform available_nodes = RpmBuildNode.all.map{ |n| n.user_id }.compact.uniq diff --git a/app/models/build_list.rb b/app/models/build_list.rb index 54fe6ca5d..aafbb8ae5 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -259,8 +259,8 @@ class BuildList < ActiveRecord::Base end end - later :publish, queue: :clone_build - later :add_job_to_abf_worker_queue, queue: :clone_build + later :publish, queue: :middle + later :add_job_to_abf_worker_queue, queue: :middle HUMAN_CONTAINER_STATUSES = { WAITING_FOR_RESPONSE => :waiting_for_publish, BUILD_PUBLISHED => :container_published, @@ -572,7 +572,7 @@ class BuildList < ActiveRecord::Base def delayed_add_job_to_abf_worker_queue(*args) restart_job if status == BUILD_PENDING end - later :delayed_add_job_to_abf_worker_queue, delay: 60, queue: :clone_build + later :delayed_add_job_to_abf_worker_queue, delay: 60, queue: :middle protected diff --git a/app/models/concerns/file_store_clean.rb b/app/models/concerns/file_store_clean.rb index f4fe70c42..c04d934a1 100644 --- a/app/models/concerns/file_store_clean.rb +++ b/app/models/concerns/file_store_clean.rb @@ -6,7 +6,7 @@ module FileStoreClean destroy_files_from_file_store if Rails.env.production? super end - later :destroy, queue: :clone_build + later :destroy, queue: :middle def sha1_of_file_store_files raise NotImplementedError, "You should implement this method" @@ -31,7 +31,7 @@ module FileStoreClean def later_destroy_files_from_file_store(args) destroy_files_from_file_store(args) end - later :later_destroy_files_from_file_store, queue: :clone_build + later :later_destroy_files_from_file_store, queue: :middle end def self.file_exist_on_file_store?(sha1) diff --git a/app/models/mass_build.rb b/app/models/mass_build.rb index 998d2faea..a8f720f05 100644 --- a/app/models/mass_build.rb +++ b/app/models/mass_build.rb @@ -59,7 +59,7 @@ class MassBuild < ActiveRecord::Base end end end - later :build_all, queue: :clone_build + later :build_all, queue: :low def generate_failed_builds_list generate_list BuildList::BUILD_ERROR @@ -75,17 +75,17 @@ class MassBuild < ActiveRecord::Base bl.cancel end end - later :cancel_all, queue: :clone_build + later :cancel_all, queue: :low def publish_success_builds(user) publish user, BuildList::SUCCESS, BuildList::FAILED_PUBLISH end - later :publish_success_builds, queue: :clone_build + later :publish_success_builds, queue: :low def publish_test_failed_builds(user) publish user, BuildList::TESTS_FAILED end - later :publish_test_failed_builds, queue: :clone_build + later :publish_test_failed_builds, queue: :low COUNT_STATUSES.each do |stat| stat_count = "#{stat}_count" diff --git a/app/models/platform.rb b/app/models/platform.rb index 1f043350e..ee6a6fd40 100644 --- a/app/models/platform.rb +++ b/app/models/platform.rb @@ -205,7 +205,7 @@ class Platform < ActiveRecord::Base def destroy with_skip {super} # avoid cascade XML RPC requests end - later :destroy, queue: :clone_build + later :destroy, queue: :low def default_host EventLog.current_controller.request.host_with_port rescue ::Rosa::Application.config.action_mailer.default_url_options[:host] @@ -274,7 +274,7 @@ class Platform < ActiveRecord::Base def fs_clone(old_name = parent.name, new_name = name) FileUtils.cp_r "#{parent.path}/repository", path end - later :fs_clone, queue: :clone_build + later :fs_clone, queue: :low def freeze_platform_and_update_repos if released_changed? && released == true diff --git a/app/models/project.rb b/app/models/project.rb index 1f9f846d7..db2351fec 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -417,7 +417,7 @@ class Project < ActiveRecord::Base PullRequest.where(from_project_id: id).each{ |p| p.update_relations(old_name) } pull_requests.where('from_project_id != to_project_id').each(&:update_relations) end - later :update_path_to_project, queue: :clone_build + later :update_path_to_project, queue: :middle def check_default_branch if self.repo.branches.count > 0 && self.repo.branches.map(&:name).exclude?(self.default_branch) diff --git a/app/models/pull_request.rb b/app/models/pull_request.rb index 1e98c5629..b482a2e97 100644 --- a/app/models/pull_request.rb +++ b/app/models/pull_request.rb @@ -57,7 +57,7 @@ class PullRequest < ActiveRecord::Base system 'git', 'remote', 'set-url', 'head', from_project.path if cross_pull? end end - later :update_relations, queue: :clone_build + later :update_relations, queue: :middle def cross_pull? from_project_id != to_project_id diff --git a/app/models/repository.rb b/app/models/repository.rb index 561d99902..e42ce367c 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -59,8 +59,7 @@ class Repository < ActiveRecord::Base from.projects.find_each {|p| self.projects << p} end end - later :clone_relations, loner: true, queue: :clone_build - + later :clone_relations, loner: true, queue: :low def add_projects(list, user) current_ability = Ability.new(user) list.lines.each do |line| @@ -75,7 +74,7 @@ class Repository < ActiveRecord::Base end end end - later :add_projects, queue: :clone_build + later :add_projects, queue: :middle def remove_projects(list) list.lines.each do |name| @@ -87,7 +86,7 @@ class Repository < ActiveRecord::Base end end end - later :remove_projects, queue: :clone_build + later :remove_projects, queue: :middle def full_clone(attrs = {}) base_clone(attrs).tap do |c| @@ -148,7 +147,7 @@ class Repository < ActiveRecord::Base def destroy with_skip {super} # avoid cascade XML RPC requests end - later :destroy, queue: :clone_build + later :destroy, queue: :low def self.custom_sort(repos) repos.select{ |r| SORT.keys.include?(r.name) }.sort{ |a,b| SORT[a.name] <=> SORT[b.name] } | repos.sort_by(&:name) diff --git a/config/resque_schedule.yml b/config/resque_schedule.yml index fbdade941..79565cbb0 100644 --- a/config/resque_schedule.yml +++ b/config/resque_schedule.yml @@ -2,33 +2,33 @@ clean_rpm_build_nodes: every: - '1m' class: 'CleanRpmBuildNodeJob' - queue: hook + queue: middle description: 'Cleans RPM build nodes' build_lists_publish_task_manager: every: - '3m' class: 'BuildListsPublishTaskManagerJob' - queue: hook + queue: middle description: 'Creates tasks for publishing' build_lists_queues_monitoring: every: - '1m' class: 'BuildListsQueuesMonitoringJob' - queue: hook + queue: middle description: 'Monitoring for "user/mass-build" queues' clean_api_defender_statistics: every: - '1d' class: 'CleanApiDefenderStatisticsJob' - queue: clone_build + queue: low description: 'Cleans ApiDefender statistics' restart_nodes: every: - '5m' class: 'RestartNodesJob' - queue: hook + queue: low description: 'Restarts unavailable nodes' \ No newline at end of file diff --git a/lib/recipes/resque.rb b/lib/recipes/resque.rb index 3ffbb70dd..dbf72ce71 100644 --- a/lib/recipes/resque.rb +++ b/lib/recipes/resque.rb @@ -32,9 +32,11 @@ Capistrano::Configuration.instance(:must_exist).load do :fork_import, :hook, :clone_build, + :middle, :notification ].join(',') - run "cd #{fetch :current_path} && COUNT=#{workers_count} QUEUE=#{queue} #{rails_env} BACKGROUND=yes bundle exec rake resque:workers" + run "cd #{fetch :current_path} && COUNT=#{workers_count - 1} QUEUE=#{queue} INTERVAL=0.1 #{rails_env} BACKGROUND=yes bundle exec rake resque:workers" + run "cd #{fetch :current_path} && COUNT=1 QUEUE=low #{rails_env} BACKGROUND=yes bundle exec rake resque:workers" end def remote_file_exists?(full_path)