From fe20a9788a770d93633bda4681bdc580065778a9 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Tue, 25 Feb 2014 00:36:51 +0400 Subject: [PATCH] Use resque-schedule for cleanup --- .../build_lists_publish_task_manager_job.rb | 8 ++++++++ app/jobs/clean_build_lists_queues_job.rb | 18 +++++++++++++++++ app/jobs/clean_rpm_build_node_job.rb | 10 ++++++++++ app/models/build_list.rb | 1 - app/models/rpm_build_node.rb | 6 ------ config/application.rb | 1 + config/initializers/resque.rb | 1 + config/resque_schedule.yml | 20 +++++++++++++++++++ config/schedule.rb | 8 -------- lib/abf_worker/model_helper.rb | 12 ----------- 10 files changed, 58 insertions(+), 27 deletions(-) create mode 100644 app/jobs/build_lists_publish_task_manager_job.rb create mode 100644 app/jobs/clean_build_lists_queues_job.rb create mode 100644 app/jobs/clean_rpm_build_node_job.rb create mode 100644 config/resque_schedule.yml diff --git a/app/jobs/build_lists_publish_task_manager_job.rb b/app/jobs/build_lists_publish_task_manager_job.rb new file mode 100644 index 000000000..1fc5dca83 --- /dev/null +++ b/app/jobs/build_lists_publish_task_manager_job.rb @@ -0,0 +1,8 @@ +class BuildListsPublishTaskManagerJob + @queue = :hook + + def self.perform + AbfWorker::BuildListsPublishTaskManager.new.run + end + +end diff --git a/app/jobs/clean_build_lists_queues_job.rb b/app/jobs/clean_build_lists_queues_job.rb new file mode 100644 index 000000000..853c523b4 --- /dev/null +++ b/app/jobs/clean_build_lists_queues_job.rb @@ -0,0 +1,18 @@ +class CleanBuildListsQueuesJob + @queue = :hook + + def self.perform + redis = Resque.redis + redis.smembers('queues').each do |key| + next if key !~ /(user|mass)_build_/ + queue = "queue:#{key}" + if redis.llen(queue) == 0 + redis.multi do + redis.del queue + redis.srem 'queues', key + end + end + end + end + +end diff --git a/app/jobs/clean_rpm_build_node_job.rb b/app/jobs/clean_rpm_build_node_job.rb new file mode 100644 index 000000000..67c2f212e --- /dev/null +++ b/app/jobs/clean_rpm_build_node_job.rb @@ -0,0 +1,10 @@ +class CleanRpmBuildNodeJob + @queue = :hook + + def self.perform + RpmBuildNode.all.each do |n| + n.delete unless n.user_id + end + end + +end diff --git a/app/models/build_list.rb b/app/models/build_list.rb index 7caed5549..561345a18 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -558,7 +558,6 @@ class BuildList < ActiveRecord::Base if task build_list = BuildList.where(id: task['args'][0]['id']).first - build_list.cleanup_build_sets build_list.delayed_add_job_to_abf_worker_queue build_list end diff --git a/app/models/rpm_build_node.rb b/app/models/rpm_build_node.rb index f03a5b161..c9a95f6bb 100644 --- a/app/models/rpm_build_node.rb +++ b/app/models/rpm_build_node.rb @@ -17,12 +17,6 @@ class RpmBuildNode < Ohm::Model User.where(id: user_id).first end - def self.cleanup! - RpmBuildNode.all.each do |n| - n.delete unless n.user_id - end - end - def self.total_statistics systems, others, busy = 0, 0, 0 RpmBuildNode.all.select{ |n| n.user_id }.each do |n| diff --git a/config/application.rb b/config/application.rb index e59074d7e..fc556a46e 100644 --- a/config/application.rb +++ b/config/application.rb @@ -27,6 +27,7 @@ module Rosa # Custom directories with classes and modules you want to be autoloadable. # config.autoload_paths += %W(#{config.root}/extras) config.autoload_paths += %W(#{config.root}/app/presenters) + config.autoload_paths += %W(#{config.root}/app/jobs) # Only load the plugins named here, in the order given (default is alphabetical). # :all can be used as a placeholder for all plugins not explicitly named. diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb index f2c69e5c3..b810398ec 100644 --- a/config/initializers/resque.rb +++ b/config/initializers/resque.rb @@ -10,6 +10,7 @@ Resque::Mailer.excluded_environments = [:test] unless Rails.env.test? PerformLater.config.enabled = true # this will default to false if unset + Resque.schedule = YAML.load_file(File.join(Rails.root, 'config/resque_schedule.yml')) # load the schedule end Resque::Plugins::Status::Hash.expire_in = (24 * 60 * 60) # 24hrs in seconds diff --git a/config/resque_schedule.yml b/config/resque_schedule.yml new file mode 100644 index 000000000..1c443b286 --- /dev/null +++ b/config/resque_schedule.yml @@ -0,0 +1,20 @@ +clean_rpm_build_nodes: + every: + - '1m' + class: 'CleanRpmBuildNodeJob' + queue: hook + description: 'Cleans RPM build nodes' + +build_lists_publish_task_manager: + every: + - '3m' + class: 'BuildListsPublishTaskManagerJob' + queue: hook + description: 'Creates tasks for publishing' + +clean_build_lists_queues: + every: + - '1m' + class: 'CleanBuildListsQueuesJob' + queue: hook + description: 'Cleans build_lists queues' \ No newline at end of file diff --git a/config/schedule.rb b/config/schedule.rb index b0b7e6031..320b7603e 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -26,14 +26,6 @@ every :day, at: '3:00 am' do rake 'activity_feeds:clear', output: 'log/activity_feeds.log' end -every 3.minute do - runner 'AbfWorker::BuildListsPublishTaskManager.new.run', output: 'log/task_manager.log' -end - -every 1.minute do - runner 'RpmBuildNode.cleanup!' -end - every 1.hour do rake 'buildlist:clear:outdated_canceling', output: 'log/canceling_build_list_clear.log' end diff --git a/lib/abf_worker/model_helper.rb b/lib/abf_worker/model_helper.rb index 8b32bfc58..9fc717a17 100644 --- a/lib/abf_worker/model_helper.rb +++ b/lib/abf_worker/model_helper.rb @@ -58,7 +58,6 @@ module AbfWorker::ModelHelper worker_queue_class, abf_worker_args ) - cleanup_build_sets result end @@ -82,17 +81,6 @@ module AbfWorker::ModelHelper "AbfWorker::#{abf_worker_base_queue.classify}#{abf_worker_priority.capitalize}" end - def cleanup_build_sets - return unless is_a?(BuildList) - queue = worker_queue_with_priority - if Resque.redis.llen("queue:#{queue}") == 0 - key = mass_build_id ? MASS_BUILDS_SET : USER_BUILDS_SET - Resque.redis.srem key, mass_build_id || user_id - Resque.redis.del "queue:#{queue}" - Resque.redis.srem 'queues', queue - end - end - private