From 790169156e4cb1abefe087875450ae0feb233e45 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 1 Mar 2013 19:26:58 +0400 Subject: [PATCH 1/5] #960: old packages should be saved immediately before removing project from repository --- app/models/project.rb | 3 +- .../build_lists_publish_task_manager.rb | 89 +++++++++++-------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index 3b9ecc077..6d5d5e8f2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -10,12 +10,13 @@ class Project < ActiveRecord::Base has_many :issues, :dependent => :destroy has_many :pull_requests, :dependent => :destroy, :foreign_key => 'to_project_id' has_many :labels, :dependent => :destroy - has_many :build_lists, :dependent => :destroy has_many :project_imports, :dependent => :destroy has_many :project_to_repositories, :dependent => :destroy has_many :repositories, :through => :project_to_repositories has_many :project_tags, :dependent => :destroy + + has_many :build_lists, :dependent => :destroy has_many :relations, :as => :target, :dependent => :destroy has_many :collaborators, :through => :relations, :source => :actor, :source_type => 'User' diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 31d6468fa..09262b389 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -9,6 +9,7 @@ module AbfWorker LOCKED_REPOSITORIES LOCKED_REP_AND_PLATFORMS LOCKED_BUILD_LISTS + PACKAGES_FOR_CLEANUP REGENERATE_METADATA).each do |kind| const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}" end @@ -29,15 +30,18 @@ module AbfWorker if repository.platform.personal? Platform.main.each do |main_platform| redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{main_platform.id}" + gather_old_packages project.id, repository.id, main_platform.id end else redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{repository.platform.id}" + gather_old_packages project.id, repository.id, repository.platform.id end end def cleanup_completed(projects_for_cleanup) projects_for_cleanup.each do |key| redis.lrem LOCKED_PROJECTS_FOR_CLEANUP, 0, key + redis.hdel PACKAGES_FOR_CLEANUP, key end end @@ -116,6 +120,29 @@ module AbfWorker private + def gather_old_packages(project_id, repository_id, main_platform_id) + build_lists_for_cleanup = [] + Arch.pluck(:id).each do |arch_id| + bl = BuildList.where(:project_id => project_id). + where(:new_core => true, :status => BuildList::BUILD_PUBLISHED). + where(:save_to_repository_id => repository_id). + where(:build_for_platform_id => main_platform_id). + where(:arch_id => arch_id). + order(:updated_at).first + build_lists_for_cleanup << bl if bl + end + + old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} + + build_lists_for_cleanup.each do |bl| + bl.last_published.includes(:packages).limit(5).each{ |old_bl| + fill_packages(old_bl, old_packages, :fullname) + } + end + + @redis.hset PACKAGES_FOR_CLEANUP, "#{project_id}-#{repository_id}-#{main_platform_id}", old_packages.to_json + end + def locked_repositories @redis.lrange LOCKED_REPOSITORIES, 0, -1 end @@ -200,57 +227,52 @@ module AbfWorker projects_for_cleanup = @redis.lrange(PROJECTS_FOR_CLEANUP, 0, -1). select{ |k| k =~ /#{save_to_repository_id}\-#{build_for_platform_id}$/ } - build_lists_for_cleanup = [] + old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} + projects_for_cleanup.each do |key| - pr, rep, pl = *key.split('-') - added = false - Arch.pluck(:id).each do |arch_id| - bl = BuildList.where(:project_id => pr). - where(:new_core => true, :status => BuildList::BUILD_PUBLISHED). - where(:save_to_repository_id => save_to_repository_id). - where(:build_for_platform_id => build_for_platform_id). - where(:arch_id => arch_id). - order(:updated_at).first - if bl - build_lists_for_cleanup << bl - added = true - end + @redis.lrem PROJECTS_FOR_CLEANUP, 0, key + packages = @redis.hget PACKAGES_FOR_CLEANUP, key + next unless packages + packages = JSON.parse packages + old_packages[:sources] |= packages['sources'] + [:x86_64, :i586).each do |arch| + old_packages[:binaries][arch] |= packages['binaries'][arch.to_s] end - # No packages for removing - @redis.lrem(PROJECTS_FOR_CLEANUP, 0, key) unless added end - bl = build_lists.first || build_lists_for_cleanup.first - return false unless bl + bl = build_lists.first + return false if !bl && old_packages[:sources].empty? - platform_path = "#{bl.save_to_platform.path}/repository" - if bl.save_to_platform.personal? - platform_path << '/' << bl.build_for_platform.name + save_to_repository = Repository.find save_to_repository_id + save_to_platform = save_to_repository.platform + build_for_platform = Platform.find build_for_platform_id + platform_path = "#{save_to_platform.path}/repository" + if save_to_platform.personal? + platform_path << '/' << build_for_platform.name system "mkdir -p #{platform_path}" end - worker_queue = bl.worker_queue_with_priority("publish_worker") - worker_class = bl.worker_queue_class("AbfWorker::PublishWorker") + worker_queue = bl ? bl.worker_queue_with_priority("publish_worker") : 'publish_worker_default' + worker_class = bl ? bl.worker_queue_class("AbfWorker::PublishWorker") : 'AbfWorker::PublishWorkerDefault' - distrib_type = bl.build_for_platform.distrib_type + distrib_type = build_for_platform.distrib_type cmd_params = { - 'RELEASED' => bl.save_to_platform.released, - 'REPOSITORY_NAME' => bl.save_to_repository.name, + 'RELEASED' => save_to_platform.released, + 'REPOSITORY_NAME' => save_to_repository.name, 'TYPE' => distrib_type }.map{ |k, v| "#{k}=#{v}" }.join(' ') options = { - :id => bl.id, - :arch => bl.arch.name, + :id => (bl ? bl.id : Time.now.to_i), + :arch => (bl ? bl.arch.name : 'x86_64'), :distrib_type => distrib_type, :cmd_params => cmd_params, :platform => {:platform_path => platform_path}, - :repository => {:id => bl.save_to_repository.id}, + :repository => {:id => save_to_repository_id}, :type => :publish, :time_living => 9600 # 160 min } packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} - old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} build_list_ids = [] new_sources = {} @@ -266,12 +288,6 @@ module AbfWorker end packages[:sources] = new_sources.values.compact - build_lists_for_cleanup.each do |bl| - bl.last_published.includes(:packages).limit(5).each{ |old_bl| - fill_packages(old_bl, old_packages, :fullname) - } - end - Resque.push( worker_queue, 'class' => worker_class, @@ -284,7 +300,6 @@ module AbfWorker ) projects_for_cleanup.each do |key| - @redis.lrem PROJECTS_FOR_CLEANUP, 0, key @redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key end From 66e1bcc4e849f7a5f9b22f161e3e0feeaeb0de3a Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 1 Mar 2013 19:37:12 +0400 Subject: [PATCH 2/5] #960: hot fix --- lib/abf_worker/build_lists_publish_task_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 90dfe2bc4..fac47ac0f 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -235,7 +235,7 @@ module AbfWorker next unless packages packages = JSON.parse packages old_packages[:sources] |= packages['sources'] - [:x86_64, :i586).each do |arch| + [:x86_64, :i586].each do |arch| old_packages[:binaries][arch] |= packages['binaries'][arch.to_s] end end From 4d88827da49331c345ec6312c77147112d5b52e2 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 1 Mar 2013 20:19:15 +0400 Subject: [PATCH 3/5] #960: fix: NoMethodError "fill_packages" --- .../build_lists_publish_task_manager.rb | 74 ++++++++++--------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index fac47ac0f..76f990328 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -116,32 +116,46 @@ module AbfWorker }] ) end + + def gather_old_packages(project_id, repository_id, platform_id) + build_lists_for_cleanup = [] + Arch.pluck(:id).each do |arch_id| + bl = BuildList.where(:project_id => project_id). + where(:new_core => true, :status => BuildList::BUILD_PUBLISHED). + where(:save_to_repository_id => repository_id). + where(:build_for_platform_id => platform_id). + where(:arch_id => arch_id). + order(:updated_at).first + build_lists_for_cleanup << bl if bl + end + + old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} + + build_lists_for_cleanup.each do |bl| + bl.last_published.includes(:packages).limit(5).each{ |old_bl| + fill_packages(old_bl, old_packages, :fullname) + } + end + + redis.hset PACKAGES_FOR_CLEANUP, "#{project_id}-#{repository_id}-#{platform_id}", old_packages.to_json + end + + def fill_packages(bl, results_map, field = :sha1) + results_map[:sources] |= bl.packages.by_package_type('source').pluck(field).compact if field != :sha1 + + binaries = bl.packages.by_package_type('binary').pluck(field).compact + arch = bl.arch.name.to_sym + results_map[:binaries][arch] |= binaries + # Publish/remove i686 RHEL packages into/from x86_64 + if arch == :i586 && bl.build_for_platform.distrib_type == 'rhel' && bl.project.publish_i686_into_x86_64? + results_map[:binaries][:x86_64] |= binaries + end + end + end private - def gather_old_packages(project_id, repository_id, platform_id) - build_lists_for_cleanup = [] - Arch.pluck(:id).each do |arch_id| - bl = BuildList.where(:project_id => project_id). - where(:new_core => true, :status => BuildList::BUILD_PUBLISHED). - where(:save_to_repository_id => repository_id). - where(:build_for_platform_id => platform_id). - where(:arch_id => arch_id). - order(:updated_at).first - build_lists_for_cleanup << bl if bl - end - - old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} - - build_lists_for_cleanup.each do |bl| - bl.last_published.includes(:packages).limit(5).each{ |old_bl| - fill_packages(old_bl, old_packages, :fullname) - } - end - - @redis.hset PACKAGES_FOR_CLEANUP, "#{project_id}-#{repository_id}-#{platform_id}", old_packages.to_json - end def locked_repositories @redis.lrange LOCKED_REPOSITORIES, 0, -1 @@ -281,9 +295,9 @@ module AbfWorker build_lists.each do |bl| # remove duplicates of sources for different arches bl.packages.by_package_type('source').each{ |s| new_sources["#{s.fullname}"] = s.sha1 } - fill_packages(bl, packages) + self.class.fill_packages(bl, packages) bl.last_published.includes(:packages).limit(5).each{ |old_bl| - fill_packages(old_bl, old_packages, :fullname) + self.class.fill_packages(old_bl, old_packages, :fullname) } build_list_ids << bl.id @redis.lpush(LOCKED_BUILD_LISTS, bl.id) @@ -309,18 +323,6 @@ module AbfWorker return true end - def fill_packages(bl, results_map, field = :sha1) - results_map[:sources] |= bl.packages.by_package_type('source').pluck(field).compact if field != :sha1 - - binaries = bl.packages.by_package_type('binary').pluck(field).compact - arch = bl.arch.name.to_sym - results_map[:binaries][arch] |= binaries - # Publish/remove i686 RHEL packages into/from x86_64 - if arch == :i586 && bl.build_for_platform.distrib_type == 'rhel' && bl.project.publish_i686_into_x86_64? - results_map[:binaries][:x86_64] |= binaries - end - end - def create_tasks_for_repository_regenerate_metadata worker_queue = 'publish_worker_default' worker_class = 'AbfWorker::PublishWorkerDefault' From 2c9297c4323711316136a843bd2dc9b51da472c4 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 1 Mar 2013 21:12:09 +0400 Subject: [PATCH 4/5] #960: remove empty line --- lib/abf_worker/build_lists_publish_task_manager.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 76f990328..e56159643 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -130,7 +130,6 @@ module AbfWorker end old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} - build_lists_for_cleanup.each do |bl| bl.last_published.includes(:packages).limit(5).each{ |old_bl| fill_packages(old_bl, old_packages, :fullname) From 1dca61afd1bdcdf8f0793c11463b3f45faf50580 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 1 Mar 2013 21:23:56 +0400 Subject: [PATCH 5/5] #960: remove packages from last 2 build lists instead of 5 --- lib/abf_worker/build_lists_publish_task_manager.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index e56159643..dc731f2dc 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -131,7 +131,7 @@ module AbfWorker old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} build_lists_for_cleanup.each do |bl| - bl.last_published.includes(:packages).limit(5).each{ |old_bl| + bl.last_published.includes(:packages).limit(2).each{ |old_bl| fill_packages(old_bl, old_packages, :fullname) } end @@ -295,7 +295,7 @@ module AbfWorker # remove duplicates of sources for different arches bl.packages.by_package_type('source').each{ |s| new_sources["#{s.fullname}"] = s.sha1 } self.class.fill_packages(bl, packages) - bl.last_published.includes(:packages).limit(5).each{ |old_bl| + bl.last_published.includes(:packages).limit(2).each{ |old_bl| self.class.fill_packages(old_bl, old_packages, :fullname) } build_list_ids << bl.id