diff --git a/app/models/project.rb b/app/models/project.rb index 3b9ecc077..6d5d5e8f2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -10,12 +10,13 @@ class Project < ActiveRecord::Base has_many :issues, :dependent => :destroy has_many :pull_requests, :dependent => :destroy, :foreign_key => 'to_project_id' has_many :labels, :dependent => :destroy - has_many :build_lists, :dependent => :destroy has_many :project_imports, :dependent => :destroy has_many :project_to_repositories, :dependent => :destroy has_many :repositories, :through => :project_to_repositories has_many :project_tags, :dependent => :destroy + + has_many :build_lists, :dependent => :destroy has_many :relations, :as => :target, :dependent => :destroy has_many :collaborators, :through => :relations, :source => :actor, :source_type => 'User' diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index ddb109651..dc731f2dc 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -9,6 +9,7 @@ module AbfWorker LOCKED_REPOSITORIES LOCKED_REP_AND_PLATFORMS LOCKED_BUILD_LISTS + PACKAGES_FOR_CLEANUP REGENERATE_METADATA).each do |kind| const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}" end @@ -29,15 +30,18 @@ module AbfWorker if repository.platform.personal? Platform.main.each do |main_platform| redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{main_platform.id}" + gather_old_packages project.id, repository.id, main_platform.id end else redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{repository.platform.id}" + gather_old_packages project.id, repository.id, repository.platform.id end end def cleanup_completed(projects_for_cleanup) projects_for_cleanup.each do |key| redis.lrem LOCKED_PROJECTS_FOR_CLEANUP, 0, key + redis.hdel PACKAGES_FOR_CLEANUP, key end end @@ -112,10 +116,46 @@ module AbfWorker }] ) end + + def gather_old_packages(project_id, repository_id, platform_id) + build_lists_for_cleanup = [] + Arch.pluck(:id).each do |arch_id| + bl = BuildList.where(:project_id => project_id). + where(:new_core => true, :status => BuildList::BUILD_PUBLISHED). + where(:save_to_repository_id => repository_id). + where(:build_for_platform_id => platform_id). + where(:arch_id => arch_id). + order(:updated_at).first + build_lists_for_cleanup << bl if bl + end + + old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} + build_lists_for_cleanup.each do |bl| + bl.last_published.includes(:packages).limit(2).each{ |old_bl| + fill_packages(old_bl, old_packages, :fullname) + } + end + + redis.hset PACKAGES_FOR_CLEANUP, "#{project_id}-#{repository_id}-#{platform_id}", old_packages.to_json + end + + def fill_packages(bl, results_map, field = :sha1) + results_map[:sources] |= bl.packages.by_package_type('source').pluck(field).compact if field != :sha1 + + binaries = bl.packages.by_package_type('binary').pluck(field).compact + arch = bl.arch.name.to_sym + results_map[:binaries][arch] |= binaries + # Publish/remove i686 RHEL packages into/from x86_64 + if arch == :i586 && bl.build_for_platform.distrib_type == 'rhel' && bl.project.publish_i686_into_x86_64? + results_map[:binaries][:x86_64] |= binaries + end + end + end private + def locked_repositories @redis.lrange LOCKED_REPOSITORIES, 0, -1 end @@ -200,80 +240,69 @@ module AbfWorker projects_for_cleanup = @redis.lrange(PROJECTS_FOR_CLEANUP, 0, -1). select{ |k| k =~ /#{save_to_repository_id}\-#{build_for_platform_id}$/ } - build_lists_for_cleanup = [] + old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} + projects_for_cleanup.each do |key| - pr, rep, pl = *key.split('-') - added = false - Arch.pluck(:id).each do |arch_id| - bl = BuildList.where(:project_id => pr). - where(:new_core => true, :status => BuildList::BUILD_PUBLISHED). - where(:save_to_repository_id => save_to_repository_id). - where(:build_for_platform_id => build_for_platform_id). - where(:arch_id => arch_id). - order(:updated_at).first - if bl - build_lists_for_cleanup << bl - added = true - end + @redis.lrem PROJECTS_FOR_CLEANUP, 0, key + packages = @redis.hget PACKAGES_FOR_CLEANUP, key + next unless packages + packages = JSON.parse packages + old_packages[:sources] |= packages['sources'] + [:x86_64, :i586].each do |arch| + old_packages[:binaries][arch] |= packages['binaries'][arch.to_s] end - # No packages for removing - @redis.lrem(PROJECTS_FOR_CLEANUP, 0, key) unless added end - bl = build_lists.first || build_lists_for_cleanup.first - return false unless bl + bl = build_lists.first + return false if !bl && old_packages[:sources].empty? - platform_path = "#{bl.save_to_platform.path}/repository" - if bl.save_to_platform.personal? - platform_path << '/' << bl.build_for_platform.name + save_to_repository = Repository.find save_to_repository_id + save_to_platform = save_to_repository.platform + build_for_platform = Platform.find build_for_platform_id + platform_path = "#{save_to_platform.path}/repository" + if save_to_platform.personal? + platform_path << '/' << build_for_platform.name system "mkdir -p #{platform_path}" end - worker_queue = bl.worker_queue_with_priority("publish_worker") - worker_class = bl.worker_queue_class("AbfWorker::PublishWorker") + worker_queue = bl ? bl.worker_queue_with_priority("publish_worker") : 'publish_worker_default' + worker_class = bl ? bl.worker_queue_class("AbfWorker::PublishWorker") : 'AbfWorker::PublishWorkerDefault' - distrib_type = bl.build_for_platform.distrib_type + distrib_type = build_for_platform.distrib_type cmd_params = { - 'RELEASED' => bl.save_to_platform.released, - 'REPOSITORY_NAME' => bl.save_to_repository.name, + 'RELEASED' => save_to_platform.released, + 'REPOSITORY_NAME' => save_to_repository.name, 'TYPE' => distrib_type }.map{ |k, v| "#{k}=#{v}" }.join(' ') lock_str = "#{save_to_repository_id}-#{build_for_platform_id}" options = { - :id => bl.id, - :arch => bl.arch.name, + :id => (bl ? bl.id : Time.now.to_i), + :arch => (bl ? bl.arch.name : 'x86_64'), :distrib_type => distrib_type, :cmd_params => cmd_params, :platform => {:platform_path => platform_path}, - :repository => {:id => bl.save_to_repository.id}, + :repository => {:id => save_to_repository_id}, :type => :publish, :time_living => 9600, # 160 min :extra => {:lock_str => lock_str} } packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} - old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} build_list_ids = [] new_sources = {} build_lists.each do |bl| # remove duplicates of sources for different arches bl.packages.by_package_type('source').each{ |s| new_sources["#{s.fullname}"] = s.sha1 } - fill_packages(bl, packages) - bl.last_published.includes(:packages).limit(5).each{ |old_bl| - fill_packages(old_bl, old_packages, :fullname) + self.class.fill_packages(bl, packages) + bl.last_published.includes(:packages).limit(2).each{ |old_bl| + self.class.fill_packages(old_bl, old_packages, :fullname) } build_list_ids << bl.id @redis.lpush(LOCKED_BUILD_LISTS, bl.id) end packages[:sources] = new_sources.values.compact - build_lists_for_cleanup.each do |bl| - bl.last_published.includes(:packages).limit(5).each{ |old_bl| - fill_packages(old_bl, old_packages, :fullname) - } - end - Resque.push( worker_queue, 'class' => worker_class, @@ -286,7 +315,6 @@ module AbfWorker ) projects_for_cleanup.each do |key| - @redis.lrem PROJECTS_FOR_CLEANUP, 0, key @redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key end @@ -294,18 +322,6 @@ module AbfWorker return true end - def fill_packages(bl, results_map, field = :sha1) - results_map[:sources] |= bl.packages.by_package_type('source').pluck(field).compact if field != :sha1 - - binaries = bl.packages.by_package_type('binary').pluck(field).compact - arch = bl.arch.name.to_sym - results_map[:binaries][arch] |= binaries - # Publish/remove i686 RHEL packages into/from x86_64 - if arch == :i586 && bl.build_for_platform.distrib_type == 'rhel' && bl.project.publish_i686_into_x86_64? - results_map[:binaries][:x86_64] |= binaries - end - end - def create_tasks_for_repository_regenerate_metadata worker_queue = 'publish_worker_default' worker_class = 'AbfWorker::PublishWorkerDefault'