diff --git a/app/models/platform.rb b/app/models/platform.rb index 4526695e6..8afd33134 100644 --- a/app/models/platform.rb +++ b/app/models/platform.rb @@ -63,6 +63,7 @@ class Platform < ActiveRecord::Base scope :by_type, lambda {|type| where(:platform_type => type) if type.present?} scope :main, by_type('main') scope :personal, by_type('personal') + scope :waiting_for_regeneration, where(:status => WAITING_FOR_REGENERATION) accepts_nested_attributes_for :platform_arch_settings, :allow_destroy => true attr_accessible :name, :distrib_type, :parent_platform_id, :platform_type, :owner, :visibility, :description, :released, :platform_arch_settings_attributes @@ -77,7 +78,7 @@ class Platform < ActiveRecord::Base event :regenerate do transition :waiting_for_regeneration => :regenerating - transition :ready => :waiting_for_regeneration + transition :ready => :waiting_for_regeneration, :if => lambda{ |p| p.main? } end HUMAN_STATUSES.each do |code,name| diff --git a/app/models/repository_status.rb b/app/models/repository_status.rb index 100ddb386..6036e70d9 100644 --- a/app/models/repository_status.rb +++ b/app/models/repository_status.rb @@ -39,6 +39,7 @@ class RepositoryStatus < ActiveRecord::Base scope :platform_ready, where(:platform => {:status => READY}).joins(:platform) scope :for_regeneration, platform_ready.where(:status => WAITING_FOR_REGENERATION) scope :for_resign, platform_ready.where(:status => [WAITING_FOR_RESIGN, WAITING_FOR_RESIGN_AND_REGENERATION]) + scope :not_ready, where('repository_statuses.status != ?', READY) state_machine :status, :initial => :ready do event :ready do @@ -58,6 +59,7 @@ class RepositoryStatus < ActiveRecord::Base event :start_regeneration do transition :waiting_for_regeneration => :regenerating + transition :waiting_for_resign_and_regeneration => :waiting_for_resign_after_regeneration end event :resign do diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index fb7ff85ef..b6bd3a6f0 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -3,12 +3,8 @@ module AbfWorker class BuildListsPublishTaskManager REDIS_MAIN_KEY = 'abf-worker::build-lists-publish-task-manager::' - # LOCKED_REP_AND_PLATFORMS: ['save_to_repository_id-build_for_platform_id', ...] - %w(RESIGN_REPOSITORIES - PROJECTS_FOR_CLEANUP + %w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP - LOCKED_REPOSITORIES - LOCKED_REP_AND_PLATFORMS LOCKED_BUILD_LISTS PACKAGES_FOR_CLEANUP).each do |kind| const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}" @@ -20,6 +16,7 @@ module AbfWorker end def run + create_tasks_for_regenerate_metadata_for_software_center create_tasks_for_resign_repositories create_tasks_for_repository_regenerate_metadata create_tasks_for_build_rpms @@ -52,18 +49,10 @@ module AbfWorker end end - def unlock_repository(repository_id) - redis.lrem LOCKED_REPOSITORIES, 0, repository_id - end - def unlock_build_list(build_list) redis.lrem LOCKED_BUILD_LISTS, 0, build_list.id end - def unlock_rep_and_platform(lock_str) - redis.lrem LOCKED_REP_AND_PLATFORMS, 0, lock_str - end - def packages_structure structure = {:sources => [], :binaries => {}} Arch.pluck(:name).each{ |name| structure[:binaries][name.to_sym] = [] } @@ -156,11 +145,6 @@ module AbfWorker private - - def locked_repositories - @redis.lrange LOCKED_REPOSITORIES, 0, -1 - end - def create_tasks_for_resign_repositories repository_statuses = RepositoryStatus.for_resign.includes(:repository => :platform) repository_statuses.each do |repository_status| @@ -205,27 +189,19 @@ module AbfWorker order(:min_updated_at). limit(@workers_count * 2) # because some repos may be locked - locked_rep = locked_repositories + locked_rep = RepositoryStatus.not_ready.joins(:platform). + where(:platforms => {:platform_type => 'main'}).pluck(:repository_id) available_repos = available_repos.where('save_to_repository_id NOT IN (?)', locked_rep) unless locked_rep.empty? - counter = 1 - - # looks like: - # ['save_to_repository_id-build_for_platform_id', ...] - locked_rep_and_pl = @redis.lrange(LOCKED_REP_AND_PLATFORMS, 0, -1) - for_cleanup = @redis.lrange(PROJECTS_FOR_CLEANUP, 0, -1).map do |key| pr, rep, pl = *key.split('-') - if locked_rep.present? && locked_rep.include?(rep) - nil - else - [rep.to_i, pl.to_i] - end + locked_rep.present? && locked_rep.include?(rep) ? nil : [rep.to_i, pl.to_i] end.compact - available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup + counter = 1 + available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup available_repos.each do |save_to_repository_id, build_for_platform_id| - next if locked_rep_and_pl.include?("#{save_to_repository_id}-#{build_for_platform_id}") + next if RepositoryStatus.not_ready.where(:repository_id => save_to_repository_id, :platform_id => build_for_platform_id).exists? break if counter > @workers_count counter += 1 if create_rpm_build_task(save_to_repository_id, build_for_platform_id) end @@ -271,6 +247,9 @@ module AbfWorker save_to_repository = Repository.find save_to_repository_id # Checks mirror sync status return false if save_to_repository.repo_lock_file_exists? + + repository_status = save_to_repository.find_or_create_by_platform_id(build_for_platform_id) + return false unless repository_status.publish save_to_platform = save_to_repository.platform build_for_platform = Platform.find build_for_platform_id @@ -291,7 +270,6 @@ module AbfWorker 'BUILD_FOR_PLATFORM' => build_for_platform.name }.map{ |k, v| "#{k}=#{v}" }.join(' ') - lock_str = "#{save_to_repository_id}-#{build_for_platform_id}" options = { :id => (bl ? bl.id : Time.now.to_i), :cmd_params => cmd_params, @@ -304,7 +282,7 @@ module AbfWorker :repository => {:id => save_to_repository_id}, :type => :publish, :time_living => 9600, # 160 min - :extra => {:lock_str => lock_str} + :extra => {:repository_status_id => repository_status.id} } packages, build_list_ids, new_sources = self.class.packages_structure, [], {} @@ -324,9 +302,9 @@ module AbfWorker worker_queue, 'class' => worker_class, 'args' => [options.merge({ - :packages => packages, - :old_packages => old_packages, - :build_list_ids => build_list_ids, + :packages => packages, + :old_packages => old_packages, + :build_list_ids => build_list_ids, :projects_for_cleanup => projects_for_cleanup })] ) @@ -335,10 +313,52 @@ module AbfWorker @redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key end - @redis.lpush(LOCKED_REP_AND_PLATFORMS, lock_str) return true end + def create_tasks_for_regenerate_metadata_for_software_center + Platfor.main.waiting_for_regeneration.each do |platform| + repos = platform.repositories + repos.repositories.each(&:regenerate) + statuses = RepositoryStatus.where(:platform_id => platform.id, :repository_id => repos.map(&:id)) + next if repos.find{ |r| r.repo_lock_file_exists? } + next if statuses.find{ |s| !s.can_start_regeneration? } + + build_for_platform = repository_statuses.platform + cmd_params = { + 'RELEASED' => platform.released, + 'REPOSITORY_NAME' => rep.name, + 'TYPE' => platform.distrib_type, + 'REGENERATE_PLATFORM_METADATA' => true, + 'SAVE_TO_PLATFORM' => platform.name, + 'BUILD_FOR_PLATFORM' => platform.name + }.map{ |k, v| "#{k}=#{v}" }.join(' ') + + statuses.each(&:start_regeneration) + platform.regenerate + Resque.push( + 'publish_worker_default', + 'class' => 'AbfWorker::PublishWorkerDefault', + 'args' => [{ + :id => Time.now.to_i, + :cmd_params => cmd_params, + :platform => { + :platform_path => "#{rep.platform.path}/repository", + :type => platform.distrib_type, + :name => platform.name, + :arch => 'x86_64' + }, + :repository => {:id => rep.id}, + :type => :publish, + :time_living => 9600, # 160 min + :skip_feedback => true, + :extra => {:repository_status_id => statuses.map(&:id), :regenerate_platform => true} + }] + ) + + end + end + def create_tasks_for_repository_regenerate_metadata repository_statuses = RepositoryStatus.for_regeneration.includes(:repository => :platform) repository_statuses.each do |repository_status| diff --git a/lib/abf_worker/publish_observer.rb b/lib/abf_worker/publish_observer.rb index 3a4a52d46..c079c149c 100644 --- a/lib/abf_worker/publish_observer.rb +++ b/lib/abf_worker/publish_observer.rb @@ -9,13 +9,21 @@ module AbfWorker def perform return if status == STARTED # do nothing when publication started - repository_status = RepositoryStatus.where(:id => options['extra']['repository_status_id']).first + repository_statuses = RepositoryStatus.where(:id => options['extra']['repository_status_ids']) begin if options['extra']['regenerate'] # Regenerate metadata - if repository_status - repository_status.last_regenerated_at = Time.now.utc + last_regenerated_at = Time.now.utc + repository_statuses.each do |repository_status| + repository_status.last_regenerated_at = last_regenerated_at repository_status.last_regenerated_status = status end + elsif options['extra']['regenerate_platform'] # Regenerate metadata for Software Center + platform = repository_statuses.first.try(:platform) + if platform + platform.last_regenerated_at = Time.now.utc + platform.last_regenerated_status = status + platform.ready + end elsif options['extra']['create_container'] # Container has been created case status when COMPLETED @@ -28,7 +36,7 @@ module AbfWorker update_rpm_builds end ensure - repository_status.ready if repository_status + repository_statuses.map(&:ready) if repository_statuses.present? end end @@ -54,8 +62,6 @@ module AbfWorker when FAILED, CANCELED AbfWorker::BuildListsPublishTaskManager.cleanup_failed options['projects_for_cleanup'] end - - AbfWorker::BuildListsPublishTaskManager.unlock_rep_and_platform options['extra']['lock_str'] end def update_results(build_list = subject)