Software
/
rosa-build
mirror of https://github.com/OpenMandrivaSoftware/rosa-build.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

#276: added ability to regenerate metadata for Software Center

This commit is contained in:
Vokhmin Alexey V 2013-08-23 18:24:01 +04:00
parent 183799d9f0
commit 7bdab43817
4 changed files with 74 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/models/platform.rb
View File

@ -63,6 +63,7 @@ class Platform < ActiveRecord::Base
scope :by_type, lambda {|type| where(:platform_type => type) if type.present?}
scope :main, by_type('main')
scope :personal, by_type('personal')
scope :waiting_for_regeneration, where(:status => WAITING_FOR_REGENERATION)
accepts_nested_attributes_for :platform_arch_settings, :allow_destroy => true
attr_accessible :name, :distrib_type, :parent_platform_id, :platform_type, :owner, :visibility, :description, :released, :platform_arch_settings_attributes
@ -77,7 +78,7 @@ class Platform < ActiveRecord::Base
event :regenerate do
transition :waiting_for_regeneration => :regenerating
transition :ready => :waiting_for_regeneration
transition :ready => :waiting_for_regeneration, :if => lambda{ |p| p.main? }
end
HUMAN_STATUSES.each do |code,name|

2
app/models/repository_status.rb
View File

@ -39,6 +39,7 @@ class RepositoryStatus < ActiveRecord::Base
scope :platform_ready, where(:platform => {:status => READY}).joins(:platform)
scope :for_regeneration, platform_ready.where(:status => WAITING_FOR_REGENERATION)
scope :for_resign, platform_ready.where(:status => [WAITING_FOR_RESIGN, WAITING_FOR_RESIGN_AND_REGENERATION])
scope :not_ready, where('repository_statuses.status != ?', READY)
state_machine :status, :initial => :ready do
event :ready do
@ -58,6 +59,7 @@ class RepositoryStatus < ActiveRecord::Base
event :start_regeneration do
transition :waiting_for_regeneration => :regenerating
transition :waiting_for_resign_and_regeneration => :waiting_for_resign_after_regeneration
end
event :resign do

96
lib/abf_worker/build_lists_publish_task_manager.rb
View File

@ -3,12 +3,8 @@ module AbfWorker
class BuildListsPublishTaskManager
REDIS_MAIN_KEY = 'abf-worker::build-lists-publish-task-manager::'
# LOCKED_REP_AND_PLATFORMS: ['save_to_repository_id-build_for_platform_id', ...]
%w(RESIGN_REPOSITORIES
PROJECTS_FOR_CLEANUP
%w(PROJECTS_FOR_CLEANUP
LOCKED_PROJECTS_FOR_CLEANUP
LOCKED_REPOSITORIES
LOCKED_REP_AND_PLATFORMS
LOCKED_BUILD_LISTS
PACKAGES_FOR_CLEANUP).each do |kind|
const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}"
@ -20,6 +16,7 @@ module AbfWorker
end
def run
create_tasks_for_regenerate_metadata_for_software_center
create_tasks_for_resign_repositories
create_tasks_for_repository_regenerate_metadata
create_tasks_for_build_rpms
@ -52,18 +49,10 @@ module AbfWorker
end
end
def unlock_repository(repository_id)
redis.lrem LOCKED_REPOSITORIES, 0, repository_id
end
def unlock_build_list(build_list)
redis.lrem LOCKED_BUILD_LISTS, 0, build_list.id
end
def unlock_rep_and_platform(lock_str)
redis.lrem LOCKED_REP_AND_PLATFORMS, 0, lock_str
end
def packages_structure
structure = {:sources => [], :binaries => {}}
Arch.pluck(:name).each{ |name| structure[:binaries][name.to_sym] = [] }
@ -156,11 +145,6 @@ module AbfWorker
private
def locked_repositories
@redis.lrange LOCKED_REPOSITORIES, 0, -1
end
def create_tasks_for_resign_repositories
repository_statuses = RepositoryStatus.for_resign.includes(:repository => :platform)
repository_statuses.each do |repository_status|
@ -205,27 +189,19 @@ module AbfWorker
order(:min_updated_at).
limit(@workers_count * 2) # because some repos may be locked
locked_rep = locked_repositories
locked_rep = RepositoryStatus.not_ready.joins(:platform).
where(:platforms => {:platform_type => 'main'}).pluck(:repository_id)
available_repos = available_repos.where('save_to_repository_id NOT IN (?)', locked_rep) unless locked_rep.empty?
counter = 1
# looks like:
# ['save_to_repository_id-build_for_platform_id', ...]
locked_rep_and_pl = @redis.lrange(LOCKED_REP_AND_PLATFORMS, 0, -1)
for_cleanup = @redis.lrange(PROJECTS_FOR_CLEANUP, 0, -1).map do |key|
pr, rep, pl = *key.split('-')
if locked_rep.present? && locked_rep.include?(rep)
nil
else
[rep.to_i, pl.to_i]
end
locked_rep.present? && locked_rep.include?(rep) ? nil : [rep.to_i, pl.to_i]
end.compact
available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup
counter = 1
available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup
available_repos.each do |save_to_repository_id, build_for_platform_id|
next if locked_rep_and_pl.include?("#{save_to_repository_id}-#{build_for_platform_id}")
next if RepositoryStatus.not_ready.where(:repository_id => save_to_repository_id, :platform_id => build_for_platform_id).exists?
break if counter > @workers_count
counter += 1 if create_rpm_build_task(save_to_repository_id, build_for_platform_id)
end
@ -271,6 +247,9 @@ module AbfWorker
save_to_repository = Repository.find save_to_repository_id
# Checks mirror sync status
return false if save_to_repository.repo_lock_file_exists?
repository_status = save_to_repository.find_or_create_by_platform_id(build_for_platform_id)
return false unless repository_status.publish
save_to_platform = save_to_repository.platform
build_for_platform = Platform.find build_for_platform_id
@ -291,7 +270,6 @@ module AbfWorker
'BUILD_FOR_PLATFORM' => build_for_platform.name
}.map{ |k, v| "#{k}=#{v}" }.join(' ')
lock_str = "#{save_to_repository_id}-#{build_for_platform_id}"
options = {
:id => (bl ? bl.id : Time.now.to_i),
:cmd_params => cmd_params,
@ -304,7 +282,7 @@ module AbfWorker
:repository => {:id => save_to_repository_id},
:type => :publish,
:time_living => 9600, # 160 min
:extra => {:lock_str => lock_str}
:extra => {:repository_status_id => repository_status.id}
}
packages, build_list_ids, new_sources = self.class.packages_structure, [], {}
@ -324,9 +302,9 @@ module AbfWorker
worker_queue,
'class' => worker_class,
'args' => [options.merge({
:packages => packages,
:old_packages => old_packages,
:build_list_ids => build_list_ids,
:packages => packages,
:old_packages => old_packages,
:build_list_ids => build_list_ids,
:projects_for_cleanup => projects_for_cleanup
})]
)
@ -335,10 +313,52 @@ module AbfWorker
@redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key
end
@redis.lpush(LOCKED_REP_AND_PLATFORMS, lock_str)
return true
end
def create_tasks_for_regenerate_metadata_for_software_center
Platfor.main.waiting_for_regeneration.each do |platform|
repos = platform.repositories
repos.repositories.each(&:regenerate)
statuses = RepositoryStatus.where(:platform_id => platform.id, :repository_id => repos.map(&:id))
next if repos.find{ |r| r.repo_lock_file_exists? }
next if statuses.find{ |s| !s.can_start_regeneration? }
build_for_platform = repository_statuses.platform
cmd_params = {
'RELEASED' => platform.released,
'REPOSITORY_NAME' => rep.name,
'TYPE' => platform.distrib_type,
'REGENERATE_PLATFORM_METADATA' => true,
'SAVE_TO_PLATFORM' => platform.name,
'BUILD_FOR_PLATFORM' => platform.name
}.map{ |k, v| "#{k}=#{v}" }.join(' ')
statuses.each(&:start_regeneration)
platform.regenerate
Resque.push(
'publish_worker_default',
'class' => 'AbfWorker::PublishWorkerDefault',
'args' => [{
:id => Time.now.to_i,
:cmd_params => cmd_params,
:platform => {
:platform_path => "#{rep.platform.path}/repository",
:type => platform.distrib_type,
:name => platform.name,
:arch => 'x86_64'
},
:repository => {:id => rep.id},
:type => :publish,
:time_living => 9600, # 160 min
:skip_feedback => true,
:extra => {:repository_status_id => statuses.map(&:id), :regenerate_platform => true}
}]
)
end
end
def create_tasks_for_repository_regenerate_metadata
repository_statuses = RepositoryStatus.for_regeneration.includes(:repository => :platform)
repository_statuses.each do |repository_status|

18
lib/abf_worker/publish_observer.rb
View File

@ -9,13 +9,21 @@ module AbfWorker
def perform
return if status == STARTED # do nothing when publication started
repository_status = RepositoryStatus.where(:id => options['extra']['repository_status_id']).first
repository_statuses = RepositoryStatus.where(:id => options['extra']['repository_status_ids'])
begin
if options['extra']['regenerate'] # Regenerate metadata
if repository_status
repository_status.last_regenerated_at = Time.now.utc
last_regenerated_at = Time.now.utc
repository_statuses.each do |repository_status|
repository_status.last_regenerated_at = last_regenerated_at
repository_status.last_regenerated_status = status
end
elsif options['extra']['regenerate_platform'] # Regenerate metadata for Software Center
platform = repository_statuses.first.try(:platform)
if platform
platform.last_regenerated_at = Time.now.utc
platform.last_regenerated_status = status
platform.ready
end
elsif options['extra']['create_container'] # Container has been created
case status
when COMPLETED
@ -28,7 +36,7 @@ module AbfWorker
update_rpm_builds
end
ensure
repository_status.ready if repository_status
repository_statuses.map(&:ready) if repository_statuses.present?
end
end
@ -54,8 +62,6 @@ module AbfWorker
when FAILED, CANCELED
AbfWorker::BuildListsPublishTaskManager.cleanup_failed options['projects_for_cleanup']
end
AbfWorker::BuildListsPublishTaskManager.unlock_rep_and_platform options['extra']['lock_str']
end
def update_results(build_list = subject)