Software
/
rosa-build
mirror of https://github.com/OpenMandrivaSoftware/rosa-build.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #989 from warpc/960-remove-packages-from-repository 

[refs #960]: Remove packages from repository when remove project from repository.
This commit is contained in:
Vladimir Sharshov 2013-03-01 09:55:17 -08:00
parent febf6a2a23 1dca61afd1
commit 6c74613a2a
2 changed files with 70 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/models/project.rb
View File

@ -10,13 +10,14 @@ class Project < ActiveRecord::Base
has_many :issues, :dependent => :destroy has_many :issues, :dependent => :destroy
has_many :pull_requests, :dependent => :destroy, :foreign_key => 'to_project_id' has_many :pull_requests, :dependent => :destroy, :foreign_key => 'to_project_id'
has_many :labels, :dependent => :destroy has_many :labels, :dependent => :destroy
has_many :build_lists, :dependent => :destroy
has_many :project_imports, :dependent => :destroy has_many :project_imports, :dependent => :destroy
has_many :project_to_repositories, :dependent => :destroy has_many :project_to_repositories, :dependent => :destroy
has_many :repositories, :through => :project_to_repositories has_many :repositories, :through => :project_to_repositories
has_many :project_tags, :dependent => :destroy has_many :project_tags, :dependent => :destroy
has_many :build_lists, :dependent => :destroy
has_many :relations, :as => :target, :dependent => :destroy has_many :relations, :as => :target, :dependent => :destroy
has_many :collaborators, :through => :relations, :source => :actor, :source_type => 'User' has_many :collaborators, :through => :relations, :source => :actor, :source_type => 'User'
has_many :groups, :through => :relations, :source => :actor, :source_type => 'Group' has_many :groups, :through => :relations, :source => :actor, :source_type => 'Group'

120
lib/abf_worker/build_lists_publish_task_manager.rb
View File

@ -9,6 +9,7 @@ module AbfWorker
LOCKED_REPOSITORIES LOCKED_REPOSITORIES
LOCKED_REP_AND_PLATFORMS LOCKED_REP_AND_PLATFORMS
LOCKED_BUILD_LISTS LOCKED_BUILD_LISTS
PACKAGES_FOR_CLEANUP
REGENERATE_METADATA).each do |kind| REGENERATE_METADATA).each do |kind|
const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}" const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}"
end end
@ -29,15 +30,18 @@ module AbfWorker
if repository.platform.personal? if repository.platform.personal?
Platform.main.each do |main_platform| Platform.main.each do |main_platform|
redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{main_platform.id}" redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{main_platform.id}"
gather_old_packages project.id, repository.id, main_platform.id
end end
else else
redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{repository.platform.id}" redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{repository.platform.id}"
gather_old_packages project.id, repository.id, repository.platform.id
end end
end end
def cleanup_completed(projects_for_cleanup) def cleanup_completed(projects_for_cleanup)
projects_for_cleanup.each do |key| projects_for_cleanup.each do |key|
redis.lrem LOCKED_PROJECTS_FOR_CLEANUP, 0, key redis.lrem LOCKED_PROJECTS_FOR_CLEANUP, 0, key
redis.hdel PACKAGES_FOR_CLEANUP, key
end end
end end
@ -112,10 +116,46 @@ module AbfWorker
}] }]
) )
end end
def gather_old_packages(project_id, repository_id, platform_id)
build_lists_for_cleanup = []
Arch.pluck(:id).each do |arch_id|
bl = BuildList.where(:project_id => project_id).
where(:new_core => true, :status => BuildList::BUILD_PUBLISHED).
where(:save_to_repository_id => repository_id).
where(:build_for_platform_id => platform_id).
where(:arch_id => arch_id).
order(:updated_at).first
build_lists_for_cleanup << bl if bl
end
old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
build_lists_for_cleanup.each do |bl|
bl.last_published.includes(:packages).limit(2).each{ |old_bl|
fill_packages(old_bl, old_packages, :fullname)
}
end
redis.hset PACKAGES_FOR_CLEANUP, "#{project_id}-#{repository_id}-#{platform_id}", old_packages.to_json
end
def fill_packages(bl, results_map, field = :sha1)
results_map[:sources] |= bl.packages.by_package_type('source').pluck(field).compact if field != :sha1
binaries = bl.packages.by_package_type('binary').pluck(field).compact
arch = bl.arch.name.to_sym
results_map[:binaries][arch] |= binaries
# Publish/remove i686 RHEL packages into/from x86_64
if arch == :i586 && bl.build_for_platform.distrib_type == 'rhel' && bl.project.publish_i686_into_x86_64?
results_map[:binaries][:x86_64] |= binaries
end
end
end end
private private
def locked_repositories def locked_repositories
@redis.lrange LOCKED_REPOSITORIES, 0, -1 @redis.lrange LOCKED_REPOSITORIES, 0, -1
end end
@ -200,80 +240,69 @@ module AbfWorker
projects_for_cleanup = @redis.lrange(PROJECTS_FOR_CLEANUP, 0, -1). projects_for_cleanup = @redis.lrange(PROJECTS_FOR_CLEANUP, 0, -1).
select{ |k| k =~ /#{save_to_repository_id}\-#{build_for_platform_id}$/ } select{ |k| k =~ /#{save_to_repository_id}\-#{build_for_platform_id}$/ }
build_lists_for_cleanup = [] old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
projects_for_cleanup.each do |key| projects_for_cleanup.each do |key|
pr, rep, pl = *key.split('-') @redis.lrem PROJECTS_FOR_CLEANUP, 0, key
added = false packages = @redis.hget PACKAGES_FOR_CLEANUP, key
Arch.pluck(:id).each do |arch_id| next unless packages
bl = BuildList.where(:project_id => pr). packages = JSON.parse packages
where(:new_core => true, :status => BuildList::BUILD_PUBLISHED). old_packages[:sources] |= packages['sources']
where(:save_to_repository_id => save_to_repository_id). [:x86_64, :i586].each do |arch|
where(:build_for_platform_id => build_for_platform_id). old_packages[:binaries][arch] |= packages['binaries'][arch.to_s]
where(:arch_id => arch_id).
order(:updated_at).first
if bl
build_lists_for_cleanup << bl
added = true
end end
end end
# No packages for removing
@redis.lrem(PROJECTS_FOR_CLEANUP, 0, key) unless added
end
bl = build_lists.first || build_lists_for_cleanup.first bl = build_lists.first
return false unless bl return false if !bl && old_packages[:sources].empty?
platform_path = "#{bl.save_to_platform.path}/repository" save_to_repository = Repository.find save_to_repository_id
if bl.save_to_platform.personal? save_to_platform = save_to_repository.platform
platform_path << '/' << bl.build_for_platform.name build_for_platform = Platform.find build_for_platform_id
platform_path = "#{save_to_platform.path}/repository"
if save_to_platform.personal?
platform_path << '/' << build_for_platform.name
system "mkdir -p #{platform_path}" system "mkdir -p #{platform_path}"
end end
worker_queue = bl.worker_queue_with_priority("publish_worker") worker_queue = bl ? bl.worker_queue_with_priority("publish_worker") : 'publish_worker_default'
worker_class = bl.worker_queue_class("AbfWorker::PublishWorker") worker_class = bl ? bl.worker_queue_class("AbfWorker::PublishWorker") : 'AbfWorker::PublishWorkerDefault'
distrib_type = bl.build_for_platform.distrib_type distrib_type = build_for_platform.distrib_type
cmd_params = { cmd_params = {
'RELEASED' => bl.save_to_platform.released, 'RELEASED' => save_to_platform.released,
'REPOSITORY_NAME' => bl.save_to_repository.name, 'REPOSITORY_NAME' => save_to_repository.name,
'TYPE' => distrib_type 'TYPE' => distrib_type
}.map{ |k, v| "#{k}=#{v}" }.join(' ') }.map{ |k, v| "#{k}=#{v}" }.join(' ')
lock_str = "#{save_to_repository_id}-#{build_for_platform_id}" lock_str = "#{save_to_repository_id}-#{build_for_platform_id}"
options = { options = {
:id => bl.id, :id => (bl ? bl.id : Time.now.to_i),
:arch => bl.arch.name, :arch => (bl ? bl.arch.name : 'x86_64'),
:distrib_type => distrib_type, :distrib_type => distrib_type,
:cmd_params => cmd_params, :cmd_params => cmd_params,
:platform => {:platform_path => platform_path}, :platform => {:platform_path => platform_path},
:repository => {:id => bl.save_to_repository.id}, :repository => {:id => save_to_repository_id},
:type => :publish, :type => :publish,
:time_living => 9600, # 160 min :time_living => 9600, # 160 min
:extra => {:lock_str => lock_str} :extra => {:lock_str => lock_str}
} }
packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}} packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
old_packages = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
build_list_ids = [] build_list_ids = []
new_sources = {} new_sources = {}
build_lists.each do |bl| build_lists.each do |bl|
# remove duplicates of sources for different arches # remove duplicates of sources for different arches
bl.packages.by_package_type('source').each{ |s| new_sources["#{s.fullname}"] = s.sha1 } bl.packages.by_package_type('source').each{ |s| new_sources["#{s.fullname}"] = s.sha1 }
fill_packages(bl, packages) self.class.fill_packages(bl, packages)
bl.last_published.includes(:packages).limit(5).each{ |old_bl| bl.last_published.includes(:packages).limit(2).each{ |old_bl|
fill_packages(old_bl, old_packages, :fullname) self.class.fill_packages(old_bl, old_packages, :fullname)
} }
build_list_ids << bl.id build_list_ids << bl.id
@redis.lpush(LOCKED_BUILD_LISTS, bl.id) @redis.lpush(LOCKED_BUILD_LISTS, bl.id)
end end
packages[:sources] = new_sources.values.compact packages[:sources] = new_sources.values.compact
build_lists_for_cleanup.each do |bl|
bl.last_published.includes(:packages).limit(5).each{ |old_bl|
fill_packages(old_bl, old_packages, :fullname)
}
end
Resque.push( Resque.push(
worker_queue, worker_queue,
'class' => worker_class, 'class' => worker_class,
@ -286,7 +315,6 @@ module AbfWorker
) )
projects_for_cleanup.each do |key| projects_for_cleanup.each do |key|
@redis.lrem PROJECTS_FOR_CLEANUP, 0, key
@redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key @redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key
end end
@ -294,18 +322,6 @@ module AbfWorker
return true return true
end end
def fill_packages(bl, results_map, field = :sha1)
results_map[:sources] |= bl.packages.by_package_type('source').pluck(field).compact if field != :sha1
binaries = bl.packages.by_package_type('binary').pluck(field).compact
arch = bl.arch.name.to_sym
results_map[:binaries][arch] |= binaries
# Publish/remove i686 RHEL packages into/from x86_64
if arch == :i586 && bl.build_for_platform.distrib_type == 'rhel' && bl.project.publish_i686_into_x86_64?
results_map[:binaries][:x86_64] |= binaries
end
end
def create_tasks_for_repository_regenerate_metadata def create_tasks_for_repository_regenerate_metadata
worker_queue = 'publish_worker_default' worker_queue = 'publish_worker_default'
worker_class = 'AbfWorker::PublishWorkerDefault' worker_class = 'AbfWorker::PublishWorkerDefault'