Software
/
rosa-build
mirror of https://github.com/OpenMandrivaSoftware/rosa-build.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

#403: add filter for build_lists without packages

This commit is contained in:
Vokhmin Alexey V 2014-06-20 23:23:31 +04:00
parent 3d08c050ad
commit dd259714f6
6 changed files with 72 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/jobs/publish_task_manager_job.rb
View File

@ -5,7 +5,7 @@ class PublishTaskManagerJob
regenerate_metadata_for_software_center
resign_repositories
regenerate_metadata
AbfWorkerService::Rpm.new.publish!
AbfWorkerService::Rpm.publish!
end
protected

1
app/models/build_list.rb
View File

@ -89,6 +89,7 @@ class BuildList < ActiveRecord::Base
# %w(PROJECT_SOURCE_ERROR 6),
# %w(DEPENDENCIES_ERROR 555),
%w(BUILD_ERROR 666),
%w(PACKAGES_FAIL 777),
%w(BUILD_STARTED 3000),
%w(BUILD_CANCELED 5000),
%w(WAITING_FOR_RESPONSE 4000),

80
app/services/abf_worker_service/rpm.rb
View File

@ -3,14 +3,20 @@ module AbfWorkerService
WORKERS_COUNT = APP_CONFIG['abf_worker']['publish_workers_count']
def publish!
attr_accessor :save_to_repository_id, :build_for_platform_id, :testing
def initialize(save_to_repository_id, build_for_platform_id, testing)
@save_to_repository_id = save_to_repository_id
@build_for_platform_id = build_for_platform_id
@testing = testing
end
def self.publish!
build_rpms
build_rpms(true)
end
protected
def build_rpms(testing = false)
def self.build_rpms(testing = false)
available_repos = BuildList.
select('MIN(updated_at) as min_updated_at, save_to_repository_id, build_for_platform_id').
where(new_core: true, status: (testing ? BuildList::BUILD_PUBLISH_INTO_TESTING : BuildList::BUILD_PUBLISH)).
@ -40,19 +46,23 @@ module AbfWorkerService
available_repos.each do |save_to_repository_id, build_for_platform_id|
next if RepositoryStatus.not_ready.where(repository_id: save_to_repository_id, platform_id: build_for_platform_id).exists?
break if counter > WORKERS_COUNT
counter += 1 if create_rpm_build_task(save_to_repository_id, build_for_platform_id, testing)
service = AbfWorkerService::Rpm.new(
save_to_repository_id,
build_for_platform_id,
testing
)
counter += 1 if service.create
end
end
def create_rpm_build_task(save_to_repository_id, build_for_platform_id, testing)
def create
key = "#{save_to_repository_id}-#{build_for_platform_id}"
projects_for_cleanup = Redis.current.lrange(PROJECTS_FOR_CLEANUP, 0, -1).select do |k|
(testing && k =~ /^testing-[\d]+-#{key}$/) || (!testing && k =~ /^[\d]+-#{key}$/)
end
prepare_build_lists(projects_for_cleanup, save_to_repository_id, testing)
prepare_build_lists(projects_for_cleanup)
build_lists = find_build_lists(build_for_platform_id, save_to_repository_id, testing)
old_packages = packages_structure
projects_for_cleanup.each do |key|
@ -74,7 +84,7 @@ module AbfWorkerService
end
build_lists_for_cleanup_from_testing ||= []
bl = build_lists.first
bl = build_lists[0]
return false if !bl && old_packages[:sources].empty? && old_packages[:binaries].values.flatten.empty?
save_to_repository = ::Repository.find(save_to_repository_id)
@ -121,7 +131,7 @@ module AbfWorkerService
}
}
packages, build_list_ids, new_sources = fill_in_packages(build_lists, testing)
packages, build_list_ids, new_sources = fill_in_packages
push(options.merge({
packages: packages,
old_packages: old_packages,
@ -129,11 +139,13 @@ module AbfWorkerService
projects_for_cleanup: projects_for_cleanup
}))
lock_projects(projects_for_cleanup)
cleanup(save_to_repository_id, build_for_platform_id, build_lists_for_cleanup_from_testing)
cleanup(build_lists_for_cleanup_from_testing)
return true
end
def fill_in_packages(build_lists, testing)
protected
def fill_in_packages
packages, build_list_ids, new_sources = packages_structure, [], {}
build_lists.each do |bl|
# remove duplicates of sources for different arches
@ -161,7 +173,7 @@ module AbfWorkerService
end
end
def cleanup(save_to_repository_id, build_for_platform_id, build_lists_for_cleanup_from_testing)
def cleanup(build_lists_for_cleanup_from_testing)
rep_pl = "#{save_to_repository_id}-#{build_for_platform_id}"
r_key = "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{rep_pl}"
build_lists_for_cleanup_from_testing.each do |key|
@ -180,7 +192,7 @@ module AbfWorkerService
)
end
def prepare_build_lists(projects_for_cleanup, save_to_repository_id, testing)
def prepare_build_lists(projects_for_cleanup)
# We should not to publish new builds into repository
# if project of builds has been removed from repository.
BuildList.where(
@ -190,15 +202,37 @@ module AbfWorkerService
).update_all(status: BuildList::FAILED_PUBLISH)
end
def find_build_lists(build_for_platform_id, save_to_repository_id, testing)
build_lists = BuildList.
where(new_core: true, status: (testing ? BuildList::BUILD_PUBLISH_INTO_TESTING : BuildList::BUILD_PUBLISH)).
where(save_to_repository_id: save_to_repository_id).
where(build_for_platform_id: build_for_platform_id).
order(:updated_at)
locked_ids = Redis.current.lrange(LOCKED_BUILD_LISTS, 0, -1)
build_lists = build_lists.where('build_lists.id NOT IN (?)', locked_ids) if locked_ids.present?
build_lists.limit(150)
def build_lists
@build_lists ||= begin
build_lists = BuildList.
where(new_core: true, status: (testing ? BuildList::BUILD_PUBLISH_INTO_TESTING : BuildList::BUILD_PUBLISH)).
where(save_to_repository_id: save_to_repository_id).
where(build_for_platform_id: build_for_platform_id).
order(:updated_at)
locked_ids = Redis.current.lrange(LOCKED_BUILD_LISTS, 0, -1)
build_lists = build_lists.where('build_lists.id NOT IN (?)', locked_ids) if locked_ids.present?
build_lists = build_lists.limit(150)
filter_build_lists_without_packages(build_lists.to_a)
end
end
def filter_build_lists_without_packages(build_lists)
ids = []
build_lists = build_lists.select do |build_list|
sha1 = build_list.packages.pluck(:sha1).find do |sha1|
!FileStoreService::File.new(sha1: sha1).exist?
end
if sha1.present?
ids << build_list.id
false
else
true
end
end
BuildList.where(id: ids).update_all(status: BuildList::PACKAGES_FAIL)
build_lists
end
end

1
config/locales/models/build_list.en.yml
View File

@ -161,6 +161,7 @@ en:
build_publish: Build is being published
failed_publish: Publishing error
dependencies_fail: Dependences not found
packages_fail: Packages not found
waiting_for_response: Waiting for response
build_pending: Build pending
build_canceling: Build is canceling

1
config/locales/models/build_list.ru.yml
View File

@ -160,6 +160,7 @@ ru:
build_publish: публикуется
failed_publish: ошибка публикации
dependencies_fail: зависимости не найдены
packages_fail: пакеты не найдены
waiting_for_response: ожидает ответа
build_pending: ожидает сборку
build_canceled: сборка отменена

20
spec/services/abf_worker_service/rpm_spec.rb
View File

@ -16,6 +16,8 @@ describe AbfWorkerService::Rpm do
stub_symlink_methods
end
subject { AbfWorkerService::Rpm }
context '#publish!' do
before do
@ -30,7 +32,7 @@ describe AbfWorkerService::Rpm do
it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
subject.publish!
@redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
@ -48,7 +50,7 @@ describe AbfWorkerService::Rpm do
%w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind|
it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
@ -59,7 +61,7 @@ describe AbfWorkerService::Rpm do
end
it "ensures that 'locked build lists' has only one item" do
queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1)
queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(1).item
queue.should include(build_list.id.to_s)
end
@ -91,7 +93,7 @@ describe AbfWorkerService::Rpm do
subject.publish!
subject.publish!
@redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1).should have(4).items
@redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1).should have(4).items
end
it "ensures that new tasks for publishing has been created" do
@ -120,7 +122,7 @@ describe AbfWorkerService::Rpm do
it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
subject.publish!
subject.publish!
@redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
@ -133,7 +135,7 @@ describe AbfWorkerService::Rpm do
it "ensures that 'locked build lists' has 2 items" do
subject.publish!
subject.publish!
queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1)
queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(2).item
queue.should include(build_list.id.to_s, build_list2.id.to_s)
end
@ -170,7 +172,7 @@ describe AbfWorkerService::Rpm do
subject.publish!
subject.publish!
queue = @redis_instance.lrange(AbfWorkerService::Rpm::PROJECTS_FOR_CLEANUP, 0, -1)
queue = @redis_instance.lrange(subject::PROJECTS_FOR_CLEANUP, 0, -1)
queue.should have(1).item
queue.should include("testing-#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}")
end
@ -186,7 +188,7 @@ describe AbfWorkerService::Rpm do
subject.publish!
subject.publish!
queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1)
queue = @redis_instance.lrange(subject::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1)
queue.should have(1).item
queue.should include("#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}")
end
@ -202,7 +204,7 @@ describe AbfWorkerService::Rpm do
subject.publish!
subject.publish!
queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1)
queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(1).item
queue.should include(build_list.id.to_s)
end