#403: added specs for AbfWorkerService::Rpm, updated travis.sh

This commit is contained in:
Vokhmin Alexey V 2014-06-20 01:18:53 +04:00
parent d67cf144e7
commit 8b90aec335
5 changed files with 218 additions and 381 deletions

View File

@ -50,7 +50,7 @@ module AbfWorkerService
(testing && k =~ /^testing-[\d]+-#{key}$/) || (!testing && k =~ /^[\d]+-#{key}$/)
end
prepare_build_lists(projects_for_cleanup, save_to_repository_id)
prepare_build_lists(projects_for_cleanup, save_to_repository_id, testing)
build_lists = find_build_lists(build_for_platform_id, save_to_repository_id, testing)
old_packages = packages_structure
@ -77,7 +77,7 @@ module AbfWorkerService
bl = build_lists.first
return false if !bl && old_packages[:sources].empty? && old_packages[:binaries].values.flatten.empty?
save_to_repository = Repository.find save_to_repository_id
save_to_repository = ::Repository.find(save_to_repository_id)
# Checks mirror sync status
return false if save_to_repository.repo_lock_file_exists? || !save_to_repository.platform.ready?
@ -180,7 +180,7 @@ module AbfWorkerService
)
end
def prepare_build_lists(projects_for_cleanup, save_to_repository_id)
def prepare_build_lists(projects_for_cleanup, save_to_repository_id, testing)
# We should not to publish new builds into repository
# if project of builds has been removed from repository.
BuildList.where(

View File

@ -1,199 +0,0 @@
module AbfWorker
class BuildListsPublishTaskManager
REDIS_MAIN_KEY = 'abf-worker::build-lists-publish-task-manager::'
%w(PROJECTS_FOR_CLEANUP
LOCKED_PROJECTS_FOR_CLEANUP
LOCKED_BUILD_LISTS
PACKAGES_FOR_CLEANUP
REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING
BUILD_LISTS_FOR_CLEANUP_FROM_TESTING).each do |kind|
const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}"
end
def initialize
@workers_count = APP_CONFIG['abf_worker']['publish_workers_count']
end
def run
# create_tasks_for_regenerate_metadata_for_software_center
# create_tasks_for_resign_repositories
# create_tasks_for_repository_regenerate_metadata
create_tasks_for_build_rpms
create_tasks_for_build_rpms true
end
private
def create_tasks_for_build_rpms(testing = false)
available_repos = BuildList.
select('MIN(updated_at) as min_updated_at, save_to_repository_id, build_for_platform_id').
where(new_core: true, status: (testing ? BuildList::BUILD_PUBLISH_INTO_TESTING : BuildList::BUILD_PUBLISH)).
group(:save_to_repository_id, :build_for_platform_id).
order('min_updated_at ASC').
limit(@workers_count * 2) # because some repos may be locked
locked_rep = RepositoryStatus.not_ready.joins(:platform).
where(platforms: {platform_type: 'main'}).pluck(:repository_id)
available_repos = available_repos.where('save_to_repository_id NOT IN (?)', locked_rep) unless locked_rep.empty?
for_cleanup = Redis.current.lrange(PROJECTS_FOR_CLEANUP, 0, -1).map do |key|
next if testing && key !~ /^testing-/
rep, pl = *key.split('-').last(2)
locked_rep.present? && locked_rep.include?(rep.to_i) ? nil : [rep.to_i, pl.to_i]
end.compact
for_cleanup_from_testing = Redis.current.smembers(REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING).map do |key|
next if Redis.current.scard("#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{key}") == 0
rep, pl = *key.split('-')
locked_rep.present? && locked_rep.include?(rep.to_i) ? nil : [rep.to_i, pl.to_i]
end.compact if testing
for_cleanup_from_testing ||= []
counter = 1
available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup | for_cleanup_from_testing
available_repos.each do |save_to_repository_id, build_for_platform_id|
next if RepositoryStatus.not_ready.where(repository_id: save_to_repository_id, platform_id: build_for_platform_id).exists?
break if counter > @workers_count
counter += 1 if create_rpm_build_task(save_to_repository_id, build_for_platform_id, testing)
end
end
def create_rpm_build_task(save_to_repository_id, build_for_platform_id, testing)
key = "#{save_to_repository_id}-#{build_for_platform_id}"
projects_for_cleanup = Redis.current.lrange(PROJECTS_FOR_CLEANUP, 0, -1).select do |k|
(testing && k =~ /^testing-[\d]+-#{key}$/) || (!testing && k =~ /^[\d]+-#{key}$/)
end
# We should not to publish new builds into repository
# if project of builds has been removed from repository.
BuildList.where(
project_id: projects_for_cleanup.map{ |k| k.split('-')[testing ? 1 : 0] }.uniq,
save_to_repository_id: save_to_repository_id,
status: [BuildList::BUILD_PUBLISH, BuildList::BUILD_PUBLISH_INTO_TESTING]
).update_all(status: BuildList::FAILED_PUBLISH)
build_lists = BuildList.
where(new_core: true, status: (testing ? BuildList::BUILD_PUBLISH_INTO_TESTING : BuildList::BUILD_PUBLISH)).
where(save_to_repository_id: save_to_repository_id).
where(build_for_platform_id: build_for_platform_id).
order(:updated_at)
locked_ids = Redis.current.lrange(LOCKED_BUILD_LISTS, 0, -1)
build_lists = build_lists.where('build_lists.id NOT IN (?)', locked_ids) unless locked_ids.empty?
build_lists = build_lists.limit(150)
old_packages = self.class.packages_structure
projects_for_cleanup.each do |key|
Redis.current.lrem PROJECTS_FOR_CLEANUP, 0, key
packages = Redis.current.hget PACKAGES_FOR_CLEANUP, key
next unless packages
packages = JSON.parse packages
old_packages[:sources] |= packages['sources']
Arch.pluck(:name).each do |arch|
old_packages[:binaries][arch.to_sym] |= packages['binaries'][arch] || []
end
end
if testing
build_lists_for_cleanup_from_testing = Redis.current.smembers("#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}")
BuildList.where(id: build_lists_for_cleanup_from_testing).each do |b|
self.class.fill_packages(b, old_packages, :fullname)
end if build_lists_for_cleanup_from_testing.present?
end
build_lists_for_cleanup_from_testing ||= []
bl = build_lists.first
return false if !bl && old_packages[:sources].empty? && old_packages[:binaries].values.flatten.empty?
save_to_repository = Repository.find save_to_repository_id
# Checks mirror sync status
return false if save_to_repository.repo_lock_file_exists? || !save_to_repository.platform.ready?
repository_status = save_to_repository.repository_statuses.find_or_create_by(platform_id: build_for_platform_id)
return false unless repository_status.publish
save_to_platform = save_to_repository.platform
build_for_platform = Platform.find build_for_platform_id
platform_path = "#{save_to_platform.path}/repository"
if save_to_platform.personal?
platform_path << '/' << build_for_platform.name
system "mkdir -p #{platform_path}"
end
distrib_type = build_for_platform.distrib_type
cmd_params = {
'RELEASED' => save_to_platform.released,
'REPOSITORY_NAME' => save_to_repository.name,
'TYPE' => distrib_type,
'SAVE_TO_PLATFORM' => save_to_platform.name,
'BUILD_FOR_PLATFORM' => build_for_platform.name,
'TESTING' => testing
}.map{ |k, v| "#{k}=#{v}" }.join(' ')
options = {
id: (bl ? bl.id : Time.now.to_i),
cmd_params: cmd_params,
main_script: 'build.sh',
rollback_script: 'rollback.sh',
platform: {
platform_path: platform_path,
type: distrib_type,
name: build_for_platform.name,
arch: (bl ? bl.arch.name : 'x86_64')
},
repository: {id: save_to_repository_id},
time_living: 9600, # 160 min
extra: {
repository_status_id: repository_status.id,
build_lists_for_cleanup_from_testing: build_lists_for_cleanup_from_testing
}
}
packages, build_list_ids, new_sources = self.class.packages_structure, [], {}
build_lists.each do |bl|
# remove duplicates of sources for different arches
bl.packages.by_package_type('source').each{ |s| new_sources["#{s.fullname}"] = s.sha1 }
self.class.fill_packages(bl, packages)
bl.last_published(testing).includes(:packages).limit(2).each{ |old_bl|
self.class.fill_packages(old_bl, old_packages, :fullname)
}
# TODO: do more flexible
# Removes old packages which already in the main repo
bl.last_published(false).includes(:packages).limit(3).each{ |old_bl|
self.class.fill_packages(old_bl, old_packages, :fullname)
} if testing
build_list_ids << bl.id
Redis.current.lpush(LOCKED_BUILD_LISTS, bl.id)
end
packages[:sources] = new_sources.values.compact
Resque.push(
'publish_worker_default',
'class' => 'AbfWorker::PublishWorkerDefault',
'args' => [options.merge({
packages: packages,
old_packages: old_packages,
build_list_ids: build_list_ids,
projects_for_cleanup: projects_for_cleanup
})]
)
projects_for_cleanup.each do |key|
Redis.current.lpush LOCKED_PROJECTS_FOR_CLEANUP, key
end
rep_pl = "#{save_to_repository_id}-#{build_for_platform_id}"
r_key = "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{rep_pl}"
build_lists_for_cleanup_from_testing.each do |key|
Redis.current.srem r_key, key
end
if Redis.current.scard(r_key) == 0
Redis.current.srem REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, rep_pl
end
return true
end
end
end

View File

@ -1,177 +0,0 @@
require 'spec_helper'
describe AbfWorker::BuildListsPublishTaskManager do
before(:all) do
@publish_workers_count = APP_CONFIG['abf_worker']['publish_workers_count']
APP_CONFIG['abf_worker']['publish_workers_count'] = 2
end
before do
stub_symlink_methods
FactoryGirl.create(:build_list)
end
subject { AbfWorker::BuildListsPublishTaskManager }
let(:build_list) { FactoryGirl.create(:build_list) }
context 'when no items for publishing' do
before { subject.new.run }
%w(PROJECTS_FOR_CLEANUP
LOCKED_PROJECTS_FOR_CLEANUP
LOCKED_BUILD_LISTS).each do |kind|
it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
%w(publish_worker_default publish_worker).each do |kind|
it "ensures that no tasks in '#{kind}' queue" do
@redis_instance.lrange(kind, 0, -1).should be_empty
end
end
end
context 'when one build_list for publishing' do
before do
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
2.times{ subject.new.run }
end
%w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind|
it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
it "ensures that repository_status has status publish" do
build_list.save_to_repository.repository_statuses.
find_by(platform_id: build_list.build_for_platform_id).publish?.
should be_true
end
it "ensures that 'locked build lists' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(1).item
queue.should include(build_list.id.to_s)
end
it "ensures that new task for publishing has been created" do
@redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(1).item
end
end
context 'grouping build lists for publishing into same repository' do
let(:build_list2) { FactoryGirl.create(:build_list,
new_core: true,
save_to_platform: build_list.save_to_platform,
save_to_repository: build_list.save_to_repository,
build_for_platform: build_list.build_for_platform
) }
before do
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
build_list2.update_column(:status, BuildList::BUILD_PUBLISH)
2.times{ subject.new.run }
end
%w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind|
it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
it "ensures that only one repository_status has status publish" do
RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(1).item
end
it "ensures that 'locked build lists' has 2 items" do
queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(2).item
queue.should include(build_list.id.to_s, build_list2.id.to_s)
end
it "ensures that new task for publishing has been created" do
@redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(1).item
end
end
context 'creates not more than 4 tasks for publishing' do
before do
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
4.times {
bl = FactoryGirl.create(:build_list, new_core: true)
bl.update_column(:status, BuildList::BUILD_PUBLISH)
}
2.times{ subject.new.run }
end
it "ensures that 4 repository_statuses have status publish" do
RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(4).items
end
it "ensures that 'locked build lists' has 4 items" do
@redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1).should have(4).items
end
it "ensures that new tasks for publishing has been created" do
@redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(4).items
end
end
context 'grouping build lists for publishing and tasks for removing project from repository' do
let(:build_list2) { FactoryGirl.create(:build_list,
new_core: true,
save_to_platform: build_list.save_to_platform,
save_to_repository: build_list.save_to_repository,
build_for_platform: build_list.build_for_platform
) }
let(:build_list3) { FactoryGirl.create(:build_list,
new_core: true,
save_to_platform: build_list.save_to_platform,
save_to_repository: build_list.save_to_repository,
build_for_platform: build_list.build_for_platform
) }
before do
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
build_list2.update_column(:status, BuildList::BUILD_PUBLISHED)
build_list3.update_column(:status, BuildList::BUILD_PUBLISHED)
ProjectToRepository.where(project_id: build_list3.project_id, repository_id: build_list3.save_to_repository_id).destroy_all
2.times{ subject.new.run }
end
it "ensures that no 'projects for cleanup' for main repo" do
queue = @redis_instance.lrange(subject::PROJECTS_FOR_CLEANUP, 0, -1)
queue.should have(1).item
queue.should include("testing-#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}")
end
it "ensures that only one repository_status has status publish" do
RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(1).item
end
it "ensures that 'locked projects for cleanup' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1)
queue.should have(1).item
queue.should include("#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}")
end
it "ensures that new task for publishing has been created" do
@redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(1).item
end
it "ensures that 'locked build lists' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(1).item
queue.should include(build_list.id.to_s)
end
end
after(:all) do
APP_CONFIG['abf_worker']['publish_workers_count'] = @publish_workers_count
FileUtils.rm_rf(APP_CONFIG['root_path'])
end
end

View File

@ -0,0 +1,213 @@
require 'spec_helper'
describe AbfWorkerService::Rpm do
let(:build_list) { FactoryGirl.create(:build_list) }
before(:all) do
@publish_workers_count = APP_CONFIG['abf_worker']['publish_workers_count']
APP_CONFIG['abf_worker']['publish_workers_count'] = 2
end
after(:all) do
APP_CONFIG['abf_worker']['publish_workers_count'] = @publish_workers_count
end
before do
stub_symlink_methods
end
context '#publish!' do
before do
FactoryGirl.create(:build_list)
end
context 'no items for publishing' do
%w(PROJECTS_FOR_CLEANUP
LOCKED_PROJECTS_FOR_CLEANUP
LOCKED_BUILD_LISTS).each do |kind|
it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
subject.publish!
@redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty
end
end
it 'ensures that no tasks' do
expect(Resque).to_not receive(:push)
subject.publish!
end
end
context 'when one build_list for publishing' do
before do
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
2.times{ subject.publish! }
end
%w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind|
it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty
end
end
it "ensures that repository_status has status publish" do
build_list.save_to_repository.repository_statuses.
find_by(platform_id: build_list.build_for_platform_id).publish?.
should be_true
end
it "ensures that 'locked build lists' has only one item" do
queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(1).item
queue.should include(build_list.id.to_s)
end
it "ensures that new task for publishing has been created" do
@redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(1).item
end
end
context 'creates not more than 4 tasks for publishing' do
before do
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
4.times do
bl = FactoryGirl.build(:build_list)
bl.status = BuildList::BUILD_PUBLISH
bl.save!
end
end
it "ensures that 4 repository_statuses have status publish" do
subject.publish!
subject.publish!
RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(4).items
end
it "ensures that 'locked build lists' has 4 items" do
subject.publish!
subject.publish!
@redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1).should have(4).items
end
it "ensures that new tasks for publishing has been created" do
expect(Resque).to receive(:push).exactly(4).times
subject.publish!
subject.publish!
end
end
context 'grouping build lists for publishing into same repository' do
let(:build_list2) { FactoryGirl.build(:build_list,
new_core: true,
save_to_platform: build_list.save_to_platform,
save_to_repository: build_list.save_to_repository,
build_for_platform: build_list.build_for_platform
) }
before do
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
build_list2.status = BuildList::BUILD_PUBLISH
build_list2.save!
end
%w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind|
it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
subject.publish!
subject.publish!
@redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty
end
end
it "ensures that only one repository_status has status publish" do
subject.publish!
subject.publish!
RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(1).item
end
it "ensures that 'locked build lists' has 2 items" do
subject.publish!
subject.publish!
queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(2).item
queue.should include(build_list.id.to_s, build_list2.id.to_s)
end
it "ensures that new task for publishing has been created" do
expect(Resque).to receive(:push).once
subject.publish!
subject.publish!
end
end
context 'grouping build lists for publishing and tasks for removing project from repository' do
let(:build_list2) { FactoryGirl.build(:build_list,
new_core: true,
save_to_platform: build_list.save_to_platform,
save_to_repository: build_list.save_to_repository,
build_for_platform: build_list.build_for_platform
) }
let(:build_list3) { FactoryGirl.build(:build_list,
new_core: true,
save_to_platform: build_list.save_to_platform,
save_to_repository: build_list.save_to_repository,
build_for_platform: build_list.build_for_platform
) }
before do
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
build_list2.status = BuildList::BUILD_PUBLISHED; build_list2.save!
build_list3.status = BuildList::BUILD_PUBLISHED; build_list3.save!
ProjectToRepository.where(project_id: build_list3.project_id, repository_id: build_list3.save_to_repository_id).destroy_all
end
it "ensures that no 'projects for cleanup' for main repo" do
subject.publish!
subject.publish!
queue = @redis_instance.lrange(AbfWorkerService::Rpm::PROJECTS_FOR_CLEANUP, 0, -1)
queue.should have(1).item
queue.should include("testing-#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}")
end
it "ensures that only one repository_status has status publish" do
subject.publish!
subject.publish!
RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(1).item
end
it "ensures that 'locked projects for cleanup' has only one item" do
subject.publish!
subject.publish!
queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1)
queue.should have(1).item
queue.should include("#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}")
end
it "ensures that new task for publishing has been created" do
expect(Resque).to receive(:push).once
subject.publish!
subject.publish!
end
it "ensures that 'locked build lists' has only one item" do
subject.publish!
subject.publish!
queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(1).item
queue.should include(build_list.id.to_s)
end
end
end
end

View File

@ -13,9 +13,9 @@ then
$rspec spec/controllers/api/
elif [ $SPEC_GROUP = 'models' ]
then
$rspec spec/models/
$rspec spec/models/ spec/jobs/
elif [ $SPEC_GROUP = 'others' ]
then
# $rspec spec/helpers/ spec/integration/ spec/lib/ spec/mailers/ spec/mailers/ spec/routing/
$rspec spec/integration/ spec/lib/ spec/mailers/ spec/mailers/ spec/routing/
$rspec spec/integration/ spec/services/ spec/mailers/ spec/mailers/ spec/routing/
fi