diff --git a/app/services/abf_worker_service/rpm.rb b/app/services/abf_worker_service/rpm.rb index dfa8ecb31..d8ed3488d 100644 --- a/app/services/abf_worker_service/rpm.rb +++ b/app/services/abf_worker_service/rpm.rb @@ -50,7 +50,7 @@ module AbfWorkerService (testing && k =~ /^testing-[\d]+-#{key}$/) || (!testing && k =~ /^[\d]+-#{key}$/) end - prepare_build_lists(projects_for_cleanup, save_to_repository_id) + prepare_build_lists(projects_for_cleanup, save_to_repository_id, testing) build_lists = find_build_lists(build_for_platform_id, save_to_repository_id, testing) old_packages = packages_structure @@ -77,7 +77,7 @@ module AbfWorkerService bl = build_lists.first return false if !bl && old_packages[:sources].empty? && old_packages[:binaries].values.flatten.empty? - save_to_repository = Repository.find save_to_repository_id + save_to_repository = ::Repository.find(save_to_repository_id) # Checks mirror sync status return false if save_to_repository.repo_lock_file_exists? || !save_to_repository.platform.ready? @@ -180,7 +180,7 @@ module AbfWorkerService ) end - def prepare_build_lists(projects_for_cleanup, save_to_repository_id) + def prepare_build_lists(projects_for_cleanup, save_to_repository_id, testing) # We should not to publish new builds into repository # if project of builds has been removed from repository. BuildList.where( diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb deleted file mode 100644 index 9eb7a5303..000000000 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ /dev/null @@ -1,199 +0,0 @@ -module AbfWorker - class BuildListsPublishTaskManager - REDIS_MAIN_KEY = 'abf-worker::build-lists-publish-task-manager::' - - %w(PROJECTS_FOR_CLEANUP - LOCKED_PROJECTS_FOR_CLEANUP - LOCKED_BUILD_LISTS - PACKAGES_FOR_CLEANUP - REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING - BUILD_LISTS_FOR_CLEANUP_FROM_TESTING).each do |kind| - const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}" - end - - def initialize - @workers_count = APP_CONFIG['abf_worker']['publish_workers_count'] - end - - def run - # create_tasks_for_regenerate_metadata_for_software_center - # create_tasks_for_resign_repositories - # create_tasks_for_repository_regenerate_metadata - create_tasks_for_build_rpms - create_tasks_for_build_rpms true - end - - private - - def create_tasks_for_build_rpms(testing = false) - available_repos = BuildList. - select('MIN(updated_at) as min_updated_at, save_to_repository_id, build_for_platform_id'). - where(new_core: true, status: (testing ? BuildList::BUILD_PUBLISH_INTO_TESTING : BuildList::BUILD_PUBLISH)). - group(:save_to_repository_id, :build_for_platform_id). - order('min_updated_at ASC'). - limit(@workers_count * 2) # because some repos may be locked - - locked_rep = RepositoryStatus.not_ready.joins(:platform). - where(platforms: {platform_type: 'main'}).pluck(:repository_id) - available_repos = available_repos.where('save_to_repository_id NOT IN (?)', locked_rep) unless locked_rep.empty? - - for_cleanup = Redis.current.lrange(PROJECTS_FOR_CLEANUP, 0, -1).map do |key| - next if testing && key !~ /^testing-/ - rep, pl = *key.split('-').last(2) - locked_rep.present? && locked_rep.include?(rep.to_i) ? nil : [rep.to_i, pl.to_i] - end.compact - - for_cleanup_from_testing = Redis.current.smembers(REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING).map do |key| - next if Redis.current.scard("#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{key}") == 0 - rep, pl = *key.split('-') - locked_rep.present? && locked_rep.include?(rep.to_i) ? nil : [rep.to_i, pl.to_i] - end.compact if testing - for_cleanup_from_testing ||= [] - - counter = 1 - available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup | for_cleanup_from_testing - available_repos.each do |save_to_repository_id, build_for_platform_id| - next if RepositoryStatus.not_ready.where(repository_id: save_to_repository_id, platform_id: build_for_platform_id).exists? - break if counter > @workers_count - counter += 1 if create_rpm_build_task(save_to_repository_id, build_for_platform_id, testing) - end - end - - def create_rpm_build_task(save_to_repository_id, build_for_platform_id, testing) - key = "#{save_to_repository_id}-#{build_for_platform_id}" - projects_for_cleanup = Redis.current.lrange(PROJECTS_FOR_CLEANUP, 0, -1).select do |k| - (testing && k =~ /^testing-[\d]+-#{key}$/) || (!testing && k =~ /^[\d]+-#{key}$/) - end - - # We should not to publish new builds into repository - # if project of builds has been removed from repository. - BuildList.where( - project_id: projects_for_cleanup.map{ |k| k.split('-')[testing ? 1 : 0] }.uniq, - save_to_repository_id: save_to_repository_id, - status: [BuildList::BUILD_PUBLISH, BuildList::BUILD_PUBLISH_INTO_TESTING] - ).update_all(status: BuildList::FAILED_PUBLISH) - - build_lists = BuildList. - where(new_core: true, status: (testing ? BuildList::BUILD_PUBLISH_INTO_TESTING : BuildList::BUILD_PUBLISH)). - where(save_to_repository_id: save_to_repository_id). - where(build_for_platform_id: build_for_platform_id). - order(:updated_at) - locked_ids = Redis.current.lrange(LOCKED_BUILD_LISTS, 0, -1) - build_lists = build_lists.where('build_lists.id NOT IN (?)', locked_ids) unless locked_ids.empty? - build_lists = build_lists.limit(150) - - old_packages = self.class.packages_structure - - projects_for_cleanup.each do |key| - Redis.current.lrem PROJECTS_FOR_CLEANUP, 0, key - packages = Redis.current.hget PACKAGES_FOR_CLEANUP, key - next unless packages - packages = JSON.parse packages - old_packages[:sources] |= packages['sources'] - Arch.pluck(:name).each do |arch| - old_packages[:binaries][arch.to_sym] |= packages['binaries'][arch] || [] - end - end - - if testing - build_lists_for_cleanup_from_testing = Redis.current.smembers("#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}") - BuildList.where(id: build_lists_for_cleanup_from_testing).each do |b| - self.class.fill_packages(b, old_packages, :fullname) - end if build_lists_for_cleanup_from_testing.present? - end - build_lists_for_cleanup_from_testing ||= [] - - bl = build_lists.first - return false if !bl && old_packages[:sources].empty? && old_packages[:binaries].values.flatten.empty? - - save_to_repository = Repository.find save_to_repository_id - # Checks mirror sync status - return false if save_to_repository.repo_lock_file_exists? || !save_to_repository.platform.ready? - - repository_status = save_to_repository.repository_statuses.find_or_create_by(platform_id: build_for_platform_id) - return false unless repository_status.publish - - save_to_platform = save_to_repository.platform - build_for_platform = Platform.find build_for_platform_id - platform_path = "#{save_to_platform.path}/repository" - if save_to_platform.personal? - platform_path << '/' << build_for_platform.name - system "mkdir -p #{platform_path}" - end - - distrib_type = build_for_platform.distrib_type - cmd_params = { - 'RELEASED' => save_to_platform.released, - 'REPOSITORY_NAME' => save_to_repository.name, - 'TYPE' => distrib_type, - 'SAVE_TO_PLATFORM' => save_to_platform.name, - 'BUILD_FOR_PLATFORM' => build_for_platform.name, - 'TESTING' => testing - }.map{ |k, v| "#{k}=#{v}" }.join(' ') - - options = { - id: (bl ? bl.id : Time.now.to_i), - cmd_params: cmd_params, - main_script: 'build.sh', - rollback_script: 'rollback.sh', - platform: { - platform_path: platform_path, - type: distrib_type, - name: build_for_platform.name, - arch: (bl ? bl.arch.name : 'x86_64') - }, - repository: {id: save_to_repository_id}, - time_living: 9600, # 160 min - extra: { - repository_status_id: repository_status.id, - build_lists_for_cleanup_from_testing: build_lists_for_cleanup_from_testing - } - } - - packages, build_list_ids, new_sources = self.class.packages_structure, [], {} - build_lists.each do |bl| - # remove duplicates of sources for different arches - bl.packages.by_package_type('source').each{ |s| new_sources["#{s.fullname}"] = s.sha1 } - self.class.fill_packages(bl, packages) - bl.last_published(testing).includes(:packages).limit(2).each{ |old_bl| - self.class.fill_packages(old_bl, old_packages, :fullname) - } - # TODO: do more flexible - # Removes old packages which already in the main repo - bl.last_published(false).includes(:packages).limit(3).each{ |old_bl| - self.class.fill_packages(old_bl, old_packages, :fullname) - } if testing - build_list_ids << bl.id - Redis.current.lpush(LOCKED_BUILD_LISTS, bl.id) - end - packages[:sources] = new_sources.values.compact - - Resque.push( - 'publish_worker_default', - 'class' => 'AbfWorker::PublishWorkerDefault', - 'args' => [options.merge({ - packages: packages, - old_packages: old_packages, - build_list_ids: build_list_ids, - projects_for_cleanup: projects_for_cleanup - })] - ) - - projects_for_cleanup.each do |key| - Redis.current.lpush LOCKED_PROJECTS_FOR_CLEANUP, key - end - - rep_pl = "#{save_to_repository_id}-#{build_for_platform_id}" - r_key = "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{rep_pl}" - build_lists_for_cleanup_from_testing.each do |key| - Redis.current.srem r_key, key - end - if Redis.current.scard(r_key) == 0 - Redis.current.srem REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, rep_pl - end - - return true - end - - end -end diff --git a/spec/lib/abf-worker/build_lists_publish_task_manager_spec.rb b/spec/lib/abf-worker/build_lists_publish_task_manager_spec.rb deleted file mode 100644 index dc84e39e5..000000000 --- a/spec/lib/abf-worker/build_lists_publish_task_manager_spec.rb +++ /dev/null @@ -1,177 +0,0 @@ -require 'spec_helper' - -describe AbfWorker::BuildListsPublishTaskManager do - before(:all) do - @publish_workers_count = APP_CONFIG['abf_worker']['publish_workers_count'] - APP_CONFIG['abf_worker']['publish_workers_count'] = 2 - end - - before do - stub_symlink_methods - FactoryGirl.create(:build_list) - end - - subject { AbfWorker::BuildListsPublishTaskManager } - let(:build_list) { FactoryGirl.create(:build_list) } - - context 'when no items for publishing' do - before { subject.new.run } - - %w(PROJECTS_FOR_CLEANUP - LOCKED_PROJECTS_FOR_CLEANUP - LOCKED_BUILD_LISTS).each do |kind| - - it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do - @redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty - end - end - - %w(publish_worker_default publish_worker).each do |kind| - it "ensures that no tasks in '#{kind}' queue" do - @redis_instance.lrange(kind, 0, -1).should be_empty - end - end - - end - - context 'when one build_list for publishing' do - before do - build_list.update_column(:status, BuildList::BUILD_PUBLISH) - 2.times{ subject.new.run } - end - %w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind| - it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do - @redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty - end - end - - it "ensures that repository_status has status publish" do - build_list.save_to_repository.repository_statuses. - find_by(platform_id: build_list.build_for_platform_id).publish?. - should be_true - end - - it "ensures that 'locked build lists' has only one item" do - queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1) - queue.should have(1).item - queue.should include(build_list.id.to_s) - end - - it "ensures that new task for publishing has been created" do - @redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(1).item - end - - end - - context 'grouping build lists for publishing into same repository' do - let(:build_list2) { FactoryGirl.create(:build_list, - new_core: true, - save_to_platform: build_list.save_to_platform, - save_to_repository: build_list.save_to_repository, - build_for_platform: build_list.build_for_platform - ) } - before do - build_list.update_column(:status, BuildList::BUILD_PUBLISH) - build_list2.update_column(:status, BuildList::BUILD_PUBLISH) - 2.times{ subject.new.run } - end - - %w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind| - it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do - @redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty - end - end - - it "ensures that only one repository_status has status publish" do - RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(1).item - end - - it "ensures that 'locked build lists' has 2 items" do - queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1) - queue.should have(2).item - queue.should include(build_list.id.to_s, build_list2.id.to_s) - end - - it "ensures that new task for publishing has been created" do - @redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(1).item - end - - end - - context 'creates not more than 4 tasks for publishing' do - before do - build_list.update_column(:status, BuildList::BUILD_PUBLISH) - 4.times { - bl = FactoryGirl.create(:build_list, new_core: true) - bl.update_column(:status, BuildList::BUILD_PUBLISH) - } - 2.times{ subject.new.run } - end - - it "ensures that 4 repository_statuses have status publish" do - RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(4).items - end - - it "ensures that 'locked build lists' has 4 items" do - @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1).should have(4).items - end - - it "ensures that new tasks for publishing has been created" do - @redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(4).items - end - - end - - context 'grouping build lists for publishing and tasks for removing project from repository' do - let(:build_list2) { FactoryGirl.create(:build_list, - new_core: true, - save_to_platform: build_list.save_to_platform, - save_to_repository: build_list.save_to_repository, - build_for_platform: build_list.build_for_platform - ) } - let(:build_list3) { FactoryGirl.create(:build_list, - new_core: true, - save_to_platform: build_list.save_to_platform, - save_to_repository: build_list.save_to_repository, - build_for_platform: build_list.build_for_platform - ) } - before do - build_list.update_column(:status, BuildList::BUILD_PUBLISH) - build_list2.update_column(:status, BuildList::BUILD_PUBLISHED) - build_list3.update_column(:status, BuildList::BUILD_PUBLISHED) - ProjectToRepository.where(project_id: build_list3.project_id, repository_id: build_list3.save_to_repository_id).destroy_all - 2.times{ subject.new.run } - end - - it "ensures that no 'projects for cleanup' for main repo" do - queue = @redis_instance.lrange(subject::PROJECTS_FOR_CLEANUP, 0, -1) - queue.should have(1).item - queue.should include("testing-#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}") - end - - it "ensures that only one repository_status has status publish" do - RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(1).item - end - - it "ensures that 'locked projects for cleanup' has only one item" do - queue = @redis_instance.lrange(subject::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1) - queue.should have(1).item - queue.should include("#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}") - end - - it "ensures that new task for publishing has been created" do - @redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(1).item - end - - it "ensures that 'locked build lists' has only one item" do - queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1) - queue.should have(1).item - queue.should include(build_list.id.to_s) - end - end - - after(:all) do - APP_CONFIG['abf_worker']['publish_workers_count'] = @publish_workers_count - FileUtils.rm_rf(APP_CONFIG['root_path']) - end -end diff --git a/spec/services/abf_worker_service/rpm_spec.rb b/spec/services/abf_worker_service/rpm_spec.rb new file mode 100644 index 000000000..8b8df1722 --- /dev/null +++ b/spec/services/abf_worker_service/rpm_spec.rb @@ -0,0 +1,213 @@ +require 'spec_helper' + +describe AbfWorkerService::Rpm do + let(:build_list) { FactoryGirl.create(:build_list) } + + before(:all) do + @publish_workers_count = APP_CONFIG['abf_worker']['publish_workers_count'] + APP_CONFIG['abf_worker']['publish_workers_count'] = 2 + end + + after(:all) do + APP_CONFIG['abf_worker']['publish_workers_count'] = @publish_workers_count + end + + before do + stub_symlink_methods + end + + context '#publish!' do + + before do + FactoryGirl.create(:build_list) + end + + context 'no items for publishing' do + + %w(PROJECTS_FOR_CLEANUP + LOCKED_PROJECTS_FOR_CLEANUP + LOCKED_BUILD_LISTS).each do |kind| + + it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do + subject.publish! + @redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty + end + end + + it 'ensures that no tasks' do + expect(Resque).to_not receive(:push) + subject.publish! + end + end + + context 'when one build_list for publishing' do + before do + build_list.update_column(:status, BuildList::BUILD_PUBLISH) + 2.times{ subject.publish! } + end + + %w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind| + it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do + @redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty + end + end + + it "ensures that repository_status has status publish" do + build_list.save_to_repository.repository_statuses. + find_by(platform_id: build_list.build_for_platform_id).publish?. + should be_true + end + + it "ensures that 'locked build lists' has only one item" do + queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1) + queue.should have(1).item + queue.should include(build_list.id.to_s) + end + + it "ensures that new task for publishing has been created" do + @redis_instance.lrange('resque:queue:publish_worker_default', 0, -1).should have(1).item + end + + end + + context 'creates not more than 4 tasks for publishing' do + before do + build_list.update_column(:status, BuildList::BUILD_PUBLISH) + 4.times do + bl = FactoryGirl.build(:build_list) + bl.status = BuildList::BUILD_PUBLISH + bl.save! + end + end + + it "ensures that 4 repository_statuses have status publish" do + subject.publish! + subject.publish! + + RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(4).items + end + + it "ensures that 'locked build lists' has 4 items" do + subject.publish! + subject.publish! + + @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1).should have(4).items + end + + it "ensures that new tasks for publishing has been created" do + expect(Resque).to receive(:push).exactly(4).times + + subject.publish! + subject.publish! + end + end + + context 'grouping build lists for publishing into same repository' do + let(:build_list2) { FactoryGirl.build(:build_list, + new_core: true, + save_to_platform: build_list.save_to_platform, + save_to_repository: build_list.save_to_repository, + build_for_platform: build_list.build_for_platform + ) } + + before do + build_list.update_column(:status, BuildList::BUILD_PUBLISH) + build_list2.status = BuildList::BUILD_PUBLISH + build_list2.save! + end + + %w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind| + it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do + subject.publish! + subject.publish! + @redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty + end + end + + it "ensures that only one repository_status has status publish" do + subject.publish! + subject.publish! + RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(1).item + end + + it "ensures that 'locked build lists' has 2 items" do + subject.publish! + subject.publish! + queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1) + queue.should have(2).item + queue.should include(build_list.id.to_s, build_list2.id.to_s) + end + + it "ensures that new task for publishing has been created" do + expect(Resque).to receive(:push).once + + subject.publish! + subject.publish! + end + end + + context 'grouping build lists for publishing and tasks for removing project from repository' do + let(:build_list2) { FactoryGirl.build(:build_list, + new_core: true, + save_to_platform: build_list.save_to_platform, + save_to_repository: build_list.save_to_repository, + build_for_platform: build_list.build_for_platform + ) } + let(:build_list3) { FactoryGirl.build(:build_list, + new_core: true, + save_to_platform: build_list.save_to_platform, + save_to_repository: build_list.save_to_repository, + build_for_platform: build_list.build_for_platform + ) } + before do + build_list.update_column(:status, BuildList::BUILD_PUBLISH) + build_list2.status = BuildList::BUILD_PUBLISHED; build_list2.save! + build_list3.status = BuildList::BUILD_PUBLISHED; build_list3.save! + ProjectToRepository.where(project_id: build_list3.project_id, repository_id: build_list3.save_to_repository_id).destroy_all + end + + it "ensures that no 'projects for cleanup' for main repo" do + subject.publish! + subject.publish! + + queue = @redis_instance.lrange(AbfWorkerService::Rpm::PROJECTS_FOR_CLEANUP, 0, -1) + queue.should have(1).item + queue.should include("testing-#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}") + end + + it "ensures that only one repository_status has status publish" do + subject.publish! + subject.publish! + + RepositoryStatus.where(status: RepositoryStatus::PUBLISH).should have(1).item + end + + it "ensures that 'locked projects for cleanup' has only one item" do + subject.publish! + subject.publish! + + queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1) + queue.should have(1).item + queue.should include("#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}") + end + + it "ensures that new task for publishing has been created" do + expect(Resque).to receive(:push).once + + subject.publish! + subject.publish! + end + + it "ensures that 'locked build lists' has only one item" do + subject.publish! + subject.publish! + + queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1) + queue.should have(1).item + queue.should include(build_list.id.to_s) + end + end + + end + +end diff --git a/travis.sh b/travis.sh index d54fd88b2..75f3dd158 100755 --- a/travis.sh +++ b/travis.sh @@ -13,9 +13,9 @@ then $rspec spec/controllers/api/ elif [ $SPEC_GROUP = 'models' ] then - $rspec spec/models/ + $rspec spec/models/ spec/jobs/ elif [ $SPEC_GROUP = 'others' ] then # $rspec spec/helpers/ spec/integration/ spec/lib/ spec/mailers/ spec/mailers/ spec/routing/ - $rspec spec/integration/ spec/lib/ spec/mailers/ spec/mailers/ spec/routing/ + $rspec spec/integration/ spec/services/ spec/mailers/ spec/mailers/ spec/routing/ fi