#403: add filter for build_lists without packages

app/jobs/publish_task_manager_job.rb

@@ -5,7 +5,7 @@ class PublishTaskManagerJob
     regenerate_metadata_for_software_center
     resign_repositories
     regenerate_metadata
-    AbfWorkerService::Rpm.new.publish!
+    AbfWorkerService::Rpm.publish!
   end
 
   protected

app/models/build_list.rb

@@ -89,6 +89,7 @@ class BuildList < ActiveRecord::Base
     # %w(PROJECT_SOURCE_ERROR           6),
     # %w(DEPENDENCIES_ERROR             555),
     %w(BUILD_ERROR                    666),
+    %w(PACKAGES_FAIL                  777),
     %w(BUILD_STARTED                  3000),
     %w(BUILD_CANCELED                 5000),
     %w(WAITING_FOR_RESPONSE           4000),

app/services/abf_worker_service/rpm.rb

@@ -3,14 +3,20 @@ module AbfWorkerService
 
     WORKERS_COUNT = APP_CONFIG['abf_worker']['publish_workers_count']
 
-    def publish!
+    attr_accessor :save_to_repository_id, :build_for_platform_id, :testing
+
+    def initialize(save_to_repository_id, build_for_platform_id, testing)
+      @save_to_repository_id  = save_to_repository_id
+      @build_for_platform_id  = build_for_platform_id
+      @testing                = testing
+    end
+
+    def self.publish!
       build_rpms
       build_rpms(true)
     end
 
-    protected
+    def self.build_rpms(testing = false)
-
-    def build_rpms(testing = false)
       available_repos = BuildList.
         select('MIN(updated_at) as min_updated_at, save_to_repository_id, build_for_platform_id').
         where(new_core: true, status: (testing ? BuildList::BUILD_PUBLISH_INTO_TESTING : BuildList::BUILD_PUBLISH)).
@@ -40,19 +46,23 @@ module AbfWorkerService
       available_repos.each do |save_to_repository_id, build_for_platform_id|
         next if RepositoryStatus.not_ready.where(repository_id: save_to_repository_id, platform_id: build_for_platform_id).exists?
         break if counter > WORKERS_COUNT
-        counter += 1 if create_rpm_build_task(save_to_repository_id, build_for_platform_id, testing)
+        service = AbfWorkerService::Rpm.new(
+          save_to_repository_id,
+          build_for_platform_id,
+          testing
+        )
+        counter += 1 if service.create
       end
     end
 
-    def create_rpm_build_task(save_to_repository_id, build_for_platform_id, testing)
+    def create
       key = "#{save_to_repository_id}-#{build_for_platform_id}"
       projects_for_cleanup = Redis.current.lrange(PROJECTS_FOR_CLEANUP, 0, -1).select do |k|
         (testing && k =~ /^testing-[\d]+-#{key}$/) || (!testing && k =~ /^[\d]+-#{key}$/)
       end
 
-      prepare_build_lists(projects_for_cleanup, save_to_repository_id, testing)
+      prepare_build_lists(projects_for_cleanup)
 
-      build_lists   = find_build_lists(build_for_platform_id, save_to_repository_id, testing)
       old_packages  = packages_structure
 
       projects_for_cleanup.each do |key|
@@ -74,7 +84,7 @@ module AbfWorkerService
       end
       build_lists_for_cleanup_from_testing ||= []
 
-      bl = build_lists.first
+      bl = build_lists[0]
       return false if !bl && old_packages[:sources].empty? && old_packages[:binaries].values.flatten.empty?
 
       save_to_repository  = ::Repository.find(save_to_repository_id)
@@ -121,7 +131,7 @@ module AbfWorkerService
         }
       }
 
-      packages, build_list_ids, new_sources = fill_in_packages(build_lists, testing)
+      packages, build_list_ids, new_sources = fill_in_packages
       push(options.merge({
         packages:             packages,
         old_packages:         old_packages,
@@ -129,11 +139,13 @@ module AbfWorkerService
         projects_for_cleanup: projects_for_cleanup
       }))
       lock_projects(projects_for_cleanup)
-      cleanup(save_to_repository_id, build_for_platform_id, build_lists_for_cleanup_from_testing)
+      cleanup(build_lists_for_cleanup_from_testing)
       return true
     end
 
-    def fill_in_packages(build_lists, testing)
+    protected
+
+    def fill_in_packages
       packages, build_list_ids, new_sources = packages_structure, [], {}
       build_lists.each do |bl|
         # remove duplicates of sources for different arches
@@ -161,7 +173,7 @@ module AbfWorkerService
       end
     end
 
-    def cleanup(save_to_repository_id, build_for_platform_id, build_lists_for_cleanup_from_testing)
+    def cleanup(build_lists_for_cleanup_from_testing)
       rep_pl = "#{save_to_repository_id}-#{build_for_platform_id}"
       r_key = "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{rep_pl}"
       build_lists_for_cleanup_from_testing.each do |key|
@@ -180,7 +192,7 @@ module AbfWorkerService
       )
     end
 
-    def prepare_build_lists(projects_for_cleanup, save_to_repository_id, testing)
+    def prepare_build_lists(projects_for_cleanup)
       # We should not to publish new builds into repository
       # if project of builds has been removed from repository.
       BuildList.where(
@@ -190,7 +202,8 @@ module AbfWorkerService
       ).update_all(status: BuildList::FAILED_PUBLISH)
     end
 
-    def find_build_lists(build_for_platform_id, save_to_repository_id, testing)
+    def build_lists
+      @build_lists ||= begin
         build_lists = BuildList.
           where(new_core: true, status: (testing ? BuildList::BUILD_PUBLISH_INTO_TESTING : BuildList::BUILD_PUBLISH)).
           where(save_to_repository_id: save_to_repository_id).
@@ -198,7 +211,28 @@ module AbfWorkerService
           order(:updated_at)
         locked_ids  = Redis.current.lrange(LOCKED_BUILD_LISTS, 0, -1)
         build_lists = build_lists.where('build_lists.id NOT IN (?)', locked_ids) if locked_ids.present?
-      build_lists.limit(150)
+        build_lists = build_lists.limit(150)
+        filter_build_lists_without_packages(build_lists.to_a)
+      end
+    end
+
+    def filter_build_lists_without_packages(build_lists)
+      ids = []
+      build_lists = build_lists.select do |build_list|
+        sha1 = build_list.packages.pluck(:sha1).find do |sha1|
+          !FileStoreService::File.new(sha1: sha1).exist?
+        end
+        if sha1.present?
+          ids << build_list.id
+          false
+        else
+          true
+        end
+      end
+
+      BuildList.where(id: ids).update_all(status: BuildList::PACKAGES_FAIL)
+
+      build_lists
     end
 
   end

config/locales/models/build_list.en.yml

@@ -161,6 +161,7 @@ en:
         build_publish: Build is being published
         failed_publish: Publishing error
         dependencies_fail: Dependences not found
+        packages_fail: Packages not found
         waiting_for_response: Waiting for response
         build_pending: Build pending
         build_canceling: Build is canceling

config/locales/models/build_list.ru.yml

@@ -160,6 +160,7 @@ ru:
         build_publish: публикуется
         failed_publish: ошибка публикации
         dependencies_fail: зависимости не найдены
+        packages_fail: пакеты не найдены
         waiting_for_response: ожидает ответа
         build_pending: ожидает сборку
         build_canceled: сборка отменена

spec/services/abf_worker_service/rpm_spec.rb

@@ -16,6 +16,8 @@ describe AbfWorkerService::Rpm do
     stub_symlink_methods
   end
 
+  subject { AbfWorkerService::Rpm }
+
   context '#publish!' do
 
     before do
@@ -30,7 +32,7 @@ describe AbfWorkerService::Rpm do
 
         it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
           subject.publish!
-          @redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty
+          @redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
         end
       end
 
@@ -48,7 +50,7 @@ describe AbfWorkerService::Rpm do
 
       %w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind|
         it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
-          @redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty
+          @redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
         end
       end
 
@@ -59,7 +61,7 @@ describe AbfWorkerService::Rpm do
       end
 
       it "ensures that 'locked build lists' has only one item" do
-        queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1)
+        queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
         queue.should have(1).item
         queue.should include(build_list.id.to_s)
       end
@@ -91,7 +93,7 @@ describe AbfWorkerService::Rpm do
         subject.publish!
         subject.publish!
 
-        @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1).should have(4).items
+        @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1).should have(4).items
       end
 
       it "ensures that new tasks for publishing has been created" do
@@ -120,7 +122,7 @@ describe AbfWorkerService::Rpm do
         it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
           subject.publish!
           subject.publish!
-          @redis_instance.lrange(AbfWorkerService::Rpm.const_get(kind), 0, -1).should be_empty
+          @redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
         end
       end
 
@@ -133,7 +135,7 @@ describe AbfWorkerService::Rpm do
       it "ensures that 'locked build lists' has 2 items" do
         subject.publish!
         subject.publish!
-        queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1)
+        queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
         queue.should have(2).item
         queue.should include(build_list.id.to_s, build_list2.id.to_s)
       end
@@ -170,7 +172,7 @@ describe AbfWorkerService::Rpm do
         subject.publish!
         subject.publish!
 
-        queue = @redis_instance.lrange(AbfWorkerService::Rpm::PROJECTS_FOR_CLEANUP, 0, -1)
+        queue = @redis_instance.lrange(subject::PROJECTS_FOR_CLEANUP, 0, -1)
         queue.should have(1).item
         queue.should include("testing-#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}")
       end
@@ -186,7 +188,7 @@ describe AbfWorkerService::Rpm do
         subject.publish!
         subject.publish!
 
-        queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1)
+        queue = @redis_instance.lrange(subject::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1)
         queue.should have(1).item
         queue.should include("#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}")
       end
@@ -202,7 +204,7 @@ describe AbfWorkerService::Rpm do
         subject.publish!
         subject.publish!
         
-        queue = @redis_instance.lrange(AbfWorkerService::Rpm::LOCKED_BUILD_LISTS, 0, -1)
+        queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
         queue.should have(1).item
         queue.should include(build_list.id.to_s)
       end