Merge pull request #989 from warpc/960-remove-packages-from-repository

[refs #960]: Remove packages from repository when remove project from repository.

app/models/project.rb

@@ -10,12 +10,13 @@ class Project < ActiveRecord::Base
   has_many :issues, :dependent => :destroy
   has_many :pull_requests, :dependent => :destroy, :foreign_key => 'to_project_id'
   has_many :labels, :dependent => :destroy
-  has_many :build_lists, :dependent => :destroy
 
   has_many :project_imports, :dependent => :destroy
   has_many :project_to_repositories, :dependent => :destroy
   has_many :repositories, :through => :project_to_repositories
   has_many :project_tags, :dependent => :destroy
+  
+  has_many :build_lists, :dependent => :destroy
 
   has_many :relations, :as => :target, :dependent => :destroy
   has_many :collaborators, :through => :relations, :source => :actor, :source_type => 'User'

lib/abf_worker/build_lists_publish_task_manager.rb

@@ -9,6 +9,7 @@ module AbfWorker
        LOCKED_REPOSITORIES
        LOCKED_REP_AND_PLATFORMS
        LOCKED_BUILD_LISTS
+       PACKAGES_FOR_CLEANUP
        REGENERATE_METADATA).each do |kind|
       const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}"
     end
@@ -29,15 +30,18 @@ module AbfWorker
         if repository.platform.personal?
           Platform.main.each do |main_platform|
             redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{main_platform.id}"
+            gather_old_packages project.id, repository.id, main_platform.id
           end
         else
           redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{repository.platform.id}"
+          gather_old_packages project.id, repository.id, repository.platform.id
         end
       end
 
       def cleanup_completed(projects_for_cleanup)
         projects_for_cleanup.each do |key|
           redis.lrem LOCKED_PROJECTS_FOR_CLEANUP, 0, key
+          redis.hdel PACKAGES_FOR_CLEANUP, key
         end
       end
 
@@ -112,10 +116,46 @@ module AbfWorker
           }]
         )
       end
+
+      def gather_old_packages(project_id, repository_id, platform_id)
+        build_lists_for_cleanup = []
+        Arch.pluck(:id).each do |arch_id|
+          bl = BuildList.where(:project_id => project_id).
+            where(:new_core => true, :status => BuildList::BUILD_PUBLISHED).
+            where(:save_to_repository_id => repository_id).
+            where(:build_for_platform_id => platform_id).
+            where(:arch_id => arch_id).
+            order(:updated_at).first
+          build_lists_for_cleanup << bl if bl
+        end
+
+        old_packages  = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
+        build_lists_for_cleanup.each do |bl|
+          bl.last_published.includes(:packages).limit(2).each{ |old_bl|
+            fill_packages(old_bl, old_packages, :fullname)
+          }
+        end
+
+        redis.hset PACKAGES_FOR_CLEANUP, "#{project_id}-#{repository_id}-#{platform_id}", old_packages.to_json
+      end
+
+      def fill_packages(bl, results_map, field = :sha1)
+        results_map[:sources] |= bl.packages.by_package_type('source').pluck(field).compact if field != :sha1
+        
+        binaries  = bl.packages.by_package_type('binary').pluck(field).compact
+        arch      = bl.arch.name.to_sym
+        results_map[:binaries][arch] |= binaries
+        # Publish/remove i686 RHEL packages into/from x86_64
+        if arch == :i586 && bl.build_for_platform.distrib_type == 'rhel' && bl.project.publish_i686_into_x86_64?
+          results_map[:binaries][:x86_64] |= binaries
+        end
+      end
+
     end
 
     private
 
+
     def locked_repositories
       @redis.lrange LOCKED_REPOSITORIES, 0, -1
     end
@@ -200,80 +240,69 @@ module AbfWorker
       projects_for_cleanup = @redis.lrange(PROJECTS_FOR_CLEANUP, 0, -1).
         select{ |k| k =~ /#{save_to_repository_id}\-#{build_for_platform_id}$/ }
 
-      build_lists_for_cleanup = []
+      old_packages  = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
+
       projects_for_cleanup.each do |key|
-        pr, rep, pl = *key.split('-')
+        @redis.lrem PROJECTS_FOR_CLEANUP, 0, key
-        added = false
+        packages = @redis.hget PACKAGES_FOR_CLEANUP, key
-        Arch.pluck(:id).each do |arch_id|
+        next unless packages
-          bl = BuildList.where(:project_id => pr).
+        packages = JSON.parse packages
-            where(:new_core => true, :status => BuildList::BUILD_PUBLISHED).
+        old_packages[:sources] |= packages['sources']
-            where(:save_to_repository_id => save_to_repository_id).
+        [:x86_64, :i586].each do |arch|
-            where(:build_for_platform_id => build_for_platform_id).
+          old_packages[:binaries][arch] |= packages['binaries'][arch.to_s]
-            where(:arch_id => arch_id).
-            order(:updated_at).first
-          if bl
-            build_lists_for_cleanup << bl 
-            added = true
-          end
         end
-        # No packages for removing
-        @redis.lrem(PROJECTS_FOR_CLEANUP, 0, key) unless added
       end
 
-      bl = build_lists.first || build_lists_for_cleanup.first
+      bl = build_lists.first
-      return false unless bl
+      return false if !bl && old_packages[:sources].empty?
 
-      platform_path = "#{bl.save_to_platform.path}/repository"
+      save_to_repository  = Repository.find save_to_repository_id
-      if bl.save_to_platform.personal?
+      save_to_platform    = save_to_repository.platform
-        platform_path << '/' << bl.build_for_platform.name
+      build_for_platform  = Platform.find build_for_platform_id
+      platform_path = "#{save_to_platform.path}/repository"
+      if save_to_platform.personal?
+        platform_path << '/' << build_for_platform.name
         system "mkdir -p #{platform_path}"
       end
-      worker_queue = bl.worker_queue_with_priority("publish_worker")
+      worker_queue = bl ? bl.worker_queue_with_priority("publish_worker") : 'publish_worker_default'
-      worker_class = bl.worker_queue_class("AbfWorker::PublishWorker")
+      worker_class = bl ? bl.worker_queue_class("AbfWorker::PublishWorker") : 'AbfWorker::PublishWorkerDefault'
 
-      distrib_type  = bl.build_for_platform.distrib_type
+      distrib_type  = build_for_platform.distrib_type
       cmd_params    = {
-        'RELEASED'        => bl.save_to_platform.released,
+        'RELEASED'        => save_to_platform.released,
-        'REPOSITORY_NAME' => bl.save_to_repository.name,
+        'REPOSITORY_NAME' => save_to_repository.name,
         'TYPE'            => distrib_type
       }.map{ |k, v| "#{k}=#{v}" }.join(' ')
 
       lock_str  = "#{save_to_repository_id}-#{build_for_platform_id}"
       options   = {
-        :id           => bl.id,
+        :id           => (bl ? bl.id : Time.now.to_i),
-        :arch         => bl.arch.name,
+        :arch         => (bl ? bl.arch.name : 'x86_64'),
         :distrib_type => distrib_type,
         :cmd_params   => cmd_params,
         :platform     => {:platform_path => platform_path},
-        :repository   => {:id => bl.save_to_repository.id},
+        :repository   => {:id => save_to_repository_id},
         :type         => :publish,
         :time_living  => 9600, # 160 min
         :extra        => {:lock_str => lock_str}
       }
 
       packages      = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
-      old_packages  = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
       build_list_ids = []
 
       new_sources = {}
       build_lists.each do |bl|
         # remove duplicates of sources for different arches
         bl.packages.by_package_type('source').each{ |s| new_sources["#{s.fullname}"] = s.sha1 }
-        fill_packages(bl, packages)
+        self.class.fill_packages(bl, packages)
-        bl.last_published.includes(:packages).limit(5).each{ |old_bl|
+        bl.last_published.includes(:packages).limit(2).each{ |old_bl|
-          fill_packages(old_bl, old_packages, :fullname)
+          self.class.fill_packages(old_bl, old_packages, :fullname)
         }
         build_list_ids << bl.id
         @redis.lpush(LOCKED_BUILD_LISTS, bl.id)
       end
       packages[:sources] = new_sources.values.compact
 
-      build_lists_for_cleanup.each do |bl|
-        bl.last_published.includes(:packages).limit(5).each{ |old_bl|
-          fill_packages(old_bl, old_packages, :fullname)
-        }
-      end
-
       Resque.push(
         worker_queue,
         'class' => worker_class,
@@ -286,7 +315,6 @@ module AbfWorker
       )
 
       projects_for_cleanup.each do |key|
-        @redis.lrem PROJECTS_FOR_CLEANUP, 0, key
         @redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key
       end
 
@@ -294,18 +322,6 @@ module AbfWorker
       return true
     end
 
-    def fill_packages(bl, results_map, field = :sha1)
-      results_map[:sources] |= bl.packages.by_package_type('source').pluck(field).compact if field != :sha1
-      
-      binaries  = bl.packages.by_package_type('binary').pluck(field).compact
-      arch      = bl.arch.name.to_sym
-      results_map[:binaries][arch] |= binaries
-      # Publish/remove i686 RHEL packages into/from x86_64
-      if arch == :i586 && bl.build_for_platform.distrib_type == 'rhel' && bl.project.publish_i686_into_x86_64?
-        results_map[:binaries][:x86_64] |= binaries
-      end
-    end
-
     def create_tasks_for_repository_regenerate_metadata
       worker_queue = 'publish_worker_default'
       worker_class   = 'AbfWorker::PublishWorkerDefault'