#960: old packages should be saved immediately before removing project from repository

app/models/project.rb

@@ -10,13 +10,14 @@ class Project < ActiveRecord::Base
   has_many :issues, :dependent => :destroy
   has_many :pull_requests, :dependent => :destroy, :foreign_key => 'to_project_id'
   has_many :labels, :dependent => :destroy
-  has_many :build_lists, :dependent => :destroy
 
   has_many :project_imports, :dependent => :destroy
   has_many :project_to_repositories, :dependent => :destroy
   has_many :repositories, :through => :project_to_repositories
   has_many :project_tags, :dependent => :destroy
   
+  has_many :build_lists, :dependent => :destroy
+
   has_many :relations, :as => :target, :dependent => :destroy
   has_many :collaborators, :through => :relations, :source => :actor, :source_type => 'User'
   has_many :groups,        :through => :relations, :source => :actor, :source_type => 'Group'

lib/abf_worker/build_lists_publish_task_manager.rb

@@ -9,6 +9,7 @@ module AbfWorker
        LOCKED_REPOSITORIES
        LOCKED_REP_AND_PLATFORMS
        LOCKED_BUILD_LISTS
+       PACKAGES_FOR_CLEANUP
        REGENERATE_METADATA).each do |kind|
       const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}"
     end
@@ -29,15 +30,18 @@ module AbfWorker
         if repository.platform.personal?
           Platform.main.each do |main_platform|
             redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{main_platform.id}"
+            gather_old_packages project.id, repository.id, main_platform.id
           end
         else
           redis.lpush PROJECTS_FOR_CLEANUP, "#{project.id}-#{repository.id}-#{repository.platform.id}"
+          gather_old_packages project.id, repository.id, repository.platform.id
         end
       end
 
       def cleanup_completed(projects_for_cleanup)
         projects_for_cleanup.each do |key|
           redis.lrem LOCKED_PROJECTS_FOR_CLEANUP, 0, key
+          redis.hdel PACKAGES_FOR_CLEANUP, key
         end
       end
 
@@ -116,6 +120,29 @@ module AbfWorker
 
     private
 
+    def gather_old_packages(project_id, repository_id, main_platform_id)
+      build_lists_for_cleanup = []
+      Arch.pluck(:id).each do |arch_id|
+        bl = BuildList.where(:project_id => project_id).
+          where(:new_core => true, :status => BuildList::BUILD_PUBLISHED).
+          where(:save_to_repository_id => repository_id).
+          where(:build_for_platform_id => main_platform_id).
+          where(:arch_id => arch_id).
+          order(:updated_at).first
+        build_lists_for_cleanup << bl if bl
+      end
+
+      old_packages  = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
+
+      build_lists_for_cleanup.each do |bl|
+        bl.last_published.includes(:packages).limit(5).each{ |old_bl|
+          fill_packages(old_bl, old_packages, :fullname)
+        }
+      end
+
+      @redis.hset PACKAGES_FOR_CLEANUP, "#{project_id}-#{repository_id}-#{main_platform_id}", old_packages.to_json
+    end
+
     def locked_repositories
       @redis.lrange LOCKED_REPOSITORIES, 0, -1
     end
@@ -200,57 +227,52 @@ module AbfWorker
       projects_for_cleanup = @redis.lrange(PROJECTS_FOR_CLEANUP, 0, -1).
         select{ |k| k =~ /#{save_to_repository_id}\-#{build_for_platform_id}$/ }
 
-      build_lists_for_cleanup = []
+      old_packages  = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
+
       projects_for_cleanup.each do |key|
-        pr, rep, pl = *key.split('-')
-        added = false
-        Arch.pluck(:id).each do |arch_id|
-          bl = BuildList.where(:project_id => pr).
-            where(:new_core => true, :status => BuildList::BUILD_PUBLISHED).
-            where(:save_to_repository_id => save_to_repository_id).
-            where(:build_for_platform_id => build_for_platform_id).
-            where(:arch_id => arch_id).
-            order(:updated_at).first
-          if bl
-            build_lists_for_cleanup << bl 
-            added = true
-          end
+        @redis.lrem PROJECTS_FOR_CLEANUP, 0, key
+        packages = @redis.hget PACKAGES_FOR_CLEANUP, key
+        next unless packages
+        packages = JSON.parse packages
+        old_packages[:sources] |= packages['sources']
+        [:x86_64, :i586).each do |arch|
+          old_packages[:binaries][arch] |= packages['binaries'][arch.to_s]
         end
-        # No packages for removing
-        @redis.lrem(PROJECTS_FOR_CLEANUP, 0, key) unless added
       end
 
-      bl = build_lists.first || build_lists_for_cleanup.first
-      return false unless bl
+      bl = build_lists.first
+      return false if !bl && old_packages[:sources].empty?
 
-      platform_path = "#{bl.save_to_platform.path}/repository"
-      if bl.save_to_platform.personal?
-        platform_path << '/' << bl.build_for_platform.name
+      save_to_repository  = Repository.find save_to_repository_id
+      save_to_platform    = save_to_repository.platform
+      build_for_platform  = Platform.find build_for_platform_id
+      platform_path = "#{save_to_platform.path}/repository"
+      if save_to_platform.personal?
+        platform_path << '/' << build_for_platform.name
         system "mkdir -p #{platform_path}"
       end
-      worker_queue = bl.worker_queue_with_priority("publish_worker")
-      worker_class = bl.worker_queue_class("AbfWorker::PublishWorker")
+      worker_queue = bl ? bl.worker_queue_with_priority("publish_worker") : 'publish_worker_default'
+      worker_class = bl ? bl.worker_queue_class("AbfWorker::PublishWorker") : 'AbfWorker::PublishWorkerDefault'
 
-      distrib_type  = bl.build_for_platform.distrib_type
+      distrib_type  = build_for_platform.distrib_type
       cmd_params    = {
-        'RELEASED'        => bl.save_to_platform.released,
-        'REPOSITORY_NAME' => bl.save_to_repository.name,
+        'RELEASED'        => save_to_platform.released,
+        'REPOSITORY_NAME' => save_to_repository.name,
         'TYPE'            => distrib_type
       }.map{ |k, v| "#{k}=#{v}" }.join(' ')
 
       options = {
-        :id           => bl.id,
-        :arch         => bl.arch.name,
+        :id           => (bl ? bl.id : Time.now.to_i),
+        :arch         => (bl ? bl.arch.name : 'x86_64'),
         :distrib_type => distrib_type,
         :cmd_params   => cmd_params,
         :platform     => {:platform_path => platform_path},
-        :repository   => {:id => bl.save_to_repository.id},
+        :repository   => {:id => save_to_repository_id},
         :type         => :publish,
         :time_living  => 9600 # 160 min
       }
 
       packages      = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
-      old_packages  = {:sources => [], :binaries => {:x86_64 => [], :i586 => []}}
       build_list_ids = []
 
       new_sources = {}
@@ -266,12 +288,6 @@ module AbfWorker
       end
       packages[:sources] = new_sources.values.compact
 
-      build_lists_for_cleanup.each do |bl|
-        bl.last_published.includes(:packages).limit(5).each{ |old_bl|
-          fill_packages(old_bl, old_packages, :fullname)
-        }
-      end
-
       Resque.push(
         worker_queue,
         'class' => worker_class,
@@ -284,7 +300,6 @@ module AbfWorker
       )
 
       projects_for_cleanup.each do |key|
-        @redis.lrem PROJECTS_FOR_CLEANUP, 0, key
         @redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key
       end