From df83fe65e144baddded961f67676536119ab5d04 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 10 Jan 2014 00:54:48 +0400 Subject: [PATCH 1/6] #336: added ability for clean up packages from testing repository --- app/models/build_list.rb | 9 ++++ .../build_lists_publish_task_manager.rb | 41 +++++++++++++++++-- lib/abf_worker/publish_observer.rb | 8 ++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/app/models/build_list.rb b/app/models/build_list.rb index 1981e1816..7c2b9d4e1 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -158,6 +158,7 @@ class BuildList < ActiveRecord::Base after_transition :on => :published, :do => [:set_version_and_tag, :actualize_packages] after_transition :on => :publish, :do => :set_publisher + after_transition :build_published_into_testing => :publish, :do => :cleanup_packages_from_testing after_transition :on => :cancel, :do => :cancel_job after_transition :on => [:published, :fail_publish, :build_error, :tests_failed], :do => :notify_users @@ -540,6 +541,14 @@ class BuildList < ActiveRecord::Base save end + def cleanup_packages_from_testing + AbfWorker::BuildListsPublishTaskManager.cleanup_packages_from_testing( + build_for_platform_id, + save_to_repository_id, + id + ) + end + def current_ability @current_ability ||= Ability.new(user) end diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 275fbfb69..65cacf450 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -6,7 +6,9 @@ module AbfWorker %w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP LOCKED_BUILD_LISTS - PACKAGES_FOR_CLEANUP).each do |kind| + PACKAGES_FOR_CLEANUP + REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING + BUILD_LISTS_FOR_CLEANUP_FROM_TESTING).each do |kind| const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}" end @@ -58,6 +60,13 @@ module AbfWorker end end + def cleanup_packages_from_testing(platform_id, repository_id, *build_lists) + return if build_lists.blank? + key = "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{repository_id}-#{platform_id}" + redis.lpush REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, "#{repository_id}-#{platform_id}" + redis.lpush key, build_lists + end + def unlock_build_list(build_list) redis.lrem LOCKED_BUILD_LISTS, 0, build_list.id end @@ -213,8 +222,15 @@ module AbfWorker locked_rep.present? && locked_rep.include?(rep.to_i) ? nil : [rep.to_i, pl.to_i] end.compact + for_cleanup_from_testing = @redis.lrange(REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, 0, -1).map do |key| + next if redis.llen("#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{key}") == 0 + rep, pl = *key.split('-') + locked_rep.present? && locked_rep.include?(rep.to_i) ? nil : [rep.to_i, pl.to_i] + end.compact if testing + for_cleanup_from_testing ||= [] + counter = 1 - available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup + available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup | for_cleanup_from_testing available_repos.each do |save_to_repository_id, build_for_platform_id| next if RepositoryStatus.not_ready.where(:repository_id => save_to_repository_id, :platform_id => build_for_platform_id).exists? break if counter > @workers_count @@ -258,6 +274,18 @@ module AbfWorker end end + if testing + build_lists_for_cleanup_from_testing = @redis.lrange( + "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}" + 0, -1 + ) + BuildList.where(:id => build_lists_for_cleanup_from_testing).each do |b| + self.class.fill_packages(b, old_packages) + end if build_lists_for_cleanup_from_testing.present? + end + build_lists_for_cleanup_from_testing ||= [] + + bl = build_lists.first return false if !bl && old_packages[:sources].empty? @@ -301,7 +329,10 @@ module AbfWorker }, :repository => {:id => save_to_repository_id}, :time_living => 9600, # 160 min - :extra => {:repository_status_id => repository_status.id} + :extra => { + :repository_status_id => repository_status.id, + :build_lists_for_cleanup_from_testing => build_lists_for_cleanup_from_testing + } } packages, build_list_ids, new_sources = self.class.packages_structure, [], {} @@ -337,6 +368,10 @@ module AbfWorker @redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key end + build_lists_for_cleanup_from_testing.each do |key| + @redis.lrem "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}", 0, key + end + return true end diff --git a/lib/abf_worker/publish_observer.rb b/lib/abf_worker/publish_observer.rb index 7ad128c9a..f12d9e7f1 100644 --- a/lib/abf_worker/publish_observer.rb +++ b/lib/abf_worker/publish_observer.rb @@ -37,6 +37,14 @@ module AbfWorker end update_results elsif !extra['resign'] # Simple publish + bls = extra['build_lists_for_cleanup_from_testing'] + if status != COMPLETED && bls.present? + AbfWorker::BuildListsPublishTaskManager.cleanup_packages_from_testing( + repository_status.platform_id, + repository_status.repository_id, + bls + ) + end update_rpm_builds end ensure From 6a6fcc321ffc4b58e2d2f76936302e70da810ee6 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 10 Jan 2014 02:56:11 +0400 Subject: [PATCH 2/6] #336: use set instead of list --- .../javascripts/angularjs/locales.js.erb | 4 +-- app/models/build_list.rb | 29 +++++++++---------- .../build_lists_publish_task_manager.rb | 23 +++++++-------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/app/assets/javascripts/angularjs/locales.js.erb b/app/assets/javascripts/angularjs/locales.js.erb index 5d5901301..fdd2c2d0c 100644 --- a/app/assets/javascripts/angularjs/locales.js.erb +++ b/app/assets/javascripts/angularjs/locales.js.erb @@ -11,7 +11,7 @@ var _locales = { 'Всего %1 тега', 'Всего %1 тегов' ], - <%= BuildList::STATUSES.map{|s| "'build_list.status.#{s}':'#{BuildList.human_status(s)}'"}.join(',') %> + <%= BuildList::STATUSES.map{|s| "'build_list.status.#{s}': '#{BuildList.human_status(s)}'"}.join(',') %> }, <%I18n.locale = :en%> 'en-us': { @@ -23,6 +23,6 @@ var _locales = { 'Total %1 tag', 'Total %1 tags' ], - <%= BuildList::STATUSES.map{|s| "'build_list.status.#{s}':'#{BuildList.human_status(s)}'"}.join(',') %> + <%= BuildList::STATUSES.map{|s| "'build_list.status.#{s}': '#{BuildList.human_status(s)}'"}.join(',') %> } }; \ No newline at end of file diff --git a/app/models/build_list.rb b/app/models/build_list.rb index 7c2b9d4e1..82ccf58b9 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -146,19 +146,16 @@ class BuildList < ActiveRecord::Base end end - after_transition do |build_list, transition| - status = HUMAN_STATUSES[build_list.status] - if build_list.mass_build && MassBuild::COUNT_STATUSES.include?(status) - MassBuild.increment_counter "#{status.to_s}_count", build_list.mass_build_id - end - end - after_transition :on => :place_build, :do => :add_job_to_abf_worker_queue, :if => lambda { |build_list| build_list.external_nodes.blank? } after_transition :on => :published, :do => [:set_version_and_tag, :actualize_packages] after_transition :on => :publish, :do => :set_publisher - after_transition :build_published_into_testing => :publish, :do => :cleanup_packages_from_testing + after_transition(:on => :publish) do |build_list, transition| + if transition.from == BUILD_PUBLISHED_INTO_TESTING + build_list.cleanup_packages_from_testing + end + end after_transition :on => :cancel, :do => :cancel_job after_transition :on => [:published, :fail_publish, :build_error, :tests_failed], :do => :notify_users @@ -489,6 +486,14 @@ class BuildList < ActiveRecord::Base } end + def cleanup_packages_from_testing + AbfWorker::BuildListsPublishTaskManager.cleanup_packages_from_testing( + build_for_platform_id, + save_to_repository_id, + id + ) + end + protected def create_container @@ -541,14 +546,6 @@ class BuildList < ActiveRecord::Base save end - def cleanup_packages_from_testing - AbfWorker::BuildListsPublishTaskManager.cleanup_packages_from_testing( - build_for_platform_id, - save_to_repository_id, - id - ) - end - def current_ability @current_ability ||= Ability.new(user) end diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 65cacf450..1b6c96e3e 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -62,9 +62,10 @@ module AbfWorker def cleanup_packages_from_testing(platform_id, repository_id, *build_lists) return if build_lists.blank? - key = "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{repository_id}-#{platform_id}" - redis.lpush REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, "#{repository_id}-#{platform_id}" - redis.lpush key, build_lists + rep_pl = "#{repository_id}-#{platform_id}" + key = "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{rep_pl}" + redis.sadd REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, "#{rep_pl}" + redis.sadd key, build_lists end def unlock_build_list(build_list) @@ -222,8 +223,8 @@ module AbfWorker locked_rep.present? && locked_rep.include?(rep.to_i) ? nil : [rep.to_i, pl.to_i] end.compact - for_cleanup_from_testing = @redis.lrange(REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, 0, -1).map do |key| - next if redis.llen("#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{key}") == 0 + for_cleanup_from_testing = @redis.smembers(REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING).map do |key| + next if @redis.scard("#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{key}") == 0 rep, pl = *key.split('-') locked_rep.present? && locked_rep.include?(rep.to_i) ? nil : [rep.to_i, pl.to_i] end.compact if testing @@ -270,24 +271,20 @@ module AbfWorker packages = JSON.parse packages old_packages[:sources] |= packages['sources'] Arch.pluck(:name).each do |arch| - old_packages[:binaries][arch.to_sym] |= packages['binaries'][arch] + old_packages[:binaries][arch.to_sym] |= packages['binaries'][arch] || [] end end if testing - build_lists_for_cleanup_from_testing = @redis.lrange( - "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}" - 0, -1 - ) + build_lists_for_cleanup_from_testing = @redis.smembers("#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}") BuildList.where(:id => build_lists_for_cleanup_from_testing).each do |b| self.class.fill_packages(b, old_packages) end if build_lists_for_cleanup_from_testing.present? end build_lists_for_cleanup_from_testing ||= [] - bl = build_lists.first - return false if !bl && old_packages[:sources].empty? + return false if !bl && old_packages[:binaries].values.flatten.empty? save_to_repository = Repository.find save_to_repository_id # Checks mirror sync status @@ -369,7 +366,7 @@ module AbfWorker end build_lists_for_cleanup_from_testing.each do |key| - @redis.lrem "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}", 0, key + @redis.srem "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}", key end return true From 4b4b1fe5d8552d698eae74427bb7c95934c034a2 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 10 Jan 2014 02:57:44 +0400 Subject: [PATCH 3/6] #336: rollback changes --- app/models/build_list.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/models/build_list.rb b/app/models/build_list.rb index 82ccf58b9..843cab408 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -146,6 +146,13 @@ class BuildList < ActiveRecord::Base end end + after_transition do |build_list, transition| + status = HUMAN_STATUSES[build_list.status] + if build_list.mass_build && MassBuild::COUNT_STATUSES.include?(status) + MassBuild.increment_counter "#{status.to_s}_count", build_list.mass_build_id + end + end + after_transition :on => :place_build, :do => :add_job_to_abf_worker_queue, :if => lambda { |build_list| build_list.external_nodes.blank? } after_transition :on => :published, From a16682c5f9b26e215d3b3910970180afaad38778 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 10 Jan 2014 03:01:01 +0400 Subject: [PATCH 4/6] #336: small refactoring --- lib/abf_worker/build_lists_publish_task_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 1b6c96e3e..83896cf54 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -64,7 +64,7 @@ module AbfWorker return if build_lists.blank? rep_pl = "#{repository_id}-#{platform_id}" key = "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{rep_pl}" - redis.sadd REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, "#{rep_pl}" + redis.sadd REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, rep_pl redis.sadd key, build_lists end From d101d33a7059e17f9dda6106f3e7ec59d2f68377 Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 10 Jan 2014 19:10:28 +0400 Subject: [PATCH 5/6] #336: misprint --- lib/abf_worker/build_lists_publish_task_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 83896cf54..7bf62fa99 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -278,7 +278,7 @@ module AbfWorker if testing build_lists_for_cleanup_from_testing = @redis.smembers("#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}") BuildList.where(:id => build_lists_for_cleanup_from_testing).each do |b| - self.class.fill_packages(b, old_packages) + self.class.fill_packages(b, old_packages, :fullname) end if build_lists_for_cleanup_from_testing.present? end build_lists_for_cleanup_from_testing ||= [] From 777e8c5b4327d378c159b7cf7c3151701ff1814e Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 10 Jan 2014 20:26:54 +0400 Subject: [PATCH 6/6] #336: rollback changes, small improvements --- lib/abf_worker/build_lists_publish_task_manager.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 7bf62fa99..e771ceeed 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -284,7 +284,7 @@ module AbfWorker build_lists_for_cleanup_from_testing ||= [] bl = build_lists.first - return false if !bl && old_packages[:binaries].values.flatten.empty? + return false if !bl && old_packages[:sources].empty? && old_packages[:binaries].values.flatten.empty? save_to_repository = Repository.find save_to_repository_id # Checks mirror sync status @@ -365,8 +365,13 @@ module AbfWorker @redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key end + rep_pl = "#{save_to_repository_id}-#{build_for_platform_id}" + r_key = "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{rep_pl}" build_lists_for_cleanup_from_testing.each do |key| - @redis.srem "#{BUILD_LISTS_FOR_CLEANUP_FROM_TESTING}-#{save_to_repository_id}-#{build_for_platform_id}", key + @redis.srem r_key, key + end + if @redis.scard(r_key) == 0 + @redis.srem REP_AND_PLS_OF_BUILD_LISTS_FOR_CLEANUP_FROM_TESTING, rep_pl end return true