From 183799d9f07afc71c0ecf054ee58c30a8d50afbe Mon Sep 17 00:00:00 2001 From: Vokhmin Alexey V Date: Fri, 23 Aug 2013 01:02:24 +0400 Subject: [PATCH] #276: updated migrations, models, BuildListsPublishTaskManager --- .../platforms/repositories_controller.rb | 3 +- app/models/key_pair.rb | 4 +- app/models/platform.rb | 5 ++ app/models/repository.rb | 15 +++- app/models/repository_status.rb | 74 ++++++++++++++--- ...130822154741_create_repository_statuses.rb | 2 +- .../20130822160501_add_status_to_platform.rb | 2 +- db/schema.rb | 27 +++++-- .../build_lists_publish_task_manager.rb | 79 ++++++------------- lib/abf_worker/publish_observer.rb | 12 ++- 10 files changed, 141 insertions(+), 82 deletions(-) diff --git a/app/controllers/platforms/repositories_controller.rb b/app/controllers/platforms/repositories_controller.rb index a1b7bfa06..82132f8f0 100644 --- a/app/controllers/platforms/repositories_controller.rb +++ b/app/controllers/platforms/repositories_controller.rb @@ -140,8 +140,7 @@ class Platforms::RepositoriesController < Platforms::BaseController end def regenerate_metadata - build_for_platform = Platform.main.find params[:build_for_platform_id] if @repository.platform.personal? - if AbfWorker::BuildListsPublishTaskManager.repository_regenerate_metadata @repository, (build_for_platform || @repository.platform) + if @repository.regenerate(params[:build_for_platform_id]) flash[:notice] = t('flash.repository.regenerate_in_queue') else flash[:error] = t('flash.repository.regenerate_already_in_queue') diff --git a/app/models/key_pair.rb b/app/models/key_pair.rb index e6bb7a246..17fb6c708 100644 --- a/app/models/key_pair.rb +++ b/app/models/key_pair.rb @@ -14,9 +14,7 @@ class KeyPair < ActiveRecord::Base validate :check_keys before_create { |record| record.key_id = @fingerprint } - after_create { |record| - AbfWorker::BuildListsPublishTaskManager.resign_repository(record) unless record.repository.platform.personal? - } + after_create { |record| record.repository.resign } protected diff --git a/app/models/platform.rb b/app/models/platform.rb index d6487af96..4526695e6 100644 --- a/app/models/platform.rb +++ b/app/models/platform.rb @@ -19,6 +19,7 @@ class Platform < ActiveRecord::Base has_many :products, :dependent => :destroy has_many :tokens, :as => :subject, :dependent => :destroy has_many :platform_arch_settings, :dependent => :destroy + has_many :repository_statuses has_many :relations, :as => :target, :dependent => :destroy has_many :actors, :as => :target, :class_name => 'Relation', :dependent => :destroy @@ -84,6 +85,10 @@ class Platform < ActiveRecord::Base end end + def human_status + HUMAN_STATUSES[status] + end + def clear system("rm -Rf #{ APP_CONFIG['root_path'] }/platforms/#{ self.name }/repository/*") end diff --git a/app/models/repository.rb b/app/models/repository.rb index 98d42f131..0dfb55dca 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -11,8 +11,8 @@ class Repository < ActiveRecord::Base has_many :project_to_repositories, :dependent => :destroy, :validate => true has_many :projects, :through => :project_to_repositories + has_many :repository_statuses, :dependent => :destroy has_one :key_pair, :dependent => :destroy - has_one :repository_statuses, :dependent => :destroy has_many :build_lists, :foreign_key => :save_to_repository_id, :dependent => :destroy @@ -26,6 +26,19 @@ class Repository < ActiveRecord::Base attr_accessible :name, :description, :publish_without_qa attr_readonly :name, :platform_id + def regenerate(build_for_platform_id) + build_for_platform = Platform.main.find build_for_platform_id if platform.personal? + status = repository_statuses.find_or_create_by_platform_id(build_for_platform.try(:id) || platform_id) + status.regenerate + end + + def resign + unless platform.personal? + status = repository_statuses.find_or_create_by_platform_id(platform_id) + status.resign + end + end + def base_clone(attrs = {}) dup.tap do |c| c.platform_id = nil diff --git a/app/models/repository_status.rb b/app/models/repository_status.rb index 588769bf8..100ddb386 100644 --- a/app/models/repository_status.rb +++ b/app/models/repository_status.rb @@ -1,31 +1,79 @@ class RepositoryStatus < ActiveRecord::Base READY = 0 - WAITING_FOR_REGENERATION = 100 - REGENERATING = 200 - PUBLISH = 300 + WAITING_FOR_RESIGN = 100 + WAITING_FOR_REGENERATION = 200 + REGENERATING = 300 + PUBLISH = 400 + RESIGN = 500 + WAITING_FOR_RESIGN_AFTER_PUBLISH = 600 + WAITING_FOR_RESIGN_AFTER_REGENERATION = 700 + WAITING_FOR_REGENERATION_AFTER_PUBLISH = 800 + WAITING_FOR_REGENERATION_AFTER_RESIGN = 900 + WAITING_FOR_RESIGN_AND_REGENERATION_AFTER_PUBLISH = 1000 + WAITING_FOR_RESIGN_AND_REGENERATION = 1100 - HUMAN_STATUSES = { READY => :ready, - WAITING_FOR_REGENERATION => :waiting_for_regeneration, - REGENERATING => :regenerating, - PUBLISH => :publish - }.freeze + + HUMAN_STATUSES = { + READY => :ready, + WAITING_FOR_RESIGN => :waiting_for_resign, + WAITING_FOR_REGENERATION => :waiting_for_regeneration, + REGENERATING => :regenerating, + PUBLISH => :publish, + RESIGN => :resign, + WAITING_FOR_RESIGN_AFTER_PUBLISH => :waiting_for_resign_after_publish, + WAITING_FOR_RESIGN_AFTER_REGENERATION => :waiting_for_resign_after_regeneration, + WAITING_FOR_REGENERATION_AFTER_PUBLISH => :waiting_for_regeneration_after_publish, + WAITING_FOR_REGENERATION_AFTER_RESIGN => :waiting_for_regeneration_after_resign, + WAITING_FOR_RESIGN_AND_REGENERATION_AFTER_PUBLISH => :waiting_for_resign_and_regeneration_after_publish, + WAITING_FOR_RESIGN_AND_REGENERATION => :waiting_for_resign_and_regeneration + }.freeze + + belongs_to :platform + belongs_to :repository validates :repository_id, :platform_id, :presence => true validates :repository_id, :uniqueness => {:scope => :platform_id} attr_accessible :last_regenerated_at, :last_regenerated_status, :platform_id, :repository_id, :status + scope :platform_ready, where(:platform => {:status => READY}).joins(:platform) + scope :for_regeneration, platform_ready.where(:status => WAITING_FOR_REGENERATION) + scope :for_resign, platform_ready.where(:status => [WAITING_FOR_RESIGN, WAITING_FOR_RESIGN_AND_REGENERATION]) + state_machine :status, :initial => :ready do event :ready do - transition [:regenerating, :publish] => :ready + transition [:regenerating, :publish, :resign] => :ready + transition [:waiting_for_resign_after_publish, :waiting_for_resign_after_regeneration] => :waiting_for_resign + transition [:waiting_for_regeneration_after_publish, :waiting_for_regeneration_after_resign] => :waiting_for_regeneration + transition :waiting_for_resign_and_regeneration_after_publish => :waiting_for_resign_and_regeneration end event :regenerate do - transition :waiting_for_regeneration => :regenerating transition :ready => :waiting_for_regeneration + transition :publish => :waiting_for_regeneration_after_publish + transition :resign => :waiting_for_regeneration_after_resign + transition :waiting_for_resign_after_publish => :waiting_for_resign_and_regeneration_after_publish + transition :waiting_for_resign => :waiting_for_resign_and_regeneration end - event :publish_packages do + event :start_regeneration do + transition :waiting_for_regeneration => :regenerating + end + + event :resign do + transition :ready => :waiting_for_resign + transition :publish => :waiting_for_resign_after_publish + transition :waiting_for_regeneration => :waiting_for_resign_and_regeneration + transition :waiting_for_regeneration_after_publish => :waiting_for_resign_and_regeneration_after_publish + transition :regenerating => :waiting_for_resign_after_regeneration + end + + event :start_resign do + transition :waiting_for_resign => :resign + transition :waiting_for_resign_and_regeneration => :waiting_for_regeneration_after_resign + end + + event :publish do transition :ready => :publish end @@ -34,4 +82,8 @@ class RepositoryStatus < ActiveRecord::Base end end + def human_status + HUMAN_STATUSES[status] + end + end diff --git a/db/migrate/20130822154741_create_repository_statuses.rb b/db/migrate/20130822154741_create_repository_statuses.rb index 0614488e4..3687c9532 100644 --- a/db/migrate/20130822154741_create_repository_statuses.rb +++ b/db/migrate/20130822154741_create_repository_statuses.rb @@ -3,7 +3,7 @@ class CreateRepositoryStatuses < ActiveRecord::Migration create_table :repository_statuses do |t| t.integer :repository_id, :null => false t.integer :platform_id, :null => false - t.integer :status + t.integer :status, :default => 0 t.datetime :last_regenerated_at t.integer :last_regenerated_status diff --git a/db/migrate/20130822160501_add_status_to_platform.rb b/db/migrate/20130822160501_add_status_to_platform.rb index 655dbec52..674858e73 100644 --- a/db/migrate/20130822160501_add_status_to_platform.rb +++ b/db/migrate/20130822160501_add_status_to_platform.rb @@ -1,6 +1,6 @@ class AddStatusToPlatform < ActiveRecord::Migration def change - add_column :platforms, :status, :integer + add_column :platforms, :status, :integer, :default => 0 add_column :platforms, :last_regenerated_at, :datetime add_column :platforms, :last_regenerated_status, :integer end diff --git a/db/schema.rb b/db/schema.rb index fb841f5fd..d79748bf5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 20130820195938) do +ActiveRecord::Schema.define(:version => 20130822160501) do create_table "activity_feeds", :force => true do |t| t.integer "user_id", :null => false @@ -307,16 +307,19 @@ ActiveRecord::Schema.define(:version => 20130820195938) do create_table "platforms", :force => true do |t| t.string "description" - t.string "name", :null => false + t.string "name", :null => false t.integer "parent_platform_id" t.datetime "created_at" t.datetime "updated_at" - t.boolean "released", :default => false, :null => false + t.boolean "released", :default => false, :null => false t.integer "owner_id" t.string "owner_type" - t.string "visibility", :default => "open", :null => false - t.string "platform_type", :default => "main", :null => false - t.string "distrib_type", :null => false + t.string "visibility", :default => "open", :null => false + t.string "platform_type", :default => "main", :null => false + t.string "distrib_type", :null => false + t.integer "status", :default => 0 + t.datetime "last_regenerated_at" + t.integer "last_regenerated_status" end add_index "platforms", ["name"], :name => "index_platforms_on_name", :unique => true, :case_sensitive => false @@ -484,6 +487,18 @@ ActiveRecord::Schema.define(:version => 20130820195938) do add_index "repositories", ["platform_id"], :name => "index_repositories_on_platform_id" + create_table "repository_statuses", :force => true do |t| + t.integer "repository_id", :null => false + t.integer "platform_id", :null => false + t.integer "status", :default => 0 + t.datetime "last_regenerated_at" + t.integer "last_regenerated_status" + t.datetime "created_at", :null => false + t.datetime "updated_at", :null => false + end + + add_index "repository_statuses", ["repository_id", "platform_id"], :name => "index_repository_statuses_on_repository_id_and_platform_id", :unique => true + create_table "settings_notifiers", :force => true do |t| t.integer "user_id", :null => false t.boolean "can_notify", :default => true diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 1a0230c29..fb7ff85ef 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -10,8 +10,7 @@ module AbfWorker LOCKED_REPOSITORIES LOCKED_REP_AND_PLATFORMS LOCKED_BUILD_LISTS - PACKAGES_FOR_CLEANUP - REGENERATE_METADATA).each do |kind| + PACKAGES_FOR_CLEANUP).each do |kind| const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}" end @@ -53,16 +52,6 @@ module AbfWorker end end - def resign_repository(key_pair) - redis.lpush RESIGN_REPOSITORIES, key_pair.repository_id - end - - def repository_regenerate_metadata(repository, build_for_platform) - key = "#{repository.id}-#{build_for_platform.id}" - return false if Resque.redis.lrange(REGENERATE_METADATA, 0, -1).include? key - redis.lpush REGENERATE_METADATA, key - end - def unlock_repository(repository_id) redis.lrem LOCKED_REPOSITORIES, 0, repository_id end @@ -173,14 +162,11 @@ module AbfWorker end def create_tasks_for_resign_repositories - resign_repos = @redis.lrange RESIGN_REPOSITORIES, 0, -1 - - Repository.where(:id => (resign_repos - locked_repositories)).each do |r| + repository_statuses = RepositoryStatus.for_resign.includes(:repository => :platform) + repository_statuses.each do |repository_status| + r = repository_status.repository # Checks mirror sync status next if r.repo_lock_file_exists? - @redis.lrem RESIGN_REPOSITORIES, 0, r.id - @redis.lpush LOCKED_REPOSITORIES, r.id - distrib_type = r.platform.distrib_type cmd_params = { @@ -204,9 +190,10 @@ module AbfWorker :repository => {:id => r.id}, :type => :resign, :skip_feedback => true, - :time_living => 9600 # 160 min + :time_living => 9600, # 160 min + :extra => {:repository_status_id => repository_status.id} }] - ) + ) if repository_status.start_resign end end @@ -353,30 +340,26 @@ module AbfWorker end def create_tasks_for_repository_regenerate_metadata - worker_queue = 'publish_worker_default' - worker_class = 'AbfWorker::PublishWorkerDefault' - regen_repos_and_pl = @redis.lrange REGENERATE_METADATA, 0, -1 - locked_rep_and_pl = @redis.lrange(LOCKED_REP_AND_PLATFORMS, 0, -1) - - regen_repos = regen_repos_and_pl.map{ |r| r.gsub(/\-[\d]*$/, '') } - Repository.where(:id => regen_repos).each do |rep| + repository_statuses = RepositoryStatus.for_regeneration.includes(:repository => :platform) + repository_statuses.each do |repository_status| + rep = repository_status.repository # Checks mirror sync status next if rep.repo_lock_file_exists? - regen_repos_and_pl.select{ |kind| kind =~ /^#{rep.id}\-/ }.each do |lock_str| - next if locked_rep_and_pl.include?(lock_str) - @redis.lrem REGENERATE_METADATA, 0, lock_str - build_for_platform = Platform.find lock_str.gsub(/^[\d]*\-/, '') - cmd_params = { - 'RELEASED' => rep.platform.released, - 'REPOSITORY_NAME' => rep.name, - 'TYPE' => build_for_platform.distrib_type, - 'REGENERATE_METADATA' => true, - 'SAVE_TO_PLATFORM' => rep.platform.name, - 'BUILD_FOR_PLATFORM' => build_for_platform.name - }.map{ |k, v| "#{k}=#{v}" }.join(' ') + build_for_platform = repository_statuses.platform + cmd_params = { + 'RELEASED' => rep.platform.released, + 'REPOSITORY_NAME' => rep.name, + 'TYPE' => build_for_platform.distrib_type, + 'REGENERATE_METADATA' => true, + 'SAVE_TO_PLATFORM' => rep.platform.name, + 'BUILD_FOR_PLATFORM' => build_for_platform.name + }.map{ |k, v| "#{k}=#{v}" }.join(' ') - options = { + Resque.push( + 'publish_worker_default', + 'class' => 'AbfWorker::PublishWorkerDefault', + 'args' => [{ :id => Time.now.to_i, :cmd_params => cmd_params, :platform => { @@ -389,20 +372,10 @@ module AbfWorker :type => :publish, :time_living => 9600, # 160 min :skip_feedback => true, - :extra => {:lock_str => lock_str, :regenerate => true} - } - - Resque.push( - worker_queue, - 'class' => worker_class, - 'args' => [options.merge({ - })] - ) - - @redis.lpush(LOCKED_REP_AND_PLATFORMS, lock_str) - end + :extra => {:repository_status_id => repository_status.id, :regenerate => true} + }] + ) if repository_status.start_regeneration end - return true end end end diff --git a/lib/abf_worker/publish_observer.rb b/lib/abf_worker/publish_observer.rb index a6166535a..3a4a52d46 100644 --- a/lib/abf_worker/publish_observer.rb +++ b/lib/abf_worker/publish_observer.rb @@ -9,11 +9,13 @@ module AbfWorker def perform return if status == STARTED # do nothing when publication started - if options['type'] == 'resign' - AbfWorker::BuildListsPublishTaskManager.unlock_repository options['id'] - else + repository_status = RepositoryStatus.where(:id => options['extra']['repository_status_id']).first + begin if options['extra']['regenerate'] # Regenerate metadata - AbfWorker::BuildListsPublishTaskManager.unlock_rep_and_platform options['extra']['lock_str'] + if repository_status + repository_status.last_regenerated_at = Time.now.utc + repository_status.last_regenerated_status = status + end elsif options['extra']['create_container'] # Container has been created case status when COMPLETED @@ -25,6 +27,8 @@ module AbfWorker else update_rpm_builds end + ensure + repository_status.ready if repository_status end end