Software
/
rosa-build
mirror of https://github.com/OpenMandrivaSoftware/rosa-build.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

#276: updated migrations, models, BuildListsPublishTaskManager

This commit is contained in:
Vokhmin Alexey V 2013-08-23 01:02:24 +04:00
parent f5c1509281
commit 183799d9f0
10 changed files with 141 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/controllers/platforms/repositories_controller.rb
View File

@ -140,8 +140,7 @@ class Platforms::RepositoriesController < Platforms::BaseController
end
def regenerate_metadata
build_for_platform = Platform.main.find params[:build_for_platform_id] if @repository.platform.personal?
if AbfWorker::BuildListsPublishTaskManager.repository_regenerate_metadata @repository, (build_for_platform || @repository.platform)
if @repository.regenerate(params[:build_for_platform_id])
flash[:notice] = t('flash.repository.regenerate_in_queue')
else
flash[:error] = t('flash.repository.regenerate_already_in_queue')

4
app/models/key_pair.rb
View File

@ -14,9 +14,7 @@ class KeyPair < ActiveRecord::Base
validate :check_keys
before_create { |record| record.key_id = @fingerprint }
after_create { |record|
AbfWorker::BuildListsPublishTaskManager.resign_repository(record) unless record.repository.platform.personal?
}
after_create { |record| record.repository.resign }
protected

5
app/models/platform.rb
View File

@ -19,6 +19,7 @@ class Platform < ActiveRecord::Base
has_many :products, :dependent => :destroy
has_many :tokens, :as => :subject, :dependent => :destroy
has_many :platform_arch_settings, :dependent => :destroy
has_many :repository_statuses
has_many :relations, :as => :target, :dependent => :destroy
has_many :actors, :as => :target, :class_name => 'Relation', :dependent => :destroy
@ -84,6 +85,10 @@ class Platform < ActiveRecord::Base
end
end
def human_status
HUMAN_STATUSES[status]
end
def clear
system("rm -Rf #{ APP_CONFIG['root_path'] }/platforms/#{ self.name }/repository/*")
end

15
app/models/repository.rb
View File

@ -11,8 +11,8 @@ class Repository < ActiveRecord::Base
has_many :project_to_repositories, :dependent => :destroy, :validate => true
has_many :projects, :through => :project_to_repositories
has_many :repository_statuses, :dependent => :destroy
has_one :key_pair, :dependent => :destroy
has_one :repository_statuses, :dependent => :destroy
has_many :build_lists, :foreign_key => :save_to_repository_id, :dependent => :destroy
@ -26,6 +26,19 @@ class Repository < ActiveRecord::Base
attr_accessible :name, :description, :publish_without_qa
attr_readonly :name, :platform_id
def regenerate(build_for_platform_id)
build_for_platform = Platform.main.find build_for_platform_id if platform.personal?
status = repository_statuses.find_or_create_by_platform_id(build_for_platform.try(:id) || platform_id)
status.regenerate
end
def resign
unless platform.personal?
status = repository_statuses.find_or_create_by_platform_id(platform_id)
status.resign
end
end
def base_clone(attrs = {})
dup.tap do |c|
c.platform_id = nil

74
app/models/repository_status.rb
View File

@ -1,31 +1,79 @@
class RepositoryStatus < ActiveRecord::Base
READY = 0
WAITING_FOR_REGENERATION = 100
REGENERATING = 200
PUBLISH = 300
WAITING_FOR_RESIGN = 100
WAITING_FOR_REGENERATION = 200
REGENERATING = 300
PUBLISH = 400
RESIGN = 500
WAITING_FOR_RESIGN_AFTER_PUBLISH = 600
WAITING_FOR_RESIGN_AFTER_REGENERATION = 700
WAITING_FOR_REGENERATION_AFTER_PUBLISH = 800
WAITING_FOR_REGENERATION_AFTER_RESIGN = 900
WAITING_FOR_RESIGN_AND_REGENERATION_AFTER_PUBLISH = 1000
WAITING_FOR_RESIGN_AND_REGENERATION = 1100
HUMAN_STATUSES = { READY => :ready,
WAITING_FOR_REGENERATION => :waiting_for_regeneration,
REGENERATING => :regenerating,
PUBLISH => :publish
}.freeze
HUMAN_STATUSES = {
READY => :ready,
WAITING_FOR_RESIGN => :waiting_for_resign,
WAITING_FOR_REGENERATION => :waiting_for_regeneration,
REGENERATING => :regenerating,
PUBLISH => :publish,
RESIGN => :resign,
WAITING_FOR_RESIGN_AFTER_PUBLISH => :waiting_for_resign_after_publish,
WAITING_FOR_RESIGN_AFTER_REGENERATION => :waiting_for_resign_after_regeneration,
WAITING_FOR_REGENERATION_AFTER_PUBLISH => :waiting_for_regeneration_after_publish,
WAITING_FOR_REGENERATION_AFTER_RESIGN => :waiting_for_regeneration_after_resign,
WAITING_FOR_RESIGN_AND_REGENERATION_AFTER_PUBLISH => :waiting_for_resign_and_regeneration_after_publish,
WAITING_FOR_RESIGN_AND_REGENERATION => :waiting_for_resign_and_regeneration
}.freeze
belongs_to :platform
belongs_to :repository
validates :repository_id, :platform_id, :presence => true
validates :repository_id, :uniqueness => {:scope => :platform_id}
attr_accessible :last_regenerated_at, :last_regenerated_status, :platform_id, :repository_id, :status
scope :platform_ready, where(:platform => {:status => READY}).joins(:platform)
scope :for_regeneration, platform_ready.where(:status => WAITING_FOR_REGENERATION)
scope :for_resign, platform_ready.where(:status => [WAITING_FOR_RESIGN, WAITING_FOR_RESIGN_AND_REGENERATION])
state_machine :status, :initial => :ready do
event :ready do
transition [:regenerating, :publish] => :ready
transition [:regenerating, :publish, :resign] => :ready
transition [:waiting_for_resign_after_publish, :waiting_for_resign_after_regeneration] => :waiting_for_resign
transition [:waiting_for_regeneration_after_publish, :waiting_for_regeneration_after_resign] => :waiting_for_regeneration
transition :waiting_for_resign_and_regeneration_after_publish => :waiting_for_resign_and_regeneration
end
event :regenerate do
transition :waiting_for_regeneration => :regenerating
transition :ready => :waiting_for_regeneration
transition :publish => :waiting_for_regeneration_after_publish
transition :resign => :waiting_for_regeneration_after_resign
transition :waiting_for_resign_after_publish => :waiting_for_resign_and_regeneration_after_publish
transition :waiting_for_resign => :waiting_for_resign_and_regeneration
end
event :publish_packages do
event :start_regeneration do
transition :waiting_for_regeneration => :regenerating
end
event :resign do
transition :ready => :waiting_for_resign
transition :publish => :waiting_for_resign_after_publish
transition :waiting_for_regeneration => :waiting_for_resign_and_regeneration
transition :waiting_for_regeneration_after_publish => :waiting_for_resign_and_regeneration_after_publish
transition :regenerating => :waiting_for_resign_after_regeneration
end
event :start_resign do
transition :waiting_for_resign => :resign
transition :waiting_for_resign_and_regeneration => :waiting_for_regeneration_after_resign
end
event :publish do
transition :ready => :publish
end
@ -34,4 +82,8 @@ class RepositoryStatus < ActiveRecord::Base
end
end
def human_status
HUMAN_STATUSES[status]
end
end

2
db/migrate/20130822154741_create_repository_statuses.rb
View File

@ -3,7 +3,7 @@ class CreateRepositoryStatuses < ActiveRecord::Migration
create_table :repository_statuses do |t|
t.integer :repository_id, :null => false
t.integer :platform_id, :null => false
t.integer :status
t.integer :status, :default => 0
t.datetime :last_regenerated_at
t.integer :last_regenerated_status

2
db/migrate/20130822160501_add_status_to_platform.rb
View File

@ -1,6 +1,6 @@
class AddStatusToPlatform < ActiveRecord::Migration
def change
add_column :platforms, :status, :integer
add_column :platforms, :status, :integer, :default => 0
add_column :platforms, :last_regenerated_at, :datetime
add_column :platforms, :last_regenerated_status, :integer
end

27
db/schema.rb
View File

@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
ActiveRecord::Schema.define(:version => 20130820195938) do
ActiveRecord::Schema.define(:version => 20130822160501) do
create_table "activity_feeds", :force => true do |t|
t.integer "user_id", :null => false
@ -307,16 +307,19 @@ ActiveRecord::Schema.define(:version => 20130820195938) do
create_table "platforms", :force => true do |t|
t.string "description"
t.string "name", :null => false
t.string "name", :null => false
t.integer "parent_platform_id"
t.datetime "created_at"
t.datetime "updated_at"
t.boolean "released", :default => false, :null => false
t.boolean "released", :default => false, :null => false
t.integer "owner_id"
t.string "owner_type"
t.string "visibility", :default => "open", :null => false
t.string "platform_type", :default => "main", :null => false
t.string "distrib_type", :null => false
t.string "visibility", :default => "open", :null => false
t.string "platform_type", :default => "main", :null => false
t.string "distrib_type", :null => false
t.integer "status", :default => 0
t.datetime "last_regenerated_at"
t.integer "last_regenerated_status"
end
add_index "platforms", ["name"], :name => "index_platforms_on_name", :unique => true, :case_sensitive => false
@ -484,6 +487,18 @@ ActiveRecord::Schema.define(:version => 20130820195938) do
add_index "repositories", ["platform_id"], :name => "index_repositories_on_platform_id"
create_table "repository_statuses", :force => true do |t|
t.integer "repository_id", :null => false
t.integer "platform_id", :null => false
t.integer "status", :default => 0
t.datetime "last_regenerated_at"
t.integer "last_regenerated_status"
t.datetime "created_at", :null => false
t.datetime "updated_at", :null => false
end
add_index "repository_statuses", ["repository_id", "platform_id"], :name => "index_repository_statuses_on_repository_id_and_platform_id", :unique => true
create_table "settings_notifiers", :force => true do |t|
t.integer "user_id", :null => false
t.boolean "can_notify", :default => true

79
lib/abf_worker/build_lists_publish_task_manager.rb
View File

@ -10,8 +10,7 @@ module AbfWorker
LOCKED_REPOSITORIES
LOCKED_REP_AND_PLATFORMS
LOCKED_BUILD_LISTS
PACKAGES_FOR_CLEANUP
REGENERATE_METADATA).each do |kind|
PACKAGES_FOR_CLEANUP).each do |kind|
const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}"
end
@ -53,16 +52,6 @@ module AbfWorker
end
end
def resign_repository(key_pair)
redis.lpush RESIGN_REPOSITORIES, key_pair.repository_id
end
def repository_regenerate_metadata(repository, build_for_platform)
key = "#{repository.id}-#{build_for_platform.id}"
return false if Resque.redis.lrange(REGENERATE_METADATA, 0, -1).include? key
redis.lpush REGENERATE_METADATA, key
end
def unlock_repository(repository_id)
redis.lrem LOCKED_REPOSITORIES, 0, repository_id
end
@ -173,14 +162,11 @@ module AbfWorker
end
def create_tasks_for_resign_repositories
resign_repos = @redis.lrange RESIGN_REPOSITORIES, 0, -1
Repository.where(:id => (resign_repos - locked_repositories)).each do |r|
repository_statuses = RepositoryStatus.for_resign.includes(:repository => :platform)
repository_statuses.each do |repository_status|
r = repository_status.repository
# Checks mirror sync status
next if r.repo_lock_file_exists?
@redis.lrem RESIGN_REPOSITORIES, 0, r.id
@redis.lpush LOCKED_REPOSITORIES, r.id
distrib_type = r.platform.distrib_type
cmd_params = {
@ -204,9 +190,10 @@ module AbfWorker
:repository => {:id => r.id},
:type => :resign,
:skip_feedback => true,
:time_living => 9600 # 160 min
:time_living => 9600, # 160 min
:extra => {:repository_status_id => repository_status.id}
}]
)
) if repository_status.start_resign
end
end
@ -353,30 +340,26 @@ module AbfWorker
end
def create_tasks_for_repository_regenerate_metadata
worker_queue = 'publish_worker_default'
worker_class = 'AbfWorker::PublishWorkerDefault'
regen_repos_and_pl = @redis.lrange REGENERATE_METADATA, 0, -1
locked_rep_and_pl = @redis.lrange(LOCKED_REP_AND_PLATFORMS, 0, -1)
regen_repos = regen_repos_and_pl.map{ |r| r.gsub(/\-[\d]*$/, '') }
Repository.where(:id => regen_repos).each do |rep|
repository_statuses = RepositoryStatus.for_regeneration.includes(:repository => :platform)
repository_statuses.each do |repository_status|
rep = repository_status.repository
# Checks mirror sync status
next if rep.repo_lock_file_exists?
regen_repos_and_pl.select{ |kind| kind =~ /^#{rep.id}\-/ }.each do |lock_str|
next if locked_rep_and_pl.include?(lock_str)
@redis.lrem REGENERATE_METADATA, 0, lock_str
build_for_platform = Platform.find lock_str.gsub(/^[\d]*\-/, '')
cmd_params = {
'RELEASED' => rep.platform.released,
'REPOSITORY_NAME' => rep.name,
'TYPE' => build_for_platform.distrib_type,
'REGENERATE_METADATA' => true,
'SAVE_TO_PLATFORM' => rep.platform.name,
'BUILD_FOR_PLATFORM' => build_for_platform.name
}.map{ |k, v| "#{k}=#{v}" }.join(' ')
build_for_platform = repository_statuses.platform
cmd_params = {
'RELEASED' => rep.platform.released,
'REPOSITORY_NAME' => rep.name,
'TYPE' => build_for_platform.distrib_type,
'REGENERATE_METADATA' => true,
'SAVE_TO_PLATFORM' => rep.platform.name,
'BUILD_FOR_PLATFORM' => build_for_platform.name
}.map{ |k, v| "#{k}=#{v}" }.join(' ')
options = {
Resque.push(
'publish_worker_default',
'class' => 'AbfWorker::PublishWorkerDefault',
'args' => [{
:id => Time.now.to_i,
:cmd_params => cmd_params,
:platform => {
@ -389,20 +372,10 @@ module AbfWorker
:type => :publish,
:time_living => 9600, # 160 min
:skip_feedback => true,
:extra => {:lock_str => lock_str, :regenerate => true}
}
Resque.push(
worker_queue,
'class' => worker_class,
'args' => [options.merge({
})]
)
@redis.lpush(LOCKED_REP_AND_PLATFORMS, lock_str)
end
:extra => {:repository_status_id => repository_status.id, :regenerate => true}
}]
) if repository_status.start_regeneration
end
return true
end
end
end

12
lib/abf_worker/publish_observer.rb
View File

@ -9,11 +9,13 @@ module AbfWorker
def perform
return if status == STARTED # do nothing when publication started
if options['type'] == 'resign'
AbfWorker::BuildListsPublishTaskManager.unlock_repository options['id']
else
repository_status = RepositoryStatus.where(:id => options['extra']['repository_status_id']).first
begin
if options['extra']['regenerate'] # Regenerate metadata
AbfWorker::BuildListsPublishTaskManager.unlock_rep_and_platform options['extra']['lock_str']
if repository_status
repository_status.last_regenerated_at = Time.now.utc
repository_status.last_regenerated_status = status
end
elsif options['extra']['create_container'] # Container has been created
case status
when COMPLETED
@ -25,6 +27,8 @@ module AbfWorker
else
update_rpm_builds
end
ensure
repository_status.ready if repository_status
end
end