Software
/
rosa-build
mirror of https://github.com/OpenMandrivaSoftware/rosa-build.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #279 from abf/rosa-build:276-generate-additional-metadata 

#276:  Generate additional metadata for Software Center
This commit is contained in:
avm 2013-08-26 20:40:07 +04:00
parent 09b1aba71b 12ceaefef4
commit 29cb1932aa
22 changed files with 461 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/controllers/platforms/platforms_controller.rb
View File

@ -56,6 +56,15 @@ class Platforms::PlatformsController < Platforms::BaseController
end
end
def regenerate_metadata
if @platform.regenerate
flash[:notice] = I18n.t('flash.platform.saved')
else
flash[:error] = I18n.t('flash.platform.save_error')
end
redirect_to edit_platform_path(@platform)
end
def change_visibility
if @platform.change_visibility
flash[:notice] = I18n.t("flash.platform.saved")

3
app/controllers/platforms/repositories_controller.rb
View File

@ -140,8 +140,7 @@ class Platforms::RepositoriesController < Platforms::BaseController
end
def regenerate_metadata
build_for_platform = Platform.main.find params[:build_for_platform_id] if @repository.platform.personal?
if AbfWorker::BuildListsPublishTaskManager.repository_regenerate_metadata @repository, (build_for_platform || @repository.platform)
if @repository.regenerate(params[:build_for_platform_id])
flash[:notice] = t('flash.repository.regenerate_in_queue')
else
flash[:error] = t('flash.repository.regenerate_already_in_queue')

3
app/models/ability.rb
View File

@ -122,6 +122,7 @@ class Ability
can([:remove_members, :remove_member, :add_member, :signatures], Repository) {|repository| owner?(repository.platform) || local_admin?(repository.platform)}
can([:add_project, :remove_project], Repository) {|repository| repository.members.exists?(:id => user.id)}
can(:clear, Platform) {|platform| owner?(platform) && platform.personal?}
can(:regenerate_metadata, Platform) {|platform| owner?(platform) || local_admin?(platform)}
can([:settings, :destroy, :edit, :update], Repository) {|repository| owner? repository.platform}
can([:create, :destroy], KeyPair) {|key_pair| owner?(key_pair.repository.platform) || local_admin?(key_pair.repository.platform)}
@ -161,7 +162,7 @@ class Ability
end
# Shared cannot rights for all users (registered, admin)
cannot :destroy, Platform, :platform_type => 'personal'
cannot [:regenerate_metadata, :destroy], Platform, :platform_type => 'personal'
cannot [:create, :destroy], Repository, :platform => {:platform_type => 'personal'}, :name => 'main'
cannot [:remove_members, :remove_member, :add_member, :sync_lock_file, :add_repo_lock_file, :remove_repo_lock_file], Repository, :platform => {:platform_type => 'personal'}

27
app/models/concerns/regeneration_status.rb Normal file
View File

@ -0,0 +1,27 @@
module RegenerationStatus
def human_regeneration_status
self.class::HUMAN_REGENERATION_STATUSES[last_regenerated_status] || :no_data
end
def human_status
self.class::HUMAN_STATUSES[status] || :no_data
end
READY = 0
WAITING_FOR_REGENERATION = 100
REGENERATING = 200
HUMAN_STATUSES = {
READY => :ready,
WAITING_FOR_REGENERATION => :waiting_for_regeneration,
REGENERATING => :regenerating
}
HUMAN_REGENERATION_STATUSES = {
AbfWorker::BaseObserver::COMPLETED => :completed,
AbfWorker::BaseObserver::FAILED => :failed,
AbfWorker::BaseObserver::CANCELED => :canceled
}.freeze
end

4
app/models/key_pair.rb
View File

@ -14,9 +14,7 @@ class KeyPair < ActiveRecord::Base
validate :check_keys
before_create { |record| record.key_id = @fingerprint }
after_create { |record|
AbfWorker::BuildListsPublishTaskManager.resign_repository(record) unless record.repository.platform.personal?
}
after_create { |record| record.repository.resign }
protected

23
app/models/platform.rb
View File

@ -1,7 +1,10 @@
# -*- encoding : utf-8 -*-
class Platform < ActiveRecord::Base
include RegenerationStatus
VISIBILITIES = %w(open hidden)
NAME_PATTERN = /[\w\-\.]+/
HUMAN_STATUSES = HUMAN_STATUSES.clone.freeze
belongs_to :parent, :class_name => 'Platform', :foreign_key => 'parent_platform_id'
belongs_to :owner, :polymorphic => true
@ -10,6 +13,7 @@ class Platform < ActiveRecord::Base
has_many :products, :dependent => :destroy
has_many :tokens, :as => :subject, :dependent => :destroy
has_many :platform_arch_settings, :dependent => :destroy
has_many :repository_statuses
has_many :relations, :as => :target, :dependent => :destroy
has_many :actors, :as => :target, :class_name => 'Relation', :dependent => :destroy
@ -53,6 +57,7 @@ class Platform < ActiveRecord::Base
scope :by_type, lambda {|type| where(:platform_type => type) if type.present?}
scope :main, by_type('main')
scope :personal, by_type('personal')
scope :waiting_for_regeneration, where(:status => WAITING_FOR_REGENERATION)
accepts_nested_attributes_for :platform_arch_settings, :allow_destroy => true
attr_accessible :name, :distrib_type, :parent_platform_id, :platform_type, :owner, :visibility, :description, :released, :platform_arch_settings_attributes
@ -60,6 +65,24 @@ class Platform < ActiveRecord::Base
include Modules::Models::Owner
state_machine :status, :initial => :ready do
event :ready do
transition :regenerating => :ready
end
event :regenerate do
transition :ready => :waiting_for_regeneration, :if => lambda{ |p| p.main? }
end
event :start_regeneration do
transition :waiting_for_regeneration => :regenerating
end
HUMAN_STATUSES.each do |code,name|
state name, :value => code
end
end
def clear
system("rm -Rf #{ APP_CONFIG['root_path'] }/platforms/#{ self.name }/repository/*")
end

14
app/models/repository.rb
View File

@ -11,6 +11,7 @@ class Repository < ActiveRecord::Base
has_many :project_to_repositories, :dependent => :destroy, :validate => true
has_many :projects, :through => :project_to_repositories
has_many :repository_statuses, :dependent => :destroy
has_one :key_pair, :dependent => :destroy
has_many :build_lists, :foreign_key => :save_to_repository_id, :dependent => :destroy
@ -25,6 +26,19 @@ class Repository < ActiveRecord::Base
attr_accessible :name, :description, :publish_without_qa
attr_readonly :name, :platform_id
def regenerate(build_for_platform_id = nil)
build_for_platform = Platform.main.find build_for_platform_id if platform.personal?
status = repository_statuses.find_or_create_by_platform_id(build_for_platform.try(:id) || platform_id)
status.regenerate
end
def resign
if platform.main?
status = repository_statuses.find_or_create_by_platform_id(platform_id)
status.resign
end
end
def base_clone(attrs = {})
dup.tap do |c|
c.platform_id = nil

83
app/models/repository_status.rb Normal file
View File

@ -0,0 +1,83 @@
class RepositoryStatus < ActiveRecord::Base
include RegenerationStatus
WAITING_FOR_RESIGN = 300
PUBLISH = 400
RESIGN = 500
WAITING_FOR_RESIGN_AFTER_PUBLISH = 600
WAITING_FOR_RESIGN_AFTER_REGENERATION = 700
WAITING_FOR_REGENERATION_AFTER_PUBLISH = 800
WAITING_FOR_REGENERATION_AFTER_RESIGN = 900
WAITING_FOR_RESIGN_AND_REGENERATION_AFTER_PUBLISH = 1000
WAITING_FOR_RESIGN_AND_REGENERATION = 1100
HUMAN_STATUSES = HUMAN_STATUSES.clone.merge({
WAITING_FOR_RESIGN => :waiting_for_resign,
PUBLISH => :publish,
RESIGN => :resign,
WAITING_FOR_RESIGN_AFTER_PUBLISH => :waiting_for_resign_after_publish,
WAITING_FOR_RESIGN_AFTER_REGENERATION => :waiting_for_resign_after_regeneration,
WAITING_FOR_REGENERATION_AFTER_PUBLISH => :waiting_for_regeneration_after_publish,
WAITING_FOR_REGENERATION_AFTER_RESIGN => :waiting_for_regeneration_after_resign,
WAITING_FOR_RESIGN_AND_REGENERATION_AFTER_PUBLISH => :waiting_for_resign_and_regeneration_after_publish,
WAITING_FOR_RESIGN_AND_REGENERATION => :waiting_for_resign_and_regeneration
}).freeze
belongs_to :platform
belongs_to :repository
validates :repository_id, :platform_id, :presence => true
validates :repository_id, :uniqueness => {:scope => :platform_id}
attr_accessible :platform_id, :repository_id
scope :platform_ready, where(:platforms => {:status => READY}).joins(:platform)
scope :for_regeneration, where(:status => WAITING_FOR_REGENERATION)
scope :for_resign, where(:status => [WAITING_FOR_RESIGN, WAITING_FOR_RESIGN_AND_REGENERATION])
scope :not_ready, where('repository_statuses.status != ?', READY)
state_machine :status, :initial => :ready do
event :ready do
transition [:regenerating, :publish, :resign] => :ready
transition [:waiting_for_resign_after_publish, :waiting_for_resign_after_regeneration] => :waiting_for_resign
transition [:waiting_for_regeneration_after_publish, :waiting_for_regeneration_after_resign] => :waiting_for_regeneration
transition :waiting_for_resign_and_regeneration_after_publish => :waiting_for_resign_and_regeneration
end
event :regenerate do
transition :ready => :waiting_for_regeneration
transition :publish => :waiting_for_regeneration_after_publish
transition :resign => :waiting_for_regeneration_after_resign
transition :waiting_for_resign_after_publish => :waiting_for_resign_and_regeneration_after_publish
transition :waiting_for_resign => :waiting_for_resign_and_regeneration
end
event :start_regeneration do
transition :waiting_for_regeneration => :regenerating
transition :waiting_for_resign_and_regeneration => :waiting_for_resign_after_regeneration
end
event :resign do
transition :ready => :waiting_for_resign
transition :publish => :waiting_for_resign_after_publish
transition :waiting_for_regeneration => :waiting_for_resign_and_regeneration
transition :waiting_for_regeneration_after_publish => :waiting_for_resign_and_regeneration_after_publish
transition :regenerating => :waiting_for_resign_after_regeneration
end
event :start_resign do
transition :waiting_for_resign => :resign
transition :waiting_for_resign_and_regeneration => :waiting_for_regeneration_after_resign
end
event :publish do
transition :ready => :publish
end
HUMAN_STATUSES.each do |code,name|
state name, :value => code
end
end
end

16
app/views/platforms/platforms/edit.html.haml
View File

@ -15,6 +15,22 @@
:class => 'button'
.both
- if can? :regenerate_metadata, @platform
.hr
%h3= t('layout.platforms.metadata')
.leftlist= t('activerecord.attributes.regeneration_status.status')
.rightlist= t("layout.regeneration_statuses.statuses.#{@platform.human_status}")
.both
.leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_at')
.rightlist= @platform.last_regenerated_at
.both
.leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_status')
.rightlist= t("layout.regeneration_statuses.last_regenerated_statuses.#{@platform.human_regeneration_status}")
.both
.leftside
.rightside= link_to t('layout.regeneration_statuses.regenerate_metadata'), regenerate_metadata_platform_path(@platform), :method => :put, :confirm => t('layout.confirm'), :class => 'button'
.both
- if can? :destroy, @platform
.hr
.leftside= t("layout.platforms.delete_warning")

19
app/views/platforms/repositories/edit.html.haml
View File

@ -19,6 +19,25 @@
= f.submit t('layout.repositories.regenerate_metadata'), :confirm => t('layout.confirm')
.both
- if @repository.repository_statuses.present?
%table#myTable.tablesorter.platform-repos{:cellspacing => "0", :cellpadding => "0"}
%thead
%tr
%th= t('activerecord.attributes.regeneration_status.status')
%th= t('activerecord.attributes.regeneration_status.last_regenerated_status')
%th= t('activerecord.attributes.regeneration_status.last_regenerated_at')
- unless @platform.main?
%th= t('activerecord.models.platform')
%tbody
- @repository.repository_statuses.sort_by{ |s| s.platform.name }.each do |status|
%tr{:class => cycle('odd', 'even')}
%td= t("layout.regeneration_statuses.statuses.#{status.human_status}")
%td= t("layout.regeneration_statuses.last_regenerated_statuses.#{status.human_regeneration_status}")
%td= status.last_regenerated_at
- unless @platform.main?
%td= status.platform.name
.both
- if @platform.main?
- if @repository.sync_lock_file_exists?
- label = t('layout.repositories.remove_sync_lock_file')

1
config/application.rb
View File

@ -28,6 +28,7 @@ module Rosa
# Custom directories with classes and modules you want to be autoloadable.
# config.autoload_paths += %W(#{config.root}/extras)
config.autoload_paths += %W(#{config.root}/app/presenters)
config.autoload_paths += %W(#{config.root}/app/models/concerns)
# Only load the plugins named here, in the order given (default is alphabetical).
# :all can be used as a placeholder for all plugins not explicitly named.

1
config/locales/models/platform.en.yml
View File

@ -52,6 +52,7 @@ en:
change_visibility_from_hidden: Change status to "Public"
change_visibility_from_open: Change status to "Private"
confirm_change_visibility: Are you sure you want to change visibility of this platform?
metadata: Metadata for Software Center
flash:
platform:

1
config/locales/models/platform.ru.yml
View File

@ -52,6 +52,7 @@ ru:
change_visibility_from_hidden: Сменить статус на "Публичный"
change_visibility_from_open: Сменить статус на "Приватный"
confirm_change_visibility: Вы уверены, что хотите сменить статус этой платформы?
metadata: Метаданные для Software Center
flash:
platform:

30
config/locales/models/regeneration_status.en.yml Normal file
View File

@ -0,0 +1,30 @@
en:
layout:
regeneration_statuses:
regenerate_metadata: Regenerate metadata
statuses:
no_data: No data
ready: No actions
waiting_for_regeneration: Waiting for regeneration metadata
regenerating: Regeneration of metadata
waiting_for_resign: Waiting for resign of packages
publish: Publish
resign: Resign of packages
waiting_for_resign_after_publish: Publish, waiting for resign of packages
waiting_for_resign_after_regeneration: Regeneration of metadata, waiting for resign of packages
waiting_for_regeneration_after_publish: Publish, waiting for regeneration metadata
waiting_for_regeneration_after_resign: Resign of packages, waiting for regeneration metadata
waiting_for_resign_and_regeneration_after_publish: Publish, waiting for resign of packages and regeneration metadata
waiting_for_resign_and_regeneration: Waiting for resign of packages and regeneration metadata
last_regenerated_statuses:
no_data: No data
completed: Completed succesfully
failed: Completed not succesfully
canceled: Canceled
activerecord:
attributes:
regeneration_status:
last_regenerated_at: Last regeneration of metadata
last_regenerated_status: Last status of regeneration metadata
status: Current status

30
config/locales/models/regeneration_status.ru.yml Normal file
View File

@ -0,0 +1,30 @@
ru:
layout:
regeneration_statuses:
regenerate_metadata: Регенерировать метаданные
statuses:
no_data: Нет данных
ready: Нет действий
waiting_for_regeneration: Ожидание регенерации методанных
regenerating: Идет регенерация методанных
waiting_for_resign: Ожидание подписи пакетов
publish: Идет публикация
resign: Идет подпись пакетов
waiting_for_resign_after_publish: Идет публикация, ожидание подписи пакетов
waiting_for_resign_after_regeneration: Идет регенерация методанных, ожидание подписи пакетов
waiting_for_regeneration_after_publish: Идет публикация, ожидание регенерации методанных
waiting_for_regeneration_after_resign: Идет подпись пакетов, ожидание регенерации методанных
waiting_for_resign_and_regeneration_after_publish: Идет публикация, ожидание подписи пакетов и регенерации методанных
waiting_for_resign_and_regeneration: Ожидание подписи пакетов и регенерации методанных
last_regenerated_statuses:
no_data: Нет данных
completed: Завершена успешно
failed: Завершена с ошибкой
canceled: Отменена
activerecord:
attributes:
regeneration_status:
last_regenerated_at: Последняя регенерация методанных
last_regenerated_status: Статус последней регенерации методанных
status: Текущее состояние

1
config/routes.rb
View File

@ -152,6 +152,7 @@ Rosa::Application.routes.draw do
resources :platforms do
resources :private_users, :except => [:show, :destroy, :update]
member do
put :regenerate_metadata
put :clear
get :clone
get :members

14
db/migrate/20130822154741_create_repository_statuses.rb Normal file
View File

@ -0,0 +1,14 @@
class CreateRepositoryStatuses < ActiveRecord::Migration
def change
create_table :repository_statuses do |t|
t.integer :repository_id, :null => false
t.integer :platform_id, :null => false
t.integer :status, :default => 0
t.datetime :last_regenerated_at
t.integer :last_regenerated_status
t.timestamps
end
add_index :repository_statuses, [:repository_id, :platform_id], :unique => true
end
end

7
db/migrate/20130822160501_add_status_to_platform.rb Normal file
View File

@ -0,0 +1,7 @@
class AddStatusToPlatform < ActiveRecord::Migration
def change
add_column :platforms, :status, :integer, :default => 0
add_column :platforms, :last_regenerated_at, :datetime
add_column :platforms, :last_regenerated_status, :integer
end
end

27
db/schema.rb
View File

@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
ActiveRecord::Schema.define(:version => 20130820195938) do
ActiveRecord::Schema.define(:version => 20130822160501) do
create_table "activity_feeds", :force => true do |t|
t.integer "user_id", :null => false
@ -307,16 +307,19 @@ ActiveRecord::Schema.define(:version => 20130820195938) do
create_table "platforms", :force => true do |t|
t.string "description"
t.string "name", :null => false
t.string "name", :null => false
t.integer "parent_platform_id"
t.datetime "created_at"
t.datetime "updated_at"
t.boolean "released", :default => false, :null => false
t.boolean "released", :default => false, :null => false
t.integer "owner_id"
t.string "owner_type"
t.string "visibility", :default => "open", :null => false
t.string "platform_type", :default => "main", :null => false
t.string "distrib_type", :null => false
t.string "visibility", :default => "open", :null => false
t.string "platform_type", :default => "main", :null => false
t.string "distrib_type", :null => false
t.integer "status", :default => 0
t.datetime "last_regenerated_at"
t.integer "last_regenerated_status"
end
add_index "platforms", ["name"], :name => "index_platforms_on_name", :unique => true, :case_sensitive => false
@ -484,6 +487,18 @@ ActiveRecord::Schema.define(:version => 20130820195938) do
add_index "repositories", ["platform_id"], :name => "index_repositories_on_platform_id"
create_table "repository_statuses", :force => true do |t|
t.integer "repository_id", :null => false
t.integer "platform_id", :null => false
t.integer "status", :default => 0
t.datetime "last_regenerated_at"
t.integer "last_regenerated_status"
t.datetime "created_at", :null => false
t.datetime "updated_at", :null => false
end
add_index "repository_statuses", ["repository_id", "platform_id"], :name => "index_repository_statuses_on_repository_id_and_platform_id", :unique => true
create_table "settings_notifiers", :force => true do |t|
t.integer "user_id", :null => false
t.boolean "can_notify", :default => true

199
lib/abf_worker/build_lists_publish_task_manager.rb
View File

@ -3,15 +3,10 @@ module AbfWorker
class BuildListsPublishTaskManager
REDIS_MAIN_KEY = 'abf-worker::build-lists-publish-task-manager::'
# LOCKED_REP_AND_PLATFORMS: ['save_to_repository_id-build_for_platform_id', ...]
%w(RESIGN_REPOSITORIES
PROJECTS_FOR_CLEANUP
%w(PROJECTS_FOR_CLEANUP
LOCKED_PROJECTS_FOR_CLEANUP
LOCKED_REPOSITORIES
LOCKED_REP_AND_PLATFORMS
LOCKED_BUILD_LISTS
PACKAGES_FOR_CLEANUP
REGENERATE_METADATA).each do |kind|
PACKAGES_FOR_CLEANUP).each do |kind|
const_set kind, "#{REDIS_MAIN_KEY}#{kind.downcase.gsub('_', '-')}"
end
@ -21,6 +16,7 @@ module AbfWorker
end
def run
create_tasks_for_regenerate_metadata_for_software_center
create_tasks_for_resign_repositories
create_tasks_for_repository_regenerate_metadata
create_tasks_for_build_rpms
@ -53,28 +49,10 @@ module AbfWorker
end
end
def resign_repository(key_pair)
redis.lpush RESIGN_REPOSITORIES, key_pair.repository_id
end
def repository_regenerate_metadata(repository, build_for_platform)
key = "#{repository.id}-#{build_for_platform.id}"
return false if Resque.redis.lrange(REGENERATE_METADATA, 0, -1).include? key
redis.lpush REGENERATE_METADATA, key
end
def unlock_repository(repository_id)
redis.lrem LOCKED_REPOSITORIES, 0, repository_id
end
def unlock_build_list(build_list)
redis.lrem LOCKED_BUILD_LISTS, 0, build_list.id
end
def unlock_rep_and_platform(lock_str)
redis.lrem LOCKED_REP_AND_PLATFORMS, 0, lock_str
end
def packages_structure
structure = {:sources => [], :binaries => {}}
Arch.pluck(:name).each{ |name| structure[:binaries][name.to_sym] = [] }
@ -111,6 +89,8 @@ module AbfWorker
'args' => [{
:id => build_list.id,
:cmd_params => cmd_params,
:main_script => 'build.sh',
:rollback_script => 'rollback.sh',
:platform => {
:platform_path => platform_path,
:type => distrib_type,
@ -118,7 +98,6 @@ module AbfWorker
:arch => build_list.arch.name
},
:repository => {:id => build_list.save_to_repository_id},
:type => :publish,
:time_living => 9600, # 160 min
:packages => packages,
:old_packages => packages_structure,
@ -167,20 +146,15 @@ module AbfWorker
private
def locked_repositories
@redis.lrange LOCKED_REPOSITORIES, 0, -1
end
def create_tasks_for_resign_repositories
resign_repos = @redis.lrange RESIGN_REPOSITORIES, 0, -1
Repository.where(:id => (resign_repos - locked_repositories)).each do |r|
RepositoryStatus.platform_ready
.for_resign
.includes(:repository => :platform)
.readonly(false)
.each do |repository_status|
r = repository_status.repository
# Checks mirror sync status
next if r.repo_lock_file_exists?
@redis.lrem RESIGN_REPOSITORIES, 0, r.id
@redis.lpush LOCKED_REPOSITORIES, r.id
distrib_type = r.platform.distrib_type
cmd_params = {
@ -195,6 +169,7 @@ module AbfWorker
'args' => [{
:id => r.id,
:cmd_params => cmd_params,
:main_script => 'resign.sh',
:platform => {
:platform_path => "#{r.platform.path}/repository",
:type => distrib_type,
@ -202,11 +177,11 @@ module AbfWorker
:arch => 'x86_64'
},
:repository => {:id => r.id},
:type => :resign,
:skip_feedback => true,
:time_living => 9600 # 160 min
:time_living => 9600, # 160 min
:extra => {:repository_status_id => repository_status.id, :resign => true}
}]
)
) if repository_status.start_resign
end
end
@ -218,27 +193,19 @@ module AbfWorker
order(:min_updated_at).
limit(@workers_count * 2) # because some repos may be locked
locked_rep = locked_repositories
locked_rep = RepositoryStatus.not_ready.joins(:platform).
where(:platforms => {:platform_type => 'main'}).pluck(:repository_id)
available_repos = available_repos.where('save_to_repository_id NOT IN (?)', locked_rep) unless locked_rep.empty?
counter = 1
# looks like:
# ['save_to_repository_id-build_for_platform_id', ...]
locked_rep_and_pl = @redis.lrange(LOCKED_REP_AND_PLATFORMS, 0, -1)
for_cleanup = @redis.lrange(PROJECTS_FOR_CLEANUP, 0, -1).map do |key|
pr, rep, pl = *key.split('-')
if locked_rep.present? && locked_rep.include?(rep)
nil
else
[rep.to_i, pl.to_i]
end
locked_rep.present? && locked_rep.include?(rep.to_i) ? nil : [rep.to_i, pl.to_i]
end.compact
available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup
counter = 1
available_repos = available_repos.map{ |bl| [bl.save_to_repository_id, bl.build_for_platform_id] } | for_cleanup
available_repos.each do |save_to_repository_id, build_for_platform_id|
next if locked_rep_and_pl.include?("#{save_to_repository_id}-#{build_for_platform_id}")
next if RepositoryStatus.not_ready.where(:repository_id => save_to_repository_id, :platform_id => build_for_platform_id).exists?
break if counter > @workers_count
counter += 1 if create_rpm_build_task(save_to_repository_id, build_for_platform_id)
end
@ -283,7 +250,10 @@ module AbfWorker
save_to_repository = Repository.find save_to_repository_id
# Checks mirror sync status
return false if save_to_repository.repo_lock_file_exists?
return false if save_to_repository.repo_lock_file_exists? || !save_to_repository.platform.ready?
repository_status = save_to_repository.repository_statuses.find_or_create_by_platform_id(build_for_platform_id)
return false unless repository_status.publish
save_to_platform = save_to_repository.platform
build_for_platform = Platform.find build_for_platform_id
@ -304,10 +274,11 @@ module AbfWorker
'BUILD_FOR_PLATFORM' => build_for_platform.name
}.map{ |k, v| "#{k}=#{v}" }.join(' ')
lock_str = "#{save_to_repository_id}-#{build_for_platform_id}"
options = {
:id => (bl ? bl.id : Time.now.to_i),
:cmd_params => cmd_params,
:id => (bl ? bl.id : Time.now.to_i),
:cmd_params => cmd_params,
:main_script => 'build.sh',
:rollback_script => 'rollback.sh',
:platform => {
:platform_path => platform_path,
:type => distrib_type,
@ -315,9 +286,8 @@ module AbfWorker
:arch => (bl ? bl.arch.name : 'x86_64')
},
:repository => {:id => save_to_repository_id},
:type => :publish,
:time_living => 9600, # 160 min
:extra => {:lock_str => lock_str}
:extra => {:repository_status_id => repository_status.id}
}
packages, build_list_ids, new_sources = self.class.packages_structure, [], {}
@ -337,9 +307,9 @@ module AbfWorker
worker_queue,
'class' => worker_class,
'args' => [options.merge({
:packages => packages,
:old_packages => old_packages,
:build_list_ids => build_list_ids,
:packages => packages,
:old_packages => old_packages,
:build_list_ids => build_list_ids,
:projects_for_cleanup => projects_for_cleanup
})]
)
@ -348,61 +318,88 @@ module AbfWorker
@redis.lpush LOCKED_PROJECTS_FOR_CLEANUP, key
end
@redis.lpush(LOCKED_REP_AND_PLATFORMS, lock_str)
return true
end
def create_tasks_for_repository_regenerate_metadata
worker_queue = 'publish_worker_default'
worker_class = 'AbfWorker::PublishWorkerDefault'
regen_repos_and_pl = @redis.lrange REGENERATE_METADATA, 0, -1
locked_rep_and_pl = @redis.lrange(LOCKED_REP_AND_PLATFORMS, 0, -1)
def create_tasks_for_regenerate_metadata_for_software_center
Platform.main.waiting_for_regeneration.each do |platform|
repos = platform.repositories
statuses = RepositoryStatus.where(:platform_id => platform.id)
next if repos.find{ |r| r.repo_lock_file_exists? }
next if statuses.present? &&
statuses.map{ |s| s.ready? || s.can_start_regeneration? || s.can_start_resign? }.uniq != [true]
regen_repos = regen_repos_and_pl.map{ |r| r.gsub(/\-[\d]*$/, '') }
Repository.where(:id => regen_repos).each do |rep|
cmd_params = {
'RELEASED' => platform.released,
'REPOSITORY_NAME' => platform.repositories.map(&:name).join(','),
'TYPE' => platform.distrib_type,
'REGENERATE_PLATFORM_METADATA' => true,
'SAVE_TO_PLATFORM' => platform.name,
'BUILD_FOR_PLATFORM' => platform.name
}.map{ |k, v| "#{k}=#{v}" }.join(' ')
Resque.push(
'publish_worker_default',
'class' => 'AbfWorker::PublishWorkerDefault',
'args' => [{
:id => Time.now.to_i,
:cmd_params => cmd_params,
:main_script => 'regenerate_platform_metadata.sh',
:platform => {
:platform_path => "#{platform.path}/repository",
:type => platform.distrib_type,
:name => platform.name,
:arch => 'x86_64'
},
:time_living => 9600, # 160 min
:skip_feedback => true,
:extra => {:platform_id => platform.id, :regenerate_platform => true}
}]
) if platform.start_regeneration
end
end
def create_tasks_for_repository_regenerate_metadata
RepositoryStatus.platform_ready
.for_regeneration
.includes(:repository => :platform)
.readonly(false)
.each do |repository_status|
rep = repository_status.repository
# Checks mirror sync status
next if rep.repo_lock_file_exists?
regen_repos_and_pl.select{ |kind| kind =~ /^#{rep.id}\-/ }.each do |lock_str|
next if locked_rep_and_pl.include?(lock_str)
@redis.lrem REGENERATE_METADATA, 0, lock_str
build_for_platform = Platform.find lock_str.gsub(/^[\d]*\-/, '')
cmd_params = {
'RELEASED' => rep.platform.released,
'REPOSITORY_NAME' => rep.name,
'TYPE' => build_for_platform.distrib_type,
'REGENERATE_METADATA' => true,
'SAVE_TO_PLATFORM' => rep.platform.name,
'BUILD_FOR_PLATFORM' => build_for_platform.name
}.map{ |k, v| "#{k}=#{v}" }.join(' ')
build_for_platform = repository_status.platform
cmd_params = {
'RELEASED' => rep.platform.released,
'REPOSITORY_NAME' => rep.name,
'TYPE' => build_for_platform.distrib_type,
'REGENERATE_METADATA' => true,
'SAVE_TO_PLATFORM' => rep.platform.name,
'BUILD_FOR_PLATFORM' => build_for_platform.name
}.map{ |k, v| "#{k}=#{v}" }.join(' ')
options = {
:id => Time.now.to_i,
:cmd_params => cmd_params,
Resque.push(
'publish_worker_default',
'class' => 'AbfWorker::PublishWorkerDefault',
'args' => [{
:id => Time.now.to_i,
:cmd_params => cmd_params,
:main_script => 'build.sh',
:rollback_script => 'rollback.sh',
:platform => {
:platform_path => "#{rep.platform.path}/repository",
:type => build_for_platform.distrib_type,
:name => build_for_platform.name,
:arch => 'x86_64'
},
:repository => {:id => rep.id},
:type => :publish,
:time_living => 9600, # 160 min
:skip_feedback => true,
:extra => {:lock_str => lock_str, :regenerate => true}
}
Resque.push(
worker_queue,
'class' => worker_class,
'args' => [options.merge({
})]
)
@redis.lpush(LOCKED_REP_AND_PLATFORMS, lock_str)
end
:extra => {:repository_status_id => repository_status.id, :regenerate => true}
}]
) if repository_status.start_regeneration
end
return true
end
end
end

25
lib/abf_worker/publish_observer.rb
View File

@ -9,12 +9,19 @@ module AbfWorker
def perform
return if status == STARTED # do nothing when publication started
if options['type'] == 'resign'
AbfWorker::BuildListsPublishTaskManager.unlock_repository options['id']
else
if options['extra']['regenerate'] # Regenerate metadata
AbfWorker::BuildListsPublishTaskManager.unlock_rep_and_platform options['extra']['lock_str']
elsif options['extra']['create_container'] # Container has been created
extra = options['extra']
repository_status = RepositoryStatus.where(:id => extra['repository_status_id'])
begin
if extra['regenerate'] # Regenerate metadata
repository_status.last_regenerated_at = Time.now.utc
repository_status.last_regenerated_status = status
elsif extra['regenerate_platform'] # Regenerate metadata for Software Center
if platform = Platform.where(:id => extra['platform_id'])).first
platform.last_regenerated_at = Time.now.utc
platform.last_regenerated_status = status
platform.ready
end
elsif extra['create_container'] # Container has been created
case status
when COMPLETED
subject.published_container
@ -22,9 +29,11 @@ module AbfWorker
subject.fail_publish_container
end
update_results
else
elsif !extra['resign'] # Simple publish
update_rpm_builds
end
ensure
repository_status.ready if repository_status.present?
end
end
@ -50,8 +59,6 @@ module AbfWorker
when FAILED, CANCELED
AbfWorker::BuildListsPublishTaskManager.cleanup_failed options['projects_for_cleanup']
end
AbfWorker::BuildListsPublishTaskManager.unlock_rep_and_platform options['extra']['lock_str']
end
def update_results(build_list = subject)

159
spec/lib/abf-worker/build_lists_publish_task_manager_spec.rb
View File

@ -7,7 +7,7 @@ describe AbfWorker::BuildListsPublishTaskManager do
end
before do
init_test_root
stub_redis
stub_symlink_methods
FactoryGirl.create(:build_list)
end
@ -16,25 +16,19 @@ describe AbfWorker::BuildListsPublishTaskManager do
let(:build_list) { FactoryGirl.create(:build_list) }
context 'when no items for publishing' do
before do
stub_redis
subject.new.run
end
before { subject.new.run }
%w(RESIGN_REPOSITORIES
PROJECTS_FOR_CLEANUP
%w(PROJECTS_FOR_CLEANUP
LOCKED_PROJECTS_FOR_CLEANUP
LOCKED_REPOSITORIES
LOCKED_REP_AND_PLATFORMS
LOCKED_BUILD_LISTS).each do |kind|
it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
%w(publish_worker_default publish_worker).each do |kind|
it "ensure that no tasks in '#{kind}' queue" do
it "ensures that no tasks in '#{kind}' queue" do
@redis_instance.lrange(kind, 0, -1).should be_empty
end
end
@ -43,33 +37,28 @@ describe AbfWorker::BuildListsPublishTaskManager do
context 'when one build_list for publishing' do
before do
stub_redis
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
2.times{ subject.new.run }
end
%w(RESIGN_REPOSITORIES
PROJECTS_FOR_CLEANUP
LOCKED_PROJECTS_FOR_CLEANUP
LOCKED_REPOSITORIES).each do |kind|
%w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind|
it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
it "ensure that 'locked rep and platforms' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_REP_AND_PLATFORMS, 0, -1)
queue.should have(1).item
queue.should include("#{build_list.save_to_repository_id}-#{build_list.build_for_platform_id}")
it "ensures that repository_status has status publish" do
build_list.save_to_repository.repository_statuses.
find_by_platform_id(build_list.build_for_platform_id).publish?.
should be_true
end
it "ensure that 'locked build lists' has only one item" do
it "ensures that 'locked build lists' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(1).item
queue.should include(build_list.id.to_s)
end
it "ensure that new task for publishing has been created" do
it "ensures that new task for publishing has been created" do
@redis_instance.lrange('queue:publish_worker_default', 0, -1).should have(1).item
end
@ -83,35 +72,28 @@ describe AbfWorker::BuildListsPublishTaskManager do
:build_for_platform => build_list.build_for_platform
) }
before do
stub_redis
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
build_list2.update_column(:status, BuildList::BUILD_PUBLISH)
2.times{ subject.new.run }
end
%w(RESIGN_REPOSITORIES
PROJECTS_FOR_CLEANUP
LOCKED_PROJECTS_FOR_CLEANUP
LOCKED_REPOSITORIES).each do |kind|
it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
%w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP).each do |kind|
it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
it "ensure that 'locked rep and platforms' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_REP_AND_PLATFORMS, 0, -1)
queue.should have(1).item
queue.should include("#{build_list.save_to_repository_id}-#{build_list.build_for_platform_id}")
it "ensures that only one repository_status has status publish" do
RepositoryStatus.where(:status => RepositoryStatus::PUBLISH).should have(1).item
end
it "ensure that 'locked build lists' has 2 items" do
it "ensures that 'locked build lists' has 2 items" do
queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(2).item
queue.should include(build_list.id.to_s, build_list2.id.to_s)
end
it "ensure that new task for publishing has been created" do
it "ensures that new task for publishing has been created" do
@redis_instance.lrange('queue:publish_worker_default', 0, -1).should have(1).item
end
@ -119,7 +101,6 @@ describe AbfWorker::BuildListsPublishTaskManager do
context 'creates not more than 4 tasks for publishing' do
before do
stub_redis
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
4.times {
bl = FactoryGirl.create(:build_list, :new_core => true)
@ -128,15 +109,15 @@ describe AbfWorker::BuildListsPublishTaskManager do
2.times{ subject.new.run }
end
it "ensure that 'locked rep and platforms' has 4 items" do
@redis_instance.lrange(subject::LOCKED_REP_AND_PLATFORMS, 0, -1).should have(4).items
it "ensures that 4 repository_statuses have status publish" do
RepositoryStatus.where(:status => RepositoryStatus::PUBLISH).should have(4).items
end
it "ensure that 'locked build lists' has 4 items" do
it "ensures that 'locked build lists' has 4 items" do
@redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1).should have(4).items
end
it "ensure that new tasks for publishing has been created" do
it "ensures that new tasks for publishing has been created" do
@redis_instance.lrange('queue:publish_worker_default', 0, -1).should have(4).items
end
@ -144,36 +125,29 @@ describe AbfWorker::BuildListsPublishTaskManager do
context 'creates task for removing project from repository' do
before do
stub_redis
build_list.update_column(:status, BuildList::BUILD_PUBLISHED)
FactoryGirl.create(:build_list_package, :build_list => build_list)
ProjectToRepository.where(:project_id => build_list.project_id, :repository_id => build_list.save_to_repository_id).destroy_all
2.times{ subject.new.run }
end
%w(RESIGN_REPOSITORIES
PROJECTS_FOR_CLEANUP
LOCKED_REPOSITORIES
LOCKED_BUILD_LISTS).each do |kind|
it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
%w(PROJECTS_FOR_CLEANUP LOCKED_BUILD_LISTS).each do |kind|
it "ensures that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
it "ensure that 'locked rep and platforms' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_REP_AND_PLATFORMS, 0, -1)
queue.should have(1).item
queue.should include("#{build_list.save_to_repository_id}-#{build_list.build_for_platform_id}")
it "ensures that only one repository_status has status publish" do
RepositoryStatus.where(:status => RepositoryStatus::PUBLISH).should have(1).item
end
it "ensure that 'locked projects for cleanup' has only one item" do
it "ensures that 'locked projects for cleanup' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1)
queue.should have(1).item
queue.should include("#{build_list.project_id}-#{build_list.save_to_repository_id}-#{build_list.build_for_platform_id}")
end
it "ensure that new task for publishing has been created" do
it "ensures that new task for publishing has been created" do
@redis_instance.lrange('queue:publish_worker_default', 0, -1).should have(1).item
end
@ -193,7 +167,6 @@ describe AbfWorker::BuildListsPublishTaskManager do
:build_for_platform => build_list.build_for_platform
) }
before do
stub_redis
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
build_list2.update_column(:status, BuildList::BUILD_PUBLISHED)
build_list3.update_column(:status, BuildList::BUILD_PUBLISHED)
@ -201,32 +174,25 @@ describe AbfWorker::BuildListsPublishTaskManager do
2.times{ subject.new.run }
end
%w(RESIGN_REPOSITORIES
PROJECTS_FOR_CLEANUP
LOCKED_REPOSITORIES).each do |kind|
it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
it "ensures that no 'projects for cleanup'" do
@redis_instance.lrange(subject::PROJECTS_FOR_CLEANUP, 0, -1).should be_empty
end
it "ensure that 'locked rep and platforms' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_REP_AND_PLATFORMS, 0, -1)
queue.should have(1).item
queue.should include("#{build_list.save_to_repository_id}-#{build_list.build_for_platform_id}")
it "ensures that only one repository_status has status publish" do
RepositoryStatus.where(:status => RepositoryStatus::PUBLISH).should have(1).item
end
it "ensure that 'locked projects for cleanup' has only one item" do
it "ensures that 'locked projects for cleanup' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_PROJECTS_FOR_CLEANUP, 0, -1)
queue.should have(1).item
queue.should include("#{build_list3.project_id}-#{build_list3.save_to_repository_id}-#{build_list3.build_for_platform_id}")
end
it "ensure that new task for publishing has been created" do
it "ensures that new task for publishing has been created" do
@redis_instance.lrange('queue:publish_worker_default', 0, -1).should have(1).item
end
it "ensure that 'locked build lists' has only one item" do
it "ensures that 'locked build lists' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_BUILD_LISTS, 0, -1)
queue.should have(1).item
queue.should include(build_list.id.to_s)
@ -235,27 +201,19 @@ describe AbfWorker::BuildListsPublishTaskManager do
context 'resign packages in repository' do
before do
stub_redis
build_list.update_column(:status, BuildList::BUILD_PUBLISH)
FactoryGirl.create(:key_pair, :repository => build_list.save_to_repository)
2.times{ subject.new.run }
end
%w(RESIGN_REPOSITORIES
PROJECTS_FOR_CLEANUP
LOCKED_PROJECTS_FOR_CLEANUP
LOCKED_REP_AND_PLATFORMS
LOCKED_BUILD_LISTS).each do |kind|
%w(PROJECTS_FOR_CLEANUP LOCKED_PROJECTS_FOR_CLEANUP LOCKED_BUILD_LISTS).each do |kind|
it "ensure that no '#{kind.downcase.gsub('_', ' ')}'" do
@redis_instance.lrange(subject.const_get(kind), 0, -1).should be_empty
end
end
it "ensure that 'locked repositories' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_REPOSITORIES, 0, -1)
queue.should have(1).item
queue.should include(build_list.save_to_repository_id.to_s)
it "ensures that only one repository_status has status resign" do
RepositoryStatus.where(:status => RepositoryStatus::RESIGN).should have(1).item
end
it "ensure that new task for resign has been created" do
@ -265,29 +223,18 @@ describe AbfWorker::BuildListsPublishTaskManager do
end
context 'regenerate metadata' do
before do
stub_redis
end
context 'for repository of main platform' do
let(:repository) { FactoryGirl.create(:repository) }
before do
subject.repository_regenerate_metadata repository, repository.platform
repository.regenerate
subject.new.run
end
it "ensure that 'locked rep and platforms' has only one item" do
queue = @redis_instance.lrange(subject::LOCKED_REP_AND_PLATFORMS, 0, -1)
queue.should have(1).item
queue.should include("#{repository.id}-#{repository.platform.id}")
it "ensures that only one repository_status has status regenerating" do
RepositoryStatus.where(:status => RepositoryStatus::REGENERATING).should have(1).item
end
it "ensure that 'regenerate metadata' queue without items" do
queue = @redis_instance.lrange(subject::REGENERATE_METADATA, 0, -1)
queue.should be_empty
end
it 'ensure that new task has been created' do
it 'ensures that new task has been created' do
@redis_instance.lrange('queue:publish_worker_default', 0, -1).should have(1).item
end
end
@ -296,33 +243,19 @@ describe AbfWorker::BuildListsPublishTaskManager do
let(:main_platform) { FactoryGirl.create(:platform) }
let(:repository) { FactoryGirl.create(:personal_repository) }
before do
subject.repository_regenerate_metadata repository, main_platform
repository.regenerate main_platform.id
subject.new.run
end
it "ensure that 'locked rep and platforms' has only one item" do
@redis_instance.lrange(subject::LOCKED_REP_AND_PLATFORMS, 0, -1)
.should == ["#{repository.id}-#{main_platform.id}"]
it "ensures that only one repository_status has status regenerating" do
RepositoryStatus.where(:status => RepositoryStatus::REGENERATING).should have(1).item
end
it "ensure that 'regenerate metadata' queue without items" do
@redis_instance.lrange(subject::REGENERATE_METADATA, 0, -1).should be_empty
end
it 'ensure that new task has been created' do
it 'ensures that new task has been created' do
@redis_instance.lrange('queue:publish_worker_default', 0, -1).should have(1).item
end
end
it 'ensure that two tasks for regenerate one repository will not be created' do
repository = FactoryGirl.create(:repository)
2.times do
subject.repository_regenerate_metadata repository, repository.platform
subject.new.run
end
@redis_instance.lrange('queue:publish_worker_default', 0, -1).should have(1).item
end
end
after(:all) do