Software
/
rosa-build
mirror of https://github.com/OpenMandrivaSoftware/rosa-build.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

#281: Show repodata creation logs

This commit is contained in:
Vokhmin Alexey V 2013-08-27 19:43:30 +04:00
parent 5689e36890
commit a51b1e0e9f
22 changed files with 133 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/controllers/platforms/platforms_controller.rb
View File

@ -1,5 +1,6 @@
# -*- encoding : utf-8 -*-
class Platforms::PlatformsController < Platforms::BaseController
include FileStoreHelper
before_filter :authenticate_user!
skip_before_filter :authenticate_user!, :only => [:advisories, :members, :show] if APP_CONFIG['anonymous_access']

2
app/controllers/platforms/product_build_lists_controller.rb
View File

@ -1,6 +1,6 @@
# -*- encoding : utf-8 -*-
class Platforms::ProductBuildListsController < Platforms::BaseController
include BuildsHelper
include FileStoreHelper
before_filter :authenticate_user!
skip_before_filter :authenticate_user!, :only => [:index, :show, :log] if APP_CONFIG['anonymous_access']

2
app/controllers/platforms/repositories_controller.rb
View File

@ -1,5 +1,7 @@
# -*- encoding : utf-8 -*-
class Platforms::RepositoriesController < Platforms::BaseController
include FileStoreHelper
before_filter :authenticate_user!
skip_before_filter :authenticate_user!, :only => [:index, :show, :projects_list] if APP_CONFIG['anonymous_access']

2
app/controllers/projects/build_lists_controller.rb
View File

@ -1,6 +1,6 @@
# -*- encoding : utf-8 -*-
class Projects::BuildListsController < Projects::BaseController
include BuildsHelper
include FileStoreHelper
NESTED_ACTIONS = [:index, :new, :create]

10
app/helpers/builds_helper.rb → app/helpers/file_store_helper.rb
View File

@ -1,5 +1,5 @@
# -*- encoding : utf-8 -*-
module BuildsHelper
module FileStoreHelper
def file_store_results_url(sha1, file_name)
url = "#{APP_CONFIG['file_store_url']}/api/v1/file_stores/#{sha1}"
@ -7,4 +7,12 @@ module BuildsHelper
url
end
def link_to_file_store(file_name, sha1)
if sha1.present?
link_to file_name, file_store_results_url(sha1, file_name)
else
I18n.t('layout.no_')
end
end
end

2
app/models/build_list.rb
View File

@ -1,7 +1,7 @@
# -*- encoding : utf-8 -*-
class BuildList < ActiveRecord::Base
include Modules::Models::CommitAndVersion
include Modules::Models::FileStoreClean
include FileStoreClean
include AbfWorker::ModelHelper
include Modules::Observers::ActivityFeed::BuildList

52
app/models/concerns/file_store_clean.rb Normal file
View File

@ -0,0 +1,52 @@
# -*- encoding : utf-8 -*-
module FileStoreClean
extend ActiveSupport::Concern
included do
def destroy
destroy_files_from_file_store if Rails.env.production?
super
end
later :destroy, :queue => :clone_build
def sha1_of_file_store_files
raise NotImplementedError, "You should implement this method"
end
def destroy_files_from_file_store(args = sha1_of_file_store_files)
files = *args
token = User.find_by_uname('file_store').authentication_token
uri = URI APP_CONFIG['file_store_url']
Net::HTTP.start(uri.host, uri.port) do |http|
files.each do |sha1|
begin
req = Net::HTTP::Delete.new("/api/v1/file_stores/#{sha1}.json")
req.basic_auth token, ''
http.request(req)
rescue # Dont care about it
end
end
end
end
def later_destroy_files_from_file_store(args)
destroy_files_from_file_store(args)
end
later :later_destroy_files_from_file_store, :queue => :clone_build
end
def self.file_exist_on_file_store?(sha1)
begin
resp = JSON(RestClient.get "#{APP_CONFIG['file_store_url']}/api/v1/file_stores.json", :params => {:hash => sha1})
rescue # Dont care about it
resp = []
end
if resp[0].respond_to?('[]') && resp[0]['file_name'] && resp[0]['sha1_hash']
true
else
false
end
end
end

34
app/models/concerns/regeneration_status.rb
View File

@ -1,12 +1,5 @@
module RegenerationStatus
def human_regeneration_status
self.class::HUMAN_REGENERATION_STATUSES[last_regenerated_status] || :no_data
end
def human_status
self.class::HUMAN_STATUSES[status] || :no_data
end
extend ActiveSupport::Concern
READY = 0
WAITING_FOR_REGENERATION = 100
@ -24,4 +17,29 @@ module RegenerationStatus
AbfWorker::BaseObserver::CANCELED => :canceled
}.freeze
included do
after_update :cleanup_file_store
def sha1_of_file_store_files
files = []
files << last_regenerated_log_sha1 if last_regenerated_log_sha1.present?
files
end
def human_regeneration_status
self.class::HUMAN_REGENERATION_STATUSES[last_regenerated_status] || :no_data
end
def human_status
self.class::HUMAN_STATUSES[status] || :no_data
end
def cleanup_file_store
old_log_sha1 = last_regenerated_log_sha1_was
if old_log_sha1.present? && old_log_sha1 != last_regenerated_log_sha1
later_destroy_files_from_file_store([old_log_sha1])
end
end
end
end

1
app/models/platform.rb
View File

@ -1,5 +1,6 @@
# -*- encoding : utf-8 -*-
class Platform < ActiveRecord::Base
include FileStoreClean
include RegenerationStatus
VISIBILITIES = %w(open hidden)

2
app/models/product_build_list.rb
View File

@ -2,7 +2,7 @@
class ProductBuildList < ActiveRecord::Base
include Modules::Models::CommitAndVersion
include Modules::Models::TimeLiving
include Modules::Models::FileStoreClean
include FileStoreClean
include AbfWorker::ModelHelper
delegate :url_helpers, to: 'Rails.application.routes'

4
app/models/project.rb
View File

@ -215,11 +215,11 @@ class Project < ActiveRecord::Base
format_id = ProjectTag::FORMATS["#{tag_file_format(format)}"]
project_tag = project_tags.where(:tag_name => tag.name, :format_id => format_id).first
return project_tag.sha1 if project_tag && project_tag.commit_id == tag.commit.id && Modules::Models::FileStoreClean.file_exist_on_file_store?(project_tag.sha1)
return project_tag.sha1 if project_tag && project_tag.commit_id == tag.commit.id && FileStoreClean.file_exist_on_file_store?(project_tag.sha1)
archive = archive_by_treeish_and_format tag.name, format
sha1 = Digest::SHA1.file(archive[:path]).hexdigest
unless Modules::Models::FileStoreClean.file_exist_on_file_store? sha1
unless FileStoreClean.file_exist_on_file_store? sha1
token = User.find_by_uname('rosa_system').authentication_token
begin
resp = JSON `curl --user #{token}: -POST -F 'file_store[file]=@#{archive[:path]};filename=#{name}-#{tag.name}.#{tag_file_format(format)}' #{APP_CONFIG['file_store_url']}/api/v1/upload`

2
app/models/project_tag.rb
View File

@ -1,6 +1,6 @@
# -*- encoding : utf-8 -*-
class ProjectTag < ActiveRecord::Base
include Modules::Models::FileStoreClean
include FileStoreClean
FORMATS = {
'zip' => 0,

1
app/models/repository_status.rb
View File

@ -1,4 +1,5 @@
class RepositoryStatus < ActiveRecord::Base
include FileStoreClean
include RegenerationStatus
WAITING_FOR_RESIGN = 300

3
app/views/platforms/platforms/edit.html.haml
View File

@ -27,6 +27,9 @@
.leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_status')
.rightlist= t("layout.regeneration_statuses.last_regenerated_statuses.#{@platform.human_regeneration_status}")
.both
.leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_log_sha1')
.rightlist= link_to_file_store('regeneration.log', @platform.last_regenerated_log_sha1)
.both
.leftside
.rightside= link_to t('layout.regeneration_statuses.regenerate_metadata'), regenerate_metadata_platform_path(@platform), :method => :put, :confirm => t('layout.confirm'), :class => 'button'
.both

2
app/views/platforms/repositories/edit.html.haml
View File

@ -25,6 +25,7 @@
%tr
%th= t('activerecord.attributes.regeneration_status.status')
%th= t('activerecord.attributes.regeneration_status.last_regenerated_status')
%th= t('activerecord.attributes.regeneration_status.last_regenerated_log_sha1')
%th= t('activerecord.attributes.regeneration_status.last_regenerated_at')
- unless @platform.main?
%th= t('activerecord.models.platform')
@ -33,6 +34,7 @@
%tr{:class => cycle('odd', 'even')}
%td= t("layout.regeneration_statuses.statuses.#{status.human_status}")
%td= t("layout.regeneration_statuses.last_regenerated_statuses.#{status.human_regeneration_status}")
%td= link_to_file_store('regeneration.log', @status.last_regenerated_log_sha1)
%td= status.last_regenerated_at
- unless @platform.main?
%td= status.platform.name

1
config/locales/models/regeneration_status.en.yml
View File

@ -27,4 +27,5 @@ en:
regeneration_status:
last_regenerated_at: Last regeneration of metadata
last_regenerated_status: Last status of regeneration metadata
last_regenerated_log_sha1: Last log of regeneration metadata
status: Current status

1
config/locales/models/regeneration_status.ru.yml
View File

@ -27,4 +27,5 @@ ru:
regeneration_status:
last_regenerated_at: Последняя регенерация методанных
last_regenerated_status: Статус последней регенерации методанных
last_regenerated_log_sha1: Последний лог регенерации методанных
status: Текущее состояние

6
db/migrate/20130827144022_add_last_regenerated_log_sha1_to_platform_and_repository_status.rb Normal file
View File

@ -0,0 +1,6 @@
class AddLastRegeneratedLogSha1ToPlatformAndRepositoryStatus < ActiveRecord::Migration
def change
add_column :platforms, :last_regenerated_log_sha1, :string
add_column :repository_statuses, :last_regenerated_log_sha1, :string
end
end

26
db/schema.rb
View File

@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
ActiveRecord::Schema.define(:version => 20130822160501) do
ActiveRecord::Schema.define(:version => 20130827144022) do
create_table "activity_feeds", :force => true do |t|
t.integer "user_id", :null => false
@ -307,19 +307,20 @@ ActiveRecord::Schema.define(:version => 20130822160501) do
create_table "platforms", :force => true do |t|
t.string "description"
t.string "name", :null => false
t.string "name", :null => false
t.integer "parent_platform_id"
t.datetime "created_at"
t.datetime "updated_at"
t.boolean "released", :default => false, :null => false
t.boolean "released", :default => false, :null => false
t.integer "owner_id"
t.string "owner_type"
t.string "visibility", :default => "open", :null => false
t.string "platform_type", :default => "main", :null => false
t.string "distrib_type", :null => false
t.integer "status", :default => 0
t.string "visibility", :default => "open", :null => false
t.string "platform_type", :default => "main", :null => false
t.string "distrib_type", :null => false
t.integer "status", :default => 0
t.datetime "last_regenerated_at"
t.integer "last_regenerated_status"
t.string "last_regenerated_log_sha1"
end
add_index "platforms", ["name"], :name => "index_platforms_on_name", :unique => true, :case_sensitive => false
@ -488,13 +489,14 @@ ActiveRecord::Schema.define(:version => 20130822160501) do
add_index "repositories", ["platform_id"], :name => "index_repositories_on_platform_id"
create_table "repository_statuses", :force => true do |t|
t.integer "repository_id", :null => false
t.integer "platform_id", :null => false
t.integer "status", :default => 0
t.integer "repository_id", :null => false
t.integer "platform_id", :null => false
t.integer "status", :default => 0
t.datetime "last_regenerated_at"
t.integer "last_regenerated_status"
t.datetime "created_at", :null => false
t.datetime "updated_at", :null => false
t.datetime "created_at", :null => false
t.datetime "updated_at", :null => false
t.string "last_regenerated_log_sha1"
end
add_index "repository_statuses", ["repository_id", "platform_id"], :name => "index_repository_statuses_on_repository_id_and_platform_id", :unique => true

2
lib/abf_worker/build_lists_publish_task_manager.rb
View File

@ -352,7 +352,6 @@ module AbfWorker
:arch => 'x86_64'
},
:time_living => 9600, # 160 min
:skip_feedback => true,
:extra => {:platform_id => platform.id, :regenerate_platform => true}
}]
) if platform.start_regeneration
@ -395,7 +394,6 @@ module AbfWorker
:arch => 'x86_64'
},
:time_living => 9600, # 160 min
:skip_feedback => true,
:extra => {:repository_status_id => repository_status.id, :regenerate => true}
}]
) if repository_status.start_regeneration

7
lib/abf_worker/publish_observer.rb
View File

@ -12,13 +12,20 @@ module AbfWorker
extra = options['extra']
repository_status = RepositoryStatus.where(:id => extra['repository_status_id']).first
begin
if extra['regenerate'] || extra['regenerate_platform']
log_sha1 = (options['results'].try(:first) || {}).fetch('sha1', nil)
end
if extra['regenerate'] # Regenerate metadata
repository_status.last_regenerated_at = Time.now.utc
repository_status.last_regenerated_status = status
repository_status.last_regenerated_log_sha1 = log_sha1
elsif extra['regenerate_platform'] # Regenerate metadata for Software Center
if platform = Platform.where(:id => extra['platform_id']).first
platform.last_regenerated_at = Time.now.utc
platform.last_regenerated_status = status
platform.last_regenerated_log_sha1 = log_sha1
platform.ready
end
elsif extra['create_container'] # Container has been created

50
lib/modules/models/file_store_clean.rb
View File

@ -1,50 +0,0 @@
# -*- encoding : utf-8 -*-
module Modules
module Models
module FileStoreClean
extend ActiveSupport::Concern
included do
def destroy
destroy_files_from_file_store if Rails.env.production?
super
end
later :destroy, :queue => :clone_build
def sha1_of_file_store_files
raise NotImplementedError, "You should implement this method"
end
def destroy_files_from_file_store(args = sha1_of_file_store_files)
files = *args
token = User.find_by_uname('file_store').authentication_token
uri = URI APP_CONFIG['file_store_url']
Net::HTTP.start(uri.host, uri.port) do |http|
files.each do |sha1|
begin
req = Net::HTTP::Delete.new("/api/v1/file_stores/#{sha1}.json")
req.basic_auth token, ''
http.request(req)
rescue # Dont care about it
end
end
end
end
end
def self.file_exist_on_file_store?(sha1)
begin
resp = JSON(RestClient.get "#{APP_CONFIG['file_store_url']}/api/v1/file_stores.json", :params => {:hash => sha1})
rescue # Dont care about it
resp = []
end
if resp[0].respond_to?('[]') && resp[0]['file_name'] && resp[0]['sha1_hash']
true
else
false
end
end
end
end
end