diff --git a/app/controllers/platforms/platforms_controller.rb b/app/controllers/platforms/platforms_controller.rb index 695ec7e9b..1ab678cf8 100644 --- a/app/controllers/platforms/platforms_controller.rb +++ b/app/controllers/platforms/platforms_controller.rb @@ -1,5 +1,6 @@ # -*- encoding : utf-8 -*- class Platforms::PlatformsController < Platforms::BaseController + include FileStoreHelper before_filter :authenticate_user! skip_before_filter :authenticate_user!, :only => [:advisories, :members, :show] if APP_CONFIG['anonymous_access'] diff --git a/app/controllers/platforms/product_build_lists_controller.rb b/app/controllers/platforms/product_build_lists_controller.rb index b35ad3396..4a6f0c5cc 100644 --- a/app/controllers/platforms/product_build_lists_controller.rb +++ b/app/controllers/platforms/product_build_lists_controller.rb @@ -1,6 +1,6 @@ # -*- encoding : utf-8 -*- class Platforms::ProductBuildListsController < Platforms::BaseController - include BuildsHelper + include FileStoreHelper before_filter :authenticate_user! skip_before_filter :authenticate_user!, :only => [:index, :show, :log] if APP_CONFIG['anonymous_access'] diff --git a/app/controllers/platforms/repositories_controller.rb b/app/controllers/platforms/repositories_controller.rb index 82132f8f0..534f55e6e 100644 --- a/app/controllers/platforms/repositories_controller.rb +++ b/app/controllers/platforms/repositories_controller.rb @@ -1,5 +1,7 @@ # -*- encoding : utf-8 -*- class Platforms::RepositoriesController < Platforms::BaseController + include FileStoreHelper + before_filter :authenticate_user! skip_before_filter :authenticate_user!, :only => [:index, :show, :projects_list] if APP_CONFIG['anonymous_access'] diff --git a/app/controllers/projects/build_lists_controller.rb b/app/controllers/projects/build_lists_controller.rb index 2fddaa1a0..dd6394412 100644 --- a/app/controllers/projects/build_lists_controller.rb +++ b/app/controllers/projects/build_lists_controller.rb @@ -1,6 +1,6 @@ # -*- encoding : utf-8 -*- class Projects::BuildListsController < Projects::BaseController - include BuildsHelper + include FileStoreHelper NESTED_ACTIONS = [:index, :new, :create] diff --git a/app/helpers/builds_helper.rb b/app/helpers/file_store_helper.rb similarity index 52% rename from app/helpers/builds_helper.rb rename to app/helpers/file_store_helper.rb index 7883af12e..bb12c287e 100644 --- a/app/helpers/builds_helper.rb +++ b/app/helpers/file_store_helper.rb @@ -1,5 +1,5 @@ # -*- encoding : utf-8 -*- -module BuildsHelper +module FileStoreHelper def file_store_results_url(sha1, file_name) url = "#{APP_CONFIG['file_store_url']}/api/v1/file_stores/#{sha1}" @@ -7,4 +7,12 @@ module BuildsHelper url end + def link_to_file_store(file_name, sha1) + if sha1.present? + link_to file_name, file_store_results_url(sha1, file_name) + else + I18n.t('layout.no_') + end + end + end diff --git a/app/models/build_list.rb b/app/models/build_list.rb index 603e1bba8..149fd2e11 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -1,7 +1,7 @@ # -*- encoding : utf-8 -*- class BuildList < ActiveRecord::Base include Modules::Models::CommitAndVersion - include Modules::Models::FileStoreClean + include FileStoreClean include AbfWorker::ModelHelper include Modules::Observers::ActivityFeed::BuildList diff --git a/app/models/concerns/file_store_clean.rb b/app/models/concerns/file_store_clean.rb new file mode 100644 index 000000000..cca6336a4 --- /dev/null +++ b/app/models/concerns/file_store_clean.rb @@ -0,0 +1,52 @@ +# -*- encoding : utf-8 -*- +module FileStoreClean + extend ActiveSupport::Concern + + included do + def destroy + destroy_files_from_file_store if Rails.env.production? + super + end + later :destroy, :queue => :clone_build + + def sha1_of_file_store_files + raise NotImplementedError, "You should implement this method" + end + + def destroy_files_from_file_store(args = sha1_of_file_store_files) + files = *args + token = User.find_by_uname('file_store').authentication_token + uri = URI APP_CONFIG['file_store_url'] + Net::HTTP.start(uri.host, uri.port) do |http| + files.each do |sha1| + begin + req = Net::HTTP::Delete.new("/api/v1/file_stores/#{sha1}.json") + req.basic_auth token, '' + http.request(req) + rescue # Dont care about it + end + end + end + end + + def later_destroy_files_from_file_store(args) + destroy_files_from_file_store(args) + end + later :later_destroy_files_from_file_store, :queue => :clone_build + + end + + def self.file_exist_on_file_store?(sha1) + begin + resp = JSON(RestClient.get "#{APP_CONFIG['file_store_url']}/api/v1/file_stores.json", :params => {:hash => sha1}) + rescue # Dont care about it + resp = [] + end + if resp[0].respond_to?('[]') && resp[0]['file_name'] && resp[0]['sha1_hash'] + true + else + false + end + end + +end diff --git a/app/models/concerns/regeneration_status.rb b/app/models/concerns/regeneration_status.rb index f2e920b49..ac2d52a09 100644 --- a/app/models/concerns/regeneration_status.rb +++ b/app/models/concerns/regeneration_status.rb @@ -1,12 +1,5 @@ module RegenerationStatus - - def human_regeneration_status - self.class::HUMAN_REGENERATION_STATUSES[last_regenerated_status] || :no_data - end - - def human_status - self.class::HUMAN_STATUSES[status] || :no_data - end + extend ActiveSupport::Concern READY = 0 WAITING_FOR_REGENERATION = 100 @@ -24,4 +17,29 @@ module RegenerationStatus AbfWorker::BaseObserver::CANCELED => :canceled }.freeze + included do + after_update :cleanup_file_store + + def sha1_of_file_store_files + files = [] + files << last_regenerated_log_sha1 if last_regenerated_log_sha1.present? + files + end + + def human_regeneration_status + self.class::HUMAN_REGENERATION_STATUSES[last_regenerated_status] || :no_data + end + + def human_status + self.class::HUMAN_STATUSES[status] || :no_data + end + + def cleanup_file_store + old_log_sha1 = last_regenerated_log_sha1_was + if old_log_sha1.present? && old_log_sha1 != last_regenerated_log_sha1 + later_destroy_files_from_file_store([old_log_sha1]) + end + end + end + end \ No newline at end of file diff --git a/app/models/platform.rb b/app/models/platform.rb index f77fb7373..45eb5a04d 100644 --- a/app/models/platform.rb +++ b/app/models/platform.rb @@ -1,5 +1,6 @@ # -*- encoding : utf-8 -*- class Platform < ActiveRecord::Base + include FileStoreClean include RegenerationStatus VISIBILITIES = %w(open hidden) diff --git a/app/models/product_build_list.rb b/app/models/product_build_list.rb index 20fc1e689..00bb32a25 100644 --- a/app/models/product_build_list.rb +++ b/app/models/product_build_list.rb @@ -2,7 +2,7 @@ class ProductBuildList < ActiveRecord::Base include Modules::Models::CommitAndVersion include Modules::Models::TimeLiving - include Modules::Models::FileStoreClean + include FileStoreClean include AbfWorker::ModelHelper delegate :url_helpers, to: 'Rails.application.routes' diff --git a/app/models/project.rb b/app/models/project.rb index bdcb95feb..529c21927 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -215,11 +215,11 @@ class Project < ActiveRecord::Base format_id = ProjectTag::FORMATS["#{tag_file_format(format)}"] project_tag = project_tags.where(:tag_name => tag.name, :format_id => format_id).first - return project_tag.sha1 if project_tag && project_tag.commit_id == tag.commit.id && Modules::Models::FileStoreClean.file_exist_on_file_store?(project_tag.sha1) + return project_tag.sha1 if project_tag && project_tag.commit_id == tag.commit.id && FileStoreClean.file_exist_on_file_store?(project_tag.sha1) archive = archive_by_treeish_and_format tag.name, format sha1 = Digest::SHA1.file(archive[:path]).hexdigest - unless Modules::Models::FileStoreClean.file_exist_on_file_store? sha1 + unless FileStoreClean.file_exist_on_file_store? sha1 token = User.find_by_uname('rosa_system').authentication_token begin resp = JSON `curl --user #{token}: -POST -F 'file_store[file]=@#{archive[:path]};filename=#{name}-#{tag.name}.#{tag_file_format(format)}' #{APP_CONFIG['file_store_url']}/api/v1/upload` diff --git a/app/models/project_tag.rb b/app/models/project_tag.rb index 171a3bf5e..e67dba391 100644 --- a/app/models/project_tag.rb +++ b/app/models/project_tag.rb @@ -1,6 +1,6 @@ # -*- encoding : utf-8 -*- class ProjectTag < ActiveRecord::Base - include Modules::Models::FileStoreClean + include FileStoreClean FORMATS = { 'zip' => 0, diff --git a/app/models/repository_status.rb b/app/models/repository_status.rb index aec3aa78a..e17c5bfbc 100644 --- a/app/models/repository_status.rb +++ b/app/models/repository_status.rb @@ -1,4 +1,5 @@ class RepositoryStatus < ActiveRecord::Base + include FileStoreClean include RegenerationStatus WAITING_FOR_RESIGN = 300 diff --git a/app/views/platforms/platforms/edit.html.haml b/app/views/platforms/platforms/edit.html.haml index 61a846409..9d8323e44 100644 --- a/app/views/platforms/platforms/edit.html.haml +++ b/app/views/platforms/platforms/edit.html.haml @@ -27,6 +27,9 @@ .leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_status') .rightlist= t("layout.regeneration_statuses.last_regenerated_statuses.#{@platform.human_regeneration_status}") .both + .leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_log_sha1') + .rightlist= link_to_file_store('regeneration.log', @platform.last_regenerated_log_sha1) + .both .leftside .rightside= link_to t('layout.regeneration_statuses.regenerate_metadata'), regenerate_metadata_platform_path(@platform), :method => :put, :confirm => t('layout.confirm'), :class => 'button' .both diff --git a/app/views/platforms/repositories/edit.html.haml b/app/views/platforms/repositories/edit.html.haml index 799ab908a..b7550688f 100644 --- a/app/views/platforms/repositories/edit.html.haml +++ b/app/views/platforms/repositories/edit.html.haml @@ -25,6 +25,7 @@ %tr %th= t('activerecord.attributes.regeneration_status.status') %th= t('activerecord.attributes.regeneration_status.last_regenerated_status') + %th= t('activerecord.attributes.regeneration_status.last_regenerated_log_sha1') %th= t('activerecord.attributes.regeneration_status.last_regenerated_at') - unless @platform.main? %th= t('activerecord.models.platform') @@ -33,6 +34,7 @@ %tr{:class => cycle('odd', 'even')} %td= t("layout.regeneration_statuses.statuses.#{status.human_status}") %td= t("layout.regeneration_statuses.last_regenerated_statuses.#{status.human_regeneration_status}") + %td= link_to_file_store('regeneration.log', @status.last_regenerated_log_sha1) %td= status.last_regenerated_at - unless @platform.main? %td= status.platform.name diff --git a/config/locales/models/regeneration_status.en.yml b/config/locales/models/regeneration_status.en.yml index 1b12ff2b3..8ac264eba 100644 --- a/config/locales/models/regeneration_status.en.yml +++ b/config/locales/models/regeneration_status.en.yml @@ -27,4 +27,5 @@ en: regeneration_status: last_regenerated_at: Last regeneration of metadata last_regenerated_status: Last status of regeneration metadata + last_regenerated_log_sha1: Last log of regeneration metadata status: Current status diff --git a/config/locales/models/regeneration_status.ru.yml b/config/locales/models/regeneration_status.ru.yml index 9acd20dde..cd6ca71a0 100644 --- a/config/locales/models/regeneration_status.ru.yml +++ b/config/locales/models/regeneration_status.ru.yml @@ -27,4 +27,5 @@ ru: regeneration_status: last_regenerated_at: Последняя регенерация методанных last_regenerated_status: Статус последней регенерации методанных + last_regenerated_log_sha1: Последний лог регенерации методанных status: Текущее состояние diff --git a/db/migrate/20130827144022_add_last_regenerated_log_sha1_to_platform_and_repository_status.rb b/db/migrate/20130827144022_add_last_regenerated_log_sha1_to_platform_and_repository_status.rb new file mode 100644 index 000000000..1445541b7 --- /dev/null +++ b/db/migrate/20130827144022_add_last_regenerated_log_sha1_to_platform_and_repository_status.rb @@ -0,0 +1,6 @@ +class AddLastRegeneratedLogSha1ToPlatformAndRepositoryStatus < ActiveRecord::Migration + def change + add_column :platforms, :last_regenerated_log_sha1, :string + add_column :repository_statuses, :last_regenerated_log_sha1, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index d79748bf5..e64fc12d4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 20130822160501) do +ActiveRecord::Schema.define(:version => 20130827144022) do create_table "activity_feeds", :force => true do |t| t.integer "user_id", :null => false @@ -307,19 +307,20 @@ ActiveRecord::Schema.define(:version => 20130822160501) do create_table "platforms", :force => true do |t| t.string "description" - t.string "name", :null => false + t.string "name", :null => false t.integer "parent_platform_id" t.datetime "created_at" t.datetime "updated_at" - t.boolean "released", :default => false, :null => false + t.boolean "released", :default => false, :null => false t.integer "owner_id" t.string "owner_type" - t.string "visibility", :default => "open", :null => false - t.string "platform_type", :default => "main", :null => false - t.string "distrib_type", :null => false - t.integer "status", :default => 0 + t.string "visibility", :default => "open", :null => false + t.string "platform_type", :default => "main", :null => false + t.string "distrib_type", :null => false + t.integer "status", :default => 0 t.datetime "last_regenerated_at" t.integer "last_regenerated_status" + t.string "last_regenerated_log_sha1" end add_index "platforms", ["name"], :name => "index_platforms_on_name", :unique => true, :case_sensitive => false @@ -488,13 +489,14 @@ ActiveRecord::Schema.define(:version => 20130822160501) do add_index "repositories", ["platform_id"], :name => "index_repositories_on_platform_id" create_table "repository_statuses", :force => true do |t| - t.integer "repository_id", :null => false - t.integer "platform_id", :null => false - t.integer "status", :default => 0 + t.integer "repository_id", :null => false + t.integer "platform_id", :null => false + t.integer "status", :default => 0 t.datetime "last_regenerated_at" t.integer "last_regenerated_status" - t.datetime "created_at", :null => false - t.datetime "updated_at", :null => false + t.datetime "created_at", :null => false + t.datetime "updated_at", :null => false + t.string "last_regenerated_log_sha1" end add_index "repository_statuses", ["repository_id", "platform_id"], :name => "index_repository_statuses_on_repository_id_and_platform_id", :unique => true diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index 84c1423f9..23d5feb55 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -352,7 +352,6 @@ module AbfWorker :arch => 'x86_64' }, :time_living => 9600, # 160 min - :skip_feedback => true, :extra => {:platform_id => platform.id, :regenerate_platform => true} }] ) if platform.start_regeneration @@ -395,7 +394,6 @@ module AbfWorker :arch => 'x86_64' }, :time_living => 9600, # 160 min - :skip_feedback => true, :extra => {:repository_status_id => repository_status.id, :regenerate => true} }] ) if repository_status.start_regeneration diff --git a/lib/abf_worker/publish_observer.rb b/lib/abf_worker/publish_observer.rb index 5f67d58c7..99a301459 100644 --- a/lib/abf_worker/publish_observer.rb +++ b/lib/abf_worker/publish_observer.rb @@ -12,13 +12,20 @@ module AbfWorker extra = options['extra'] repository_status = RepositoryStatus.where(:id => extra['repository_status_id']).first begin + + if extra['regenerate'] || extra['regenerate_platform'] + log_sha1 = (options['results'].try(:first) || {}).fetch('sha1', nil) + end + if extra['regenerate'] # Regenerate metadata repository_status.last_regenerated_at = Time.now.utc repository_status.last_regenerated_status = status + repository_status.last_regenerated_log_sha1 = log_sha1 elsif extra['regenerate_platform'] # Regenerate metadata for Software Center if platform = Platform.where(:id => extra['platform_id']).first platform.last_regenerated_at = Time.now.utc platform.last_regenerated_status = status + platform.last_regenerated_log_sha1 = log_sha1 platform.ready end elsif extra['create_container'] # Container has been created diff --git a/lib/modules/models/file_store_clean.rb b/lib/modules/models/file_store_clean.rb deleted file mode 100644 index cc28997cc..000000000 --- a/lib/modules/models/file_store_clean.rb +++ /dev/null @@ -1,50 +0,0 @@ -# -*- encoding : utf-8 -*- -module Modules - module Models - module FileStoreClean - extend ActiveSupport::Concern - - included do - def destroy - destroy_files_from_file_store if Rails.env.production? - super - end - later :destroy, :queue => :clone_build - - def sha1_of_file_store_files - raise NotImplementedError, "You should implement this method" - end - - def destroy_files_from_file_store(args = sha1_of_file_store_files) - files = *args - token = User.find_by_uname('file_store').authentication_token - uri = URI APP_CONFIG['file_store_url'] - Net::HTTP.start(uri.host, uri.port) do |http| - files.each do |sha1| - begin - req = Net::HTTP::Delete.new("/api/v1/file_stores/#{sha1}.json") - req.basic_auth token, '' - http.request(req) - rescue # Dont care about it - end - end - end - end - end - - def self.file_exist_on_file_store?(sha1) - begin - resp = JSON(RestClient.get "#{APP_CONFIG['file_store_url']}/api/v1/file_stores.json", :params => {:hash => sha1}) - rescue # Dont care about it - resp = [] - end - if resp[0].respond_to?('[]') && resp[0]['file_name'] && resp[0]['sha1_hash'] - true - else - false - end - end - - end - end -end