Merge pull request #282 from abf/rosa-build:281-show-repodata-creation-logs

#281: Show repodata creation logs

app/controllers/platforms/platforms_controller.rb

@@ -1,5 +1,6 @@
 # -*- encoding : utf-8 -*-
 class Platforms::PlatformsController < Platforms::BaseController
+  include FileStoreHelper
 
   before_filter :authenticate_user!
   skip_before_filter :authenticate_user!, :only => [:advisories, :members, :show] if APP_CONFIG['anonymous_access']

app/controllers/platforms/product_build_lists_controller.rb

@@ -1,6 +1,6 @@
 # -*- encoding : utf-8 -*-
 class Platforms::ProductBuildListsController < Platforms::BaseController
-  include BuildsHelper
+  include FileStoreHelper
 
   before_filter :authenticate_user!
   skip_before_filter :authenticate_user!, :only => [:index, :show, :log] if APP_CONFIG['anonymous_access']

app/controllers/platforms/repositories_controller.rb

@@ -1,5 +1,7 @@
 # -*- encoding : utf-8 -*-
 class Platforms::RepositoriesController < Platforms::BaseController
+  include FileStoreHelper
+
   before_filter :authenticate_user!
   skip_before_filter :authenticate_user!, :only => [:index, :show, :projects_list] if APP_CONFIG['anonymous_access']
 

app/controllers/projects/build_lists_controller.rb

@@ -1,6 +1,6 @@
 # -*- encoding : utf-8 -*-
 class Projects::BuildListsController < Projects::BaseController
-  include BuildsHelper
+  include FileStoreHelper
 
   NESTED_ACTIONS = [:index, :new, :create]
 

app/helpers/file_store_helper.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
-module BuildsHelper
+module FileStoreHelper
 
   def file_store_results_url(sha1, file_name)
     url = "#{APP_CONFIG['file_store_url']}/api/v1/file_stores/#{sha1}"
@@ -7,4 +7,12 @@ module BuildsHelper
     url
   end
 
+  def link_to_file_store(file_name, sha1)
+    if sha1.present?
+      link_to file_name, file_store_results_url(sha1, file_name)
+    else
+      I18n.t('layout.no_')
+    end
+  end
+
 end

app/models/concerns/regeneration_status.rb

@@ -1,27 +0,0 @@
-module RegenerationStatus
-
-  def human_regeneration_status
-    self.class::HUMAN_REGENERATION_STATUSES[last_regenerated_status] || :no_data
-  end
-
-  def human_status
-    self.class::HUMAN_STATUSES[status] || :no_data
-  end
-
-  READY                     = 0
-  WAITING_FOR_REGENERATION  = 100
-  REGENERATING              = 200
-
-  HUMAN_STATUSES = {  
-    READY                     => :ready,
-    WAITING_FOR_REGENERATION  => :waiting_for_regeneration,
-    REGENERATING              => :regenerating
-  }
-
-  HUMAN_REGENERATION_STATUSES = {
-    AbfWorker::BaseObserver::COMPLETED  => :completed,
-    AbfWorker::BaseObserver::FAILED     => :failed,
-    AbfWorker::BaseObserver::CANCELED   => :canceled
-  }.freeze
-
-end

app/models/platform.rb

@@ -1,6 +1,7 @@
 # -*- encoding : utf-8 -*-
 class Platform < ActiveRecord::Base
-  include RegenerationStatus
+  include Modules::Models::FileStoreClean
+  include Modules::Models::RegenerationStatus
 
   VISIBILITIES = %w(open hidden)
   NAME_PATTERN = /[\w\-\.]+/

app/models/repository_status.rb

@@ -1,5 +1,6 @@
 class RepositoryStatus < ActiveRecord::Base
-  include RegenerationStatus
+  include Modules::Models::FileStoreClean
+  include Modules::Models::RegenerationStatus
 
   WAITING_FOR_RESIGN        = 300
   PUBLISH                   = 400

app/views/platforms/platforms/edit.html.haml

@@ -27,6 +27,9 @@
   .leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_status')
   .rightlist= t("layout.regeneration_statuses.last_regenerated_statuses.#{@platform.human_regeneration_status}")
   .both
+  .leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_log_sha1')
+  .rightlist= link_to_file_store('regeneration.log', @platform.last_regenerated_log_sha1)
+  .both
   .leftside
   .rightside= link_to t('layout.regeneration_statuses.regenerate_metadata'), regenerate_metadata_platform_path(@platform), :method => :put, :confirm => t('layout.confirm'), :class => 'button'
   .both

app/views/platforms/repositories/edit.html.haml

@@ -25,6 +25,7 @@
         %tr
           %th= t('activerecord.attributes.regeneration_status.status')
           %th= t('activerecord.attributes.regeneration_status.last_regenerated_status')
+          %th= t('activerecord.attributes.regeneration_status.last_regenerated_log_sha1')
           %th= t('activerecord.attributes.regeneration_status.last_regenerated_at')
           - unless @platform.main?
             %th= t('activerecord.models.platform')
@@ -33,6 +34,7 @@
           %tr{:class => cycle('odd', 'even')}
             %td= t("layout.regeneration_statuses.statuses.#{status.human_status}")
             %td= t("layout.regeneration_statuses.last_regenerated_statuses.#{status.human_regeneration_status}")
+            %td= link_to_file_store('regeneration.log', status.last_regenerated_log_sha1)
             %td= status.last_regenerated_at
             - unless @platform.main?
               %td= status.platform.name

config/application.rb

@@ -28,7 +28,6 @@ module Rosa
     # Custom directories with classes and modules you want to be autoloadable.
     # config.autoload_paths += %W(#{config.root}/extras)
     config.autoload_paths += %W(#{config.root}/app/presenters)
-    config.autoload_paths += %W(#{config.root}/app/models/concerns)
 
     # Only load the plugins named here, in the order given (default is alphabetical).
     # :all can be used as a placeholder for all plugins not explicitly named.

config/locales/models/regeneration_status.en.yml

@@ -27,4 +27,5 @@ en:
       regeneration_status:
         last_regenerated_at: Last regeneration of metadata
         last_regenerated_status: Last status of regeneration metadata
+        last_regenerated_log_sha1: Last log of regeneration metadata
         status: Current status

config/locales/models/regeneration_status.ru.yml

@@ -5,17 +5,17 @@ ru:
       statuses:
         no_data: Нет данных
         ready: Нет действий
-        waiting_for_regeneration: Ожидание регенерации методанных
+        waiting_for_regeneration: Ожидание регенерации метаданных
-        regenerating: Идет регенерация методанных
+        regenerating: Идет регенерация метаданных
         waiting_for_resign: Ожидание подписи пакетов
         publish: Идет публикация
         resign: Идет подпись пакетов
         waiting_for_resign_after_publish: Идет публикация, ожидание подписи пакетов
-        waiting_for_resign_after_regeneration: Идет регенерация методанных, ожидание подписи пакетов
+        waiting_for_resign_after_regeneration: Идет регенерация метаданных, ожидание подписи пакетов
-        waiting_for_regeneration_after_publish: Идет публикация, ожидание регенерации методанных
+        waiting_for_regeneration_after_publish: Идет публикация, ожидание регенерации метаданных
-        waiting_for_regeneration_after_resign: Идет подпись пакетов, ожидание регенерации методанных
+        waiting_for_regeneration_after_resign: Идет подпись пакетов, ожидание регенерации метаданных
-        waiting_for_resign_and_regeneration_after_publish: Идет публикация, ожидание подписи пакетов и регенерации методанных
+        waiting_for_resign_and_regeneration_after_publish: Идет публикация, ожидание подписи пакетов и регенерации метаданных
-        waiting_for_resign_and_regeneration: Ожидание подписи пакетов и регенерации методанных
+        waiting_for_resign_and_regeneration: Ожидание подписи пакетов и регенерации метаданных
       last_regenerated_statuses:
         no_data: Нет данных
         completed: Завершена успешно
@@ -25,6 +25,7 @@ ru:
   activerecord:
     attributes:
       regeneration_status:
-        last_regenerated_at: Последняя регенерация методанных
+        last_regenerated_at: Последняя регенерация метаданных
-        last_regenerated_status: Статус последней регенерации методанных
+        last_regenerated_status: Статус последней регенерации метаданных
+        last_regenerated_log_sha1: Последний лог регенерации метаданных
         status: Текущее состояние

db/migrate/20130827144022_add_last_regenerated_log_sha1_to_platform_and_repository_status.rb

@@ -0,0 +1,6 @@
+class AddLastRegeneratedLogSha1ToPlatformAndRepositoryStatus < ActiveRecord::Migration
+  def change
+    add_column :platforms, :last_regenerated_log_sha1, :string
+    add_column :repository_statuses, :last_regenerated_log_sha1, :string
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20130822160501) do
+ActiveRecord::Schema.define(:version => 20130827144022) do
 
   create_table "activity_feeds", :force => true do |t|
     t.integer  "user_id",    :null => false
@@ -307,19 +307,20 @@ ActiveRecord::Schema.define(:version => 20130822160501) do
 
   create_table "platforms", :force => true do |t|
     t.string   "description"
-    t.string   "name",                                        :null => false
+    t.string   "name",                                          :null => false
     t.integer  "parent_platform_id"
     t.datetime "created_at"
     t.datetime "updated_at"
-    t.boolean  "released",                :default => false,  :null => false
+    t.boolean  "released",                  :default => false,  :null => false
     t.integer  "owner_id"
     t.string   "owner_type"
-    t.string   "visibility",              :default => "open", :null => false
+    t.string   "visibility",                :default => "open", :null => false
-    t.string   "platform_type",           :default => "main", :null => false
+    t.string   "platform_type",             :default => "main", :null => false
-    t.string   "distrib_type",                                :null => false
+    t.string   "distrib_type",                                  :null => false
-    t.integer  "status",                  :default => 0
+    t.integer  "status",                    :default => 0
     t.datetime "last_regenerated_at"
     t.integer  "last_regenerated_status"
+    t.string   "last_regenerated_log_sha1"
   end
 
   add_index "platforms", ["name"], :name => "index_platforms_on_name", :unique => true, :case_sensitive => false
@@ -488,13 +489,14 @@ ActiveRecord::Schema.define(:version => 20130822160501) do
   add_index "repositories", ["platform_id"], :name => "index_repositories_on_platform_id"
 
   create_table "repository_statuses", :force => true do |t|
-    t.integer  "repository_id",                          :null => false
+    t.integer  "repository_id",                            :null => false
-    t.integer  "platform_id",                            :null => false
+    t.integer  "platform_id",                              :null => false
-    t.integer  "status",                  :default => 0
+    t.integer  "status",                    :default => 0
     t.datetime "last_regenerated_at"
     t.integer  "last_regenerated_status"
-    t.datetime "created_at",                             :null => false
+    t.datetime "created_at",                               :null => false
-    t.datetime "updated_at",                             :null => false
+    t.datetime "updated_at",                               :null => false
+    t.string   "last_regenerated_log_sha1"
   end
 
   add_index "repository_statuses", ["repository_id", "platform_id"], :name => "index_repository_statuses_on_repository_id_and_platform_id", :unique => true

lib/abf_worker/build_lists_publish_task_manager.rb

@@ -352,7 +352,6 @@ module AbfWorker
               :arch           => 'x86_64'
             },
             :time_living  => 9600, # 160 min
-            :skip_feedback => true,
             :extra         => {:platform_id => platform.id, :regenerate_platform => true}
           }]
         ) if platform.start_regeneration
@@ -395,7 +394,6 @@ module AbfWorker
               :arch           => 'x86_64'
             },
             :time_living  => 9600, # 160 min
-            :skip_feedback => true,
             :extra         => {:repository_status_id => repository_status.id, :regenerate => true}
           }]
         ) if repository_status.start_regeneration

lib/abf_worker/publish_observer.rb

@@ -12,13 +12,20 @@ module AbfWorker
       extra = options['extra']
       repository_status = RepositoryStatus.where(:id => extra['repository_status_id']).first
       begin
+        
+        if extra['regenerate'] || extra['regenerate_platform']
+          log_sha1 = (options['results'].try(:first) || {}).fetch('sha1', nil)
+        end
+
         if extra['regenerate'] # Regenerate metadata
           repository_status.last_regenerated_at = Time.now.utc
           repository_status.last_regenerated_status = status
+          repository_status.last_regenerated_log_sha1 = log_sha1
         elsif extra['regenerate_platform'] # Regenerate metadata for Software Center
           if platform = Platform.where(:id => extra['platform_id']).first
             platform.last_regenerated_at = Time.now.utc
             platform.last_regenerated_status = status
+            platform.last_regenerated_log_sha1 = log_sha1
             platform.ready
           end
         elsif extra['create_container'] # Container has been created

lib/modules/models/file_store_clean.rb

@@ -30,6 +30,11 @@ module Modules
             end
           end
         end
+
+        def later_destroy_files_from_file_store(args)
+          destroy_files_from_file_store(args)
+        end
+        later :later_destroy_files_from_file_store, :queue => :clone_build
       end
 
       def self.file_exist_on_file_store?(sha1)
@@ -47,4 +52,4 @@ module Modules
 
     end
   end
-end
+end

lib/modules/models/regeneration_status.rb

@@ -0,0 +1,49 @@
+# -*- encoding : utf-8 -*-
+module Modules
+  module Models
+    module RegenerationStatus
+      extend ActiveSupport::Concern
+
+      READY                     = 0
+      WAITING_FOR_REGENERATION  = 100
+      REGENERATING              = 200
+
+      HUMAN_STATUSES = {  
+        READY                     => :ready,
+        WAITING_FOR_REGENERATION  => :waiting_for_regeneration,
+        REGENERATING              => :regenerating
+      }
+
+      HUMAN_REGENERATION_STATUSES = {
+        AbfWorker::BaseObserver::COMPLETED  => :completed,
+        AbfWorker::BaseObserver::FAILED     => :failed,
+        AbfWorker::BaseObserver::CANCELED   => :canceled
+      }.freeze
+
+      included do
+        after_update :cleanup_file_store
+
+        def sha1_of_file_store_files
+          files = []
+          files << last_regenerated_log_sha1 if last_regenerated_log_sha1.present?
+          files
+        end
+
+        def human_regeneration_status
+          self.class::HUMAN_REGENERATION_STATUSES[last_regenerated_status] || :no_data
+        end
+
+        def human_status
+          self.class::HUMAN_STATUSES[status] || :no_data
+        end
+
+        def cleanup_file_store
+          old_log_sha1 = last_regenerated_log_sha1_was
+          if old_log_sha1.present? && old_log_sha1 != last_regenerated_log_sha1
+            later_destroy_files_from_file_store([old_log_sha1])
+          end
+        end
+      end
+    end
+  end
+end