#281: Show repodata creation logs

app/controllers/platforms/platforms_controller.rb

@@ -1,5 +1,6 @@
 # -*- encoding : utf-8 -*-
 class Platforms::PlatformsController < Platforms::BaseController
+  include FileStoreHelper
 
   before_filter :authenticate_user!
   skip_before_filter :authenticate_user!, :only => [:advisories, :members, :show] if APP_CONFIG['anonymous_access']

app/controllers/platforms/product_build_lists_controller.rb

@@ -1,6 +1,6 @@
 # -*- encoding : utf-8 -*-
 class Platforms::ProductBuildListsController < Platforms::BaseController
-  include BuildsHelper
+  include FileStoreHelper
 
   before_filter :authenticate_user!
   skip_before_filter :authenticate_user!, :only => [:index, :show, :log] if APP_CONFIG['anonymous_access']

app/controllers/platforms/repositories_controller.rb

@@ -1,5 +1,7 @@
 # -*- encoding : utf-8 -*-
 class Platforms::RepositoriesController < Platforms::BaseController
+  include FileStoreHelper
+
   before_filter :authenticate_user!
   skip_before_filter :authenticate_user!, :only => [:index, :show, :projects_list] if APP_CONFIG['anonymous_access']
 

app/controllers/projects/build_lists_controller.rb

@@ -1,6 +1,6 @@
 # -*- encoding : utf-8 -*-
 class Projects::BuildListsController < Projects::BaseController
-  include BuildsHelper
+  include FileStoreHelper
 
   NESTED_ACTIONS = [:index, :new, :create]
 

app/helpers/file_store_helper.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
-module BuildsHelper
+module FileStoreHelper
 
   def file_store_results_url(sha1, file_name)
     url = "#{APP_CONFIG['file_store_url']}/api/v1/file_stores/#{sha1}"
@@ -7,4 +7,12 @@ module BuildsHelper
     url
   end
 
+  def link_to_file_store(file_name, sha1)
+    if sha1.present?
+      link_to file_name, file_store_results_url(sha1, file_name)
+    else
+      I18n.t('layout.no_')
+    end
+  end
+
 end

app/models/build_list.rb

@@ -1,7 +1,7 @@
 # -*- encoding : utf-8 -*-
 class BuildList < ActiveRecord::Base
   include Modules::Models::CommitAndVersion
-  include Modules::Models::FileStoreClean
+  include FileStoreClean
   include AbfWorker::ModelHelper
   include Modules::Observers::ActivityFeed::BuildList
 

app/models/concerns/file_store_clean.rb

@@ -0,0 +1,52 @@
+# -*- encoding : utf-8 -*-
+module FileStoreClean
+  extend ActiveSupport::Concern
+
+  included do
+    def destroy
+      destroy_files_from_file_store if Rails.env.production?
+      super
+    end
+    later :destroy, :queue => :clone_build
+
+    def sha1_of_file_store_files
+      raise NotImplementedError, "You should implement this method"
+    end
+
+    def destroy_files_from_file_store(args = sha1_of_file_store_files)
+      files = *args
+      token = User.find_by_uname('file_store').authentication_token
+      uri   = URI APP_CONFIG['file_store_url']
+      Net::HTTP.start(uri.host, uri.port) do |http|
+        files.each do |sha1|
+          begin
+            req = Net::HTTP::Delete.new("/api/v1/file_stores/#{sha1}.json")
+            req.basic_auth token, ''
+            http.request(req)
+          rescue # Dont care about it
+          end
+        end
+      end
+    end
+
+    def later_destroy_files_from_file_store(args)
+      destroy_files_from_file_store(args)
+    end
+    later :later_destroy_files_from_file_store, :queue => :clone_build
+
+  end
+
+  def self.file_exist_on_file_store?(sha1)
+    begin
+      resp = JSON(RestClient.get "#{APP_CONFIG['file_store_url']}/api/v1/file_stores.json", :params => {:hash => sha1})
+    rescue # Dont care about it
+      resp = []
+    end
+    if resp[0].respond_to?('[]') && resp[0]['file_name'] && resp[0]['sha1_hash']
+      true
+    else
+      false
+    end
+  end
+
+end

app/models/concerns/regeneration_status.rb

@@ -1,12 +1,5 @@
 module RegenerationStatus
-
+  extend ActiveSupport::Concern
-  def human_regeneration_status
-    self.class::HUMAN_REGENERATION_STATUSES[last_regenerated_status] || :no_data
-  end
-
-  def human_status
-    self.class::HUMAN_STATUSES[status] || :no_data
-  end
 
   READY                     = 0
   WAITING_FOR_REGENERATION  = 100
@@ -24,4 +17,29 @@ module RegenerationStatus
     AbfWorker::BaseObserver::CANCELED   => :canceled
   }.freeze
 
+  included do
+    after_update :cleanup_file_store
+
+    def sha1_of_file_store_files
+      files = []
+      files << last_regenerated_log_sha1 if last_regenerated_log_sha1.present?
+      files
+    end
+
+    def human_regeneration_status
+      self.class::HUMAN_REGENERATION_STATUSES[last_regenerated_status] || :no_data
+    end
+
+    def human_status
+      self.class::HUMAN_STATUSES[status] || :no_data
+    end
+
+    def cleanup_file_store
+      old_log_sha1 = last_regenerated_log_sha1_was
+      if old_log_sha1.present? && old_log_sha1 != last_regenerated_log_sha1
+        later_destroy_files_from_file_store([old_log_sha1])
+      end
+    end
+  end
+
 end

app/models/platform.rb

@@ -1,5 +1,6 @@
 # -*- encoding : utf-8 -*-
 class Platform < ActiveRecord::Base
+  include FileStoreClean
   include RegenerationStatus
 
   VISIBILITIES = %w(open hidden)

app/models/product_build_list.rb

@@ -2,7 +2,7 @@
 class ProductBuildList < ActiveRecord::Base
   include Modules::Models::CommitAndVersion
   include Modules::Models::TimeLiving
-  include Modules::Models::FileStoreClean
+  include FileStoreClean
   include AbfWorker::ModelHelper
   delegate :url_helpers, to: 'Rails.application.routes'
 

app/models/project.rb

@@ -215,11 +215,11 @@ class Project < ActiveRecord::Base
     format_id = ProjectTag::FORMATS["#{tag_file_format(format)}"]
     project_tag = project_tags.where(:tag_name => tag.name, :format_id => format_id).first
 
-    return project_tag.sha1 if project_tag && project_tag.commit_id == tag.commit.id && Modules::Models::FileStoreClean.file_exist_on_file_store?(project_tag.sha1)
+    return project_tag.sha1 if project_tag && project_tag.commit_id == tag.commit.id && FileStoreClean.file_exist_on_file_store?(project_tag.sha1)
 
     archive = archive_by_treeish_and_format tag.name, format
     sha1    = Digest::SHA1.file(archive[:path]).hexdigest
-    unless Modules::Models::FileStoreClean.file_exist_on_file_store? sha1
+    unless FileStoreClean.file_exist_on_file_store? sha1
       token = User.find_by_uname('rosa_system').authentication_token
       begin
         resp = JSON `curl --user #{token}: -POST -F 'file_store[file]=@#{archive[:path]};filename=#{name}-#{tag.name}.#{tag_file_format(format)}' #{APP_CONFIG['file_store_url']}/api/v1/upload`

app/models/project_tag.rb

@@ -1,6 +1,6 @@
 # -*- encoding : utf-8 -*-
 class ProjectTag < ActiveRecord::Base
-  include Modules::Models::FileStoreClean
+  include FileStoreClean
 
   FORMATS = {
     'zip'     => 0,

app/models/repository_status.rb

@@ -1,4 +1,5 @@
 class RepositoryStatus < ActiveRecord::Base
+  include FileStoreClean
   include RegenerationStatus
 
   WAITING_FOR_RESIGN        = 300

app/views/platforms/platforms/edit.html.haml

@@ -27,6 +27,9 @@
   .leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_status')
   .rightlist= t("layout.regeneration_statuses.last_regenerated_statuses.#{@platform.human_regeneration_status}")
   .both
+  .leftlist= t('activerecord.attributes.regeneration_status.last_regenerated_log_sha1')
+  .rightlist= link_to_file_store('regeneration.log', @platform.last_regenerated_log_sha1)
+  .both
   .leftside
   .rightside= link_to t('layout.regeneration_statuses.regenerate_metadata'), regenerate_metadata_platform_path(@platform), :method => :put, :confirm => t('layout.confirm'), :class => 'button'
   .both

app/views/platforms/repositories/edit.html.haml

@@ -25,6 +25,7 @@
         %tr
           %th= t('activerecord.attributes.regeneration_status.status')
           %th= t('activerecord.attributes.regeneration_status.last_regenerated_status')
+          %th= t('activerecord.attributes.regeneration_status.last_regenerated_log_sha1')
           %th= t('activerecord.attributes.regeneration_status.last_regenerated_at')
           - unless @platform.main?
             %th= t('activerecord.models.platform')
@@ -33,6 +34,7 @@
           %tr{:class => cycle('odd', 'even')}
             %td= t("layout.regeneration_statuses.statuses.#{status.human_status}")
             %td= t("layout.regeneration_statuses.last_regenerated_statuses.#{status.human_regeneration_status}")
+            %td= link_to_file_store('regeneration.log', @status.last_regenerated_log_sha1)
             %td= status.last_regenerated_at
             - unless @platform.main?
               %td= status.platform.name

config/locales/models/regeneration_status.en.yml

@@ -27,4 +27,5 @@ en:
       regeneration_status:
         last_regenerated_at: Last regeneration of metadata
         last_regenerated_status: Last status of regeneration metadata
+        last_regenerated_log_sha1: Last log of regeneration metadata
         status: Current status

config/locales/models/regeneration_status.ru.yml

@@ -27,4 +27,5 @@ ru:
       regeneration_status:
         last_regenerated_at: Последняя регенерация методанных
         last_regenerated_status: Статус последней регенерации методанных
+        last_regenerated_log_sha1: Последний лог регенерации методанных
         status: Текущее состояние

db/migrate/20130827144022_add_last_regenerated_log_sha1_to_platform_and_repository_status.rb

@@ -0,0 +1,6 @@
+class AddLastRegeneratedLogSha1ToPlatformAndRepositoryStatus < ActiveRecord::Migration
+  def change
+    add_column :platforms, :last_regenerated_log_sha1, :string
+    add_column :repository_statuses, :last_regenerated_log_sha1, :string
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20130822160501) do
+ActiveRecord::Schema.define(:version => 20130827144022) do
 
   create_table "activity_feeds", :force => true do |t|
     t.integer  "user_id",    :null => false
@@ -307,19 +307,20 @@ ActiveRecord::Schema.define(:version => 20130822160501) do
 
   create_table "platforms", :force => true do |t|
     t.string   "description"
-    t.string   "name",                                        :null => false
+    t.string   "name",                                          :null => false
     t.integer  "parent_platform_id"
     t.datetime "created_at"
     t.datetime "updated_at"
-    t.boolean  "released",                :default => false,  :null => false
+    t.boolean  "released",                  :default => false,  :null => false
     t.integer  "owner_id"
     t.string   "owner_type"
-    t.string   "visibility",              :default => "open", :null => false
+    t.string   "visibility",                :default => "open", :null => false
-    t.string   "platform_type",           :default => "main", :null => false
+    t.string   "platform_type",             :default => "main", :null => false
-    t.string   "distrib_type",                                :null => false
+    t.string   "distrib_type",                                  :null => false
-    t.integer  "status",                  :default => 0
+    t.integer  "status",                    :default => 0
     t.datetime "last_regenerated_at"
     t.integer  "last_regenerated_status"
+    t.string   "last_regenerated_log_sha1"
   end
 
   add_index "platforms", ["name"], :name => "index_platforms_on_name", :unique => true, :case_sensitive => false
@@ -488,13 +489,14 @@ ActiveRecord::Schema.define(:version => 20130822160501) do
   add_index "repositories", ["platform_id"], :name => "index_repositories_on_platform_id"
 
   create_table "repository_statuses", :force => true do |t|
-    t.integer  "repository_id",                          :null => false
+    t.integer  "repository_id",                            :null => false
-    t.integer  "platform_id",                            :null => false
+    t.integer  "platform_id",                              :null => false
-    t.integer  "status",                  :default => 0
+    t.integer  "status",                    :default => 0
     t.datetime "last_regenerated_at"
     t.integer  "last_regenerated_status"
-    t.datetime "created_at",                             :null => false
+    t.datetime "created_at",                               :null => false
-    t.datetime "updated_at",                             :null => false
+    t.datetime "updated_at",                               :null => false
+    t.string   "last_regenerated_log_sha1"
   end
 
   add_index "repository_statuses", ["repository_id", "platform_id"], :name => "index_repository_statuses_on_repository_id_and_platform_id", :unique => true

lib/abf_worker/build_lists_publish_task_manager.rb

@@ -352,7 +352,6 @@ module AbfWorker
               :arch           => 'x86_64'
             },
             :time_living  => 9600, # 160 min
-            :skip_feedback => true,
             :extra         => {:platform_id => platform.id, :regenerate_platform => true}
           }]
         ) if platform.start_regeneration
@@ -395,7 +394,6 @@ module AbfWorker
               :arch           => 'x86_64'
             },
             :time_living  => 9600, # 160 min
-            :skip_feedback => true,
             :extra         => {:repository_status_id => repository_status.id, :regenerate => true}
           }]
         ) if repository_status.start_regeneration

lib/abf_worker/publish_observer.rb

@@ -12,13 +12,20 @@ module AbfWorker
       extra = options['extra']
       repository_status = RepositoryStatus.where(:id => extra['repository_status_id']).first
       begin
+        
+        if extra['regenerate'] || extra['regenerate_platform']
+          log_sha1 = (options['results'].try(:first) || {}).fetch('sha1', nil)
+        end
+
         if extra['regenerate'] # Regenerate metadata
           repository_status.last_regenerated_at = Time.now.utc
           repository_status.last_regenerated_status = status
+          repository_status.last_regenerated_log_sha1 = log_sha1
         elsif extra['regenerate_platform'] # Regenerate metadata for Software Center
           if platform = Platform.where(:id => extra['platform_id']).first
             platform.last_regenerated_at = Time.now.utc
             platform.last_regenerated_status = status
+            platform.last_regenerated_log_sha1 = log_sha1
             platform.ready
           end
         elsif extra['create_container'] # Container has been created

lib/modules/models/file_store_clean.rb

@@ -1,50 +0,0 @@
-# -*- encoding : utf-8 -*-
-module Modules
-  module Models
-    module FileStoreClean
-      extend ActiveSupport::Concern
-
-      included do
-        def destroy
-          destroy_files_from_file_store if Rails.env.production?
-          super
-        end
-        later :destroy, :queue => :clone_build
-
-        def sha1_of_file_store_files
-          raise NotImplementedError, "You should implement this method"
-        end
-
-        def destroy_files_from_file_store(args = sha1_of_file_store_files)
-          files = *args
-          token = User.find_by_uname('file_store').authentication_token
-          uri   = URI APP_CONFIG['file_store_url']
-          Net::HTTP.start(uri.host, uri.port) do |http|
-            files.each do |sha1|
-              begin
-                req = Net::HTTP::Delete.new("/api/v1/file_stores/#{sha1}.json")
-                req.basic_auth token, ''
-                http.request(req)
-              rescue # Dont care about it
-              end
-            end
-          end
-        end
-      end
-
-      def self.file_exist_on_file_store?(sha1)
-        begin
-          resp = JSON(RestClient.get "#{APP_CONFIG['file_store_url']}/api/v1/file_stores.json", :params => {:hash => sha1})
-        rescue # Dont care about it
-          resp = []
-        end
-        if resp[0].respond_to?('[]') && resp[0]['file_name'] && resp[0]['sha1_hash']
-          true
-        else
-          false
-        end
-      end
-
-    end
-  end
-end