Removed builder clear job

1.rb

@@ -0,0 +1,23 @@
+require 'json'
+class Analyzer
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def analyze
+    data = []
+    File.open(@filename) do |f|
+        f.each_line do |line|
+          data << (parsed=JSON.parse(line))
+        end
+    end
+
+    data.group_by{|row| row["generation"]}
+        .sort{|a,b| a[0].to_i <=> b[0].to_i}
+        .each do |k,v|
+          puts "generation #{k} objects #{v.count}"
+        end
+  end
+end
+
+Analyzer.new(ARGV[0]).analyze

2.rb

@@ -0,0 +1,23 @@
+require 'json'
+class Analyzer
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def analyze
+    data = []
+    File.open(@filename) do |f|
+        f.each_line do |line|
+          parsed=JSON.parse(line)
+          data << parsed if parsed["generation"]
+        end
+    end
+    data.group_by{|row| "#{row["file"]}:#{row["line"]}"}
+        .sort{|a,b| b[1].count <=> a[1].count}
+        .each do |k,v|
+          puts "#{k} * #{v.count}"
+        end
+  end
+end
+
+Analyzer.new(ARGV[0]).analyze

app/controllers/api/v1/jobs_controller.rb

@@ -8,6 +8,7 @@ class Api::V1::JobsController < Api::V1::BaseController
   skip_after_action :verify_authorized
 
   def shift
+    clear_stale_builders
     job_shift_sem = Redis::Semaphore.new(:job_shift_lock)
     job_shift_sem.lock
     uid = BuildList.scoped_to_arch(arch_ids).
@@ -100,6 +101,14 @@ class Api::V1::JobsController < Api::V1::BaseController
 
   protected
 
+  def clear_stale_builders
+    BuildList.transaction do
+      BuildList.where(["updated_at < ?", 120.seconds.ago]).where(status: BuildList::BUILD_PENDING).where.not(builder: nil).find_each(batch_size: 50) do |bl|
+        bl.update_column(builder_id: nil)
+      end
+    end
+  end
+
   def platform_ids
     @platform_ids ||= begin
       platforms = params[:platforms].to_s.split(',')

app/controllers/projects/build_lists_controller.rb

@@ -103,7 +103,7 @@ class Projects::BuildListsController < Projects::BaseController
       if prs.present? && prs[:projects].present? && prs[:arches].present?
         project_ids = prs[:projects].select{ |k, v| v == '1'  }.keys
         arch_ids    = prs[:arches].  select{ |k, v| v == '1'  }.keys
-        #FIX
+
         Sidekiq::Client.push(
           'class' => BuildLists::DependentPackagesJob,
           'args'  => [@build_list.id,

app/jobs/abf_worker/rpm_worker_observer.rb

@@ -44,7 +44,7 @@ module AbfWorker
           when COMPLETED
             subject.build_success
             if subject.can_auto_publish? && subject.can_publish?
-              subject.now_publish
+              subject.publish
             elsif subject.auto_publish_into_testing? && subject.can_publish_into_testing?
               subject.publish_into_testing
             end

app/jobs/build_lists/clear_builder_on_stale_build_lists_job.rb

@@ -1,14 +0,0 @@
-module BuildLists
-  class ClearBuilderOnStaleBuildListsJob < BaseActiveRecordJob
-    include Sidekiq::Worker
-    sidekiq_options :queue => :low
-
-    def perform_with_ar_connection
-      BuildList.transaction do
-        BuildList.where(["updated_at < ?", 120.seconds.ago]).where(status: BuildList::BUILD_PENDING).where.not(builder: nil).find_each(batch_size: 50) do |bl|
-          bl.update_column(builder_id: nil)
-        end
-      end
-    end
-  end
-end

app/jobs/destroy_files_from_file_store_job.rb

@@ -0,0 +1,13 @@
+class DestroyFilesFromFileStoreJob
+  include Sidekiq::Worker
+
+  sidekiq_options :queue => :low
+
+  def perform(sha1)
+    file = FileStoreService::File.new
+    sha1.each do |hash|
+      file.sha1 = hash
+      file.destroy
+    end
+  end
+end

app/models/build_list.rb

@@ -281,8 +281,6 @@ class BuildList < ActiveRecord::Base
     end
   end
 
-  later :publish, queue: :middle
-
   HUMAN_CONTAINER_STATUSES = { WAITING_FOR_RESPONSE => :waiting_for_publish,
                                BUILD_PUBLISHED => :container_published,
                                BUILD_PUBLISH => :container_publish,

app/models/concerns/file_store_clean.rb

@@ -1,13 +1,8 @@
 module FileStoreClean
   extend ActiveSupport::Concern
 
-  included do
-    later :destroy, queue: :middle
-    later :later_destroy_files_from_file_store, queue: :middle
-  end
-
   def destroy
-    destroy_files_from_file_store if Rails.env.production?
+    DestroyFilesFromFileStoreJob.perform_async(sha1_of_file_store_files) if Rails.env.production?
     super
   end
 
@@ -15,15 +10,4 @@ module FileStoreClean
     raise NotImplementedError, "You should implement this method"
   end
 
-  def destroy_files_from_file_store(args = sha1_of_file_store_files)
-    files = *args
-    files.each do |sha1|
-      FileStoreService::File.new(sha1: sha1).destroy
-    end
-  end
-
-  def later_destroy_files_from_file_store(args)
-    destroy_files_from_file_store(args)
-  end
-
 end

app/models/concerns/regeneration_status.rb

@@ -37,7 +37,7 @@ module RegenerationStatus
     def cleanup_file_store
       old_log_sha1 = last_regenerated_log_sha1_was
       if old_log_sha1.present? && old_log_sha1 != last_regenerated_log_sha1
-        later_destroy_files_from_file_store([old_log_sha1])
+        DestroyFilesFromFileStoreJob.perform_async([old_log_sha1])
       end
     end
   end

app/models/mass_build.rb

@@ -193,7 +193,7 @@ class MassBuild < ActiveRecord::Base
     builds = build_lists.where(status: statuses)
     builds.update_all(publisher_id: user.id)
     builds.find_each(batch_size: 50) do |bl|
-      bl.now_publish if bl.can_publish?
+      bl.publish if bl.can_publish?
     end
   end
 

app/models/platform.rb

@@ -92,7 +92,7 @@ class Platform < ActiveRecord::Base
   after_update :freeze_platform_and_update_repos
   after_update :update_owner_relation
 
-  after_commit  -> { symlink_directory unless hidden? }, on: :create
+  #after_commit  -> { symlink_directory unless hidden? }, on: :create
 
   accepts_nested_attributes_for :platform_arch_settings, allow_destroy: true
 
@@ -212,18 +212,13 @@ class Platform < ActiveRecord::Base
     end
   end
 
-  def symlink_directory
-    # umount_directory_for_rsync # TODO ignore errors
-    Arch.all.each do |arch|
-      str = "country=Russian Federation,city=Moscow,latitude=52.18,longitude=48.88,bw=1GB,version=2011,arch=#{arch.name},type=distrib,url=#{public_downloads_url}\n"
-      File.open(File.join(path, "#{name}.#{arch.name}.list"), 'w') {|f| f.write(str) }
-    end
-  end
-  later :symlink_directory, queue: :middle
-
-  def remove_symlink_directory
-    #system("rm -Rf #{symlink_path}")
-  end
+  #def symlink_directory
+  #   umount_directory_for_rsync # TODO ignore errors
+  #  Arch.all.each do |arch|
+  #    str = "country=Russian Federation,city=Moscow,latitude=52.18,longitude=48.88,bw=1GB,version=2011,arch=#{arch.name},type=distrib,url=#{public_downloads_url}\n"
+  #    File.open(File.join(path, "#{name}.#{arch.name}.list"), 'w') {|f| f.write(str) }
+  #  end
+  #end
 
   def update_owner_relation
     if owner_id_was != owner_id

app/views/projects/build_lists/index.html.slim

@@ -30,6 +30,7 @@
                 th= t('activerecord.attributes.build_list.arch_short')
                 th= t('activerecord.attributes.build_list.user')
                 th= t('activerecord.attributes.build_list.hostname')
+                th= t('activerecord.attributes.build_list.updated_at')
             tbody
               tr[ ng-repeat = 'bl in build_lists'
                   class     = '{{::bl.status_color}}'
@@ -83,6 +84,10 @@
                   a[ ng-href = '{{::bl.user.url}}' ] {{::bl.user.fullname}}
 
                 / hostname
-                td[] {{::bl.hostname}}
+                td
+                  | {{::bl.hostname}}
+
+                td
+                  | {{::bl.updated_at}}
         rd-widget-footer ng-show="total_items > per_page"
           == angularjs_paginate( per_page: 'per_page' )

config/initializers/1_sidekiq.rb

@@ -9,3 +9,4 @@ Sidekiq.configure_server do |config|
     Sidekiq::Scheduler.reload_schedule!
   end
 end
+

config/schedule.yml

@@ -19,12 +19,12 @@ clean_api_defender_statistics:
   queue: low
   description: 'Cleans ApiDefender statistics'
 
-clean_build_list_buildroot:
-  every:
-    - '1h'
-  class: 'BuildLists::CleanBuildrootJob'
-  queue: middle
-  description: 'Cleans RPM buildroot from BuildList'
+#clean_build_list_buildroot:
+#  every:
+#    - '1h'
+#  class: 'BuildLists::CleanBuildrootJob'
+#  queue: middle
+#  description: 'Cleans RPM buildroot from BuildList'
 
 run_extra_mass_builds:
   every:
@@ -33,13 +33,6 @@ run_extra_mass_builds:
   queue: low
   description: 'Run mass builds with relations'
 
-#clear_builder_on_stale_buildlists:
-#  every:
-#    - '2m'
-#  class: 'BuildLists::ClearBuilderOnStaleBuildListsJob'
-#  queue: low
-#  description: 'Clear builder on build lists which are still pending'
-
 remove_outdated_items:
   cron: '0 3 * * *'
   class: 'RemoveOutdatedItemsJob'