From 5b734e3006909283f6cf534e6b43affca8aae0f1 Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Tue, 15 Apr 2014 01:32:25 +0400
Subject: [PATCH 01/14] #376: added NodeInstruction

---
 app/admin/node_instructions.rb                | 68 ++++++++++++++
 app/admin/shared/node_instruction.rb          | 12 +++
 app/models/node_instruction.rb                | 45 +++++++++
 config/application.yml.sample                 |  1 +
 ...20140414195426_create_node_instructions.rb | 12 +++
 db/schema.rb                                  | 91 +++++++++++--------
 6 files changed, 189 insertions(+), 40 deletions(-)
 create mode 100644 app/admin/node_instructions.rb
 create mode 100644 app/admin/shared/node_instruction.rb
 create mode 100644 app/models/node_instruction.rb
 create mode 100644 db/migrate/20140414195426_create_node_instructions.rb

diff --git a/app/admin/node_instructions.rb b/app/admin/node_instructions.rb
new file mode 100644
index 000000000..0f2e0fab5
--- /dev/null
+++ b/app/admin/node_instructions.rb
@@ -0,0 +1,68 @@
+ActiveAdmin.register NodeInstruction do
+
+  menu priority: 3
+
+  controller do
+    def scoped_collection
+      NodeInstruction.includes(:user)
+    end
+  end
+
+  filter :user_uname, as: :string
+  filter :status,     as: :select, collection: NodeInstruction::STATUSES
+  filter :updated_at
+
+  index do
+    column :id
+    column :user
+
+    column(:status, sortable: :status) do |ni|
+      status_tag(ni.status, status_color(ni))
+    end
+    column :updated_at
+
+    default_actions
+  end
+
+  form do |f|
+    f.inputs do
+      f.input :user,        as: :select, include_blank: false, collection: User.system.map { |u| [u.uname, u.id]  }
+      f.input :status,      as: :select, include_blank: false, collection: NodeInstruction::STATUSES
+      f.input :instruction, as: :text
+    end
+    f.actions
+  end
+
+  show do
+    attributes_table do
+      row :id
+      row :user
+      row(:status, sortable: :status) do |ni|
+        status_tag(ni.status, status_color(ni))
+      end
+      row :created_at
+      row :updated_at
+      row :instruction
+      row :output
+    end
+  end
+
+  sidebar 'Actions', only: :show do
+
+    %w(disable ready check restart fail).each do |state|
+      div do
+        link_to state.humanize, force_admin_node_instruction_path(resource, state: state), method: :patch
+      end if resource.send("can_#{state}?")
+    end
+
+  end
+
+  member_action :force, method: :patch do
+    if NodeInstruction::STATUSES.include?(params[:state])
+      resource.send(params[:state])
+      flash[:info] = 'Action added to queue successfully'
+    end
+    redirect_to admin_node_instruction_path(resource)
+  end
+
+end
diff --git a/app/admin/shared/node_instruction.rb b/app/admin/shared/node_instruction.rb
new file mode 100644
index 000000000..c911b8415
--- /dev/null
+++ b/app/admin/shared/node_instruction.rb
@@ -0,0 +1,12 @@
+def status_color(ni)
+  case
+  when ni.ready?
+    :green
+  when ni.disabled?
+    nil
+  when ni.failed?
+    :red
+  else
+    :orange
+  end
+end
\ No newline at end of file
diff --git a/app/models/node_instruction.rb b/app/models/node_instruction.rb
new file mode 100644
index 000000000..2fcfc2d7a
--- /dev/null
+++ b/app/models/node_instruction.rb
@@ -0,0 +1,45 @@
+class NodeInstruction < ActiveRecord::Base
+  STATUSES = [
+    DISABLED    = 'disabled',
+    READY       = 'ready',
+    CHECKING    = 'checking',
+    RESTARTING  = 'restarting',
+    FAILED      = 'failed'
+  ]
+
+  belongs_to :user
+
+  attr_encrypted :instruction, key: APP_CONFIG['keys']['node_instruction_secret_key']
+
+  validates :user,        presence: true
+  validates :instruction, presence: true, length: { maximum: 10000 }
+  validates :status,      presence: true
+  validate  -> {
+    errors.add(:status, 'Can be only single active instruction for each node') if !disabled? && NodeInstruction.where('id != ?', id.to_i).where(user_id: user_id, status: STATUSES - [DISABLED]).exists?
+  }
+
+  attr_accessible :instruction, :user_id, :output, :status
+
+  state_machine :status, initial: :ready do
+    event :ready do
+      transition %i(ready restarting disabled failed checking) => :ready
+    end
+
+    event :disable do
+      transition ready: :disabled
+    end
+
+    event :check do
+      transition ready: :checking
+    end
+
+    event :restart do
+      transition checking: :restarting
+    end
+
+    event :fail do
+      transition %i(restarting checking) => :failed
+    end
+  end
+
+end
diff --git a/config/application.yml.sample b/config/application.yml.sample
index 336c5e94c..58a80ea86 100644
--- a/config/application.yml.sample
+++ b/config/application.yml.sample
@@ -14,6 +14,7 @@ common: &common
       port: 6379
   keys:
     key_pair_secret_key: 'key_pair_secret_key'
+    node_instruction_secret_key: 'node_instruction_secret_key'
     airbrake_api_key: 'airbrake_api_key'
     logentries_key: 'logentries_key'
     secret_token: 'secret_token'
diff --git a/db/migrate/20140414195426_create_node_instructions.rb b/db/migrate/20140414195426_create_node_instructions.rb
new file mode 100644
index 000000000..36a046b17
--- /dev/null
+++ b/db/migrate/20140414195426_create_node_instructions.rb
@@ -0,0 +1,12 @@
+class CreateNodeInstructions < ActiveRecord::Migration
+  def change
+    create_table :node_instructions do |t|
+      t.integer :user_id,             null: false
+      t.text :encrypted_instruction,  null: false
+      t.text :output
+      t.string :status
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 59c1f068d..e9d8dee51 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20140407181059) do
+ActiveRecord::Schema.define(version: 20140414195426) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -287,6 +287,56 @@ ActiveRecord::Schema.define(version: 20140407181059) do
     t.boolean  "increase_release_tag",  default: false, null: false
   end
 
+  create_table "users", force: true do |t|
+    t.string   "name"
+    t.string   "email",                               default: "",   null: false
+    t.string   "encrypted_password",      limit: 128, default: "",   null: false
+    t.string   "reset_password_token"
+    t.datetime "reset_password_sent_at"
+    t.datetime "remember_created_at"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.text     "ssh_key"
+    t.string   "uname"
+    t.string   "role"
+    t.string   "language",                            default: "en"
+    t.integer  "own_projects_count",                  default: 0,    null: false
+    t.text     "professional_experience"
+    t.string   "site"
+    t.string   "company"
+    t.string   "location"
+    t.string   "avatar_file_name"
+    t.string   "avatar_content_type"
+    t.integer  "avatar_file_size"
+    t.datetime "avatar_updated_at"
+    t.integer  "failed_attempts",                     default: 0
+    t.string   "unlock_token"
+    t.datetime "locked_at"
+    t.string   "confirmation_token"
+    t.datetime "confirmed_at"
+    t.datetime "confirmation_sent_at"
+    t.string   "authentication_token"
+    t.integer  "build_priority",                      default: 50
+    t.boolean  "sound_notifications",                 default: true
+    t.index ["authentication_token"], :name => "index_users_on_authentication_token"
+    t.index ["confirmation_token"], :name => "index_users_on_confirmation_token", :unique => true
+    t.index ["email"], :name => "index_users_on_email", :unique => true
+    t.index ["reset_password_token"], :name => "index_users_on_reset_password_token", :unique => true
+    t.index ["uname"], :name => "index_users_on_uname", :unique => true
+    t.index ["unlock_token"], :name => "index_users_on_unlock_token", :unique => true
+  end
+
+  create_table "node_instructions", force: true do |t|
+    t.integer  "user_id",               null: false
+    t.text     "encrypted_instruction", null: false
+    t.text     "output"
+    t.string   "status"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.index ["user_id"], :name => "fk__node_instructions_user_id"
+    t.foreign_key ["user_id"], "users", ["id"], :on_update => :no_action, :on_delete => :no_action, :name => "fk_node_instructions_user_id"
+  end
+
   create_table "platform_arch_settings", force: true do |t|
     t.integer  "platform_id", null: false
     t.integer  "arch_id",     null: false
@@ -530,43 +580,4 @@ ActiveRecord::Schema.define(version: 20140407181059) do
     t.index ["subject_id", "subject_type"], :name => "index_tokens_on_subject_id_and_subject_type"
   end
 
-  create_table "users", force: true do |t|
-    t.string   "name"
-    t.string   "email",                               default: "",   null: false
-    t.string   "encrypted_password",      limit: 128, default: "",   null: false
-    t.string   "reset_password_token"
-    t.datetime "reset_password_sent_at"
-    t.datetime "remember_created_at"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.text     "ssh_key"
-    t.string   "uname"
-    t.string   "role"
-    t.string   "language",                            default: "en"
-    t.integer  "own_projects_count",                  default: 0,    null: false
-    t.text     "professional_experience"
-    t.string   "site"
-    t.string   "company"
-    t.string   "location"
-    t.string   "avatar_file_name"
-    t.string   "avatar_content_type"
-    t.integer  "avatar_file_size"
-    t.datetime "avatar_updated_at"
-    t.integer  "failed_attempts",                     default: 0
-    t.string   "unlock_token"
-    t.datetime "locked_at"
-    t.string   "confirmation_token"
-    t.datetime "confirmed_at"
-    t.datetime "confirmation_sent_at"
-    t.string   "authentication_token"
-    t.integer  "build_priority",                      default: 50
-    t.boolean  "sound_notifications",                 default: true
-    t.index ["authentication_token"], :name => "index_users_on_authentication_token"
-    t.index ["confirmation_token"], :name => "index_users_on_confirmation_token", :unique => true
-    t.index ["email"], :name => "index_users_on_email", :unique => true
-    t.index ["reset_password_token"], :name => "index_users_on_reset_password_token", :unique => true
-    t.index ["uname"], :name => "index_users_on_uname", :unique => true
-    t.index ["unlock_token"], :name => "index_users_on_unlock_token", :unique => true
-  end
-
 end

From 552e36cf72702abfbf04aa878f54bd23fbdd7aaf Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Tue, 15 Apr 2014 22:08:43 +0400
Subject: [PATCH 02/14] #376: added monitoring tool

---
 app/admin/node_instructions.rb                |  2 +-
 app/controllers/api/v1/jobs_controller.rb     | 27 ++++++++++---------
 app/jobs/restart_nodes_job.rb                 | 10 +++++++
 app/models/node_instruction.rb                | 11 +++-----
 .../projects/build_lists/show.json.jbuilder   |  2 +-
 config/resque_schedule.yml                    |  9 ++++++-
 6 files changed, 38 insertions(+), 23 deletions(-)
 create mode 100644 app/jobs/restart_nodes_job.rb

diff --git a/app/admin/node_instructions.rb b/app/admin/node_instructions.rb
index 0f2e0fab5..bfa18ea15 100644
--- a/app/admin/node_instructions.rb
+++ b/app/admin/node_instructions.rb
@@ -49,7 +49,7 @@ ActiveAdmin.register NodeInstruction do
 
   sidebar 'Actions', only: :show do
 
-    %w(disable ready check restart fail).each do |state|
+    %w(disable ready restart fail).each do |state|
       div do
         link_to state.humanize, force_admin_node_instruction_path(resource, state: state), method: :patch
       end if resource.send("can_#{state}?")
diff --git a/app/controllers/api/v1/jobs_controller.rb b/app/controllers/api/v1/jobs_controller.rb
index 73af36564..e634efc08 100644
--- a/app/controllers/api/v1/jobs_controller.rb
+++ b/app/controllers/api/v1/jobs_controller.rb
@@ -9,7 +9,9 @@ class Api::V1::JobsController < Api::V1::BaseController
   def shift
 
     @build_list = BuildList.next_build if current_user.system?
-    unless @build_list
+    if @build_list
+      set_builder
+    else
       platform_ids = Platform.where(name: params[:platforms].split(',')).pluck(:id) if params[:platforms].present?
       arch_ids = Arch.where(name: params[:arches].split(',')).pluck(:id) if params[:arches].present?
       build_lists = BuildList.for_status(BuildList::BUILD_PENDING).scoped_to_arch(arch_ids).
@@ -19,36 +21,29 @@ class Api::V1::JobsController < Api::V1::BaseController
       ActiveRecord::Base.transaction do
         if current_user.system?
           @build_list ||= build_lists.external_nodes(:everything).first
-          @build_list.touch if @build_list
         else
           @build_list = build_lists.external_nodes(:owned).for_user(current_user).first
           @build_list ||= build_lists.external_nodes(:everything).
             accessible_by(current_ability, :everything).readonly(false).first
-
-          if @build_list
-            @build_list.builder = current_user
-            @build_list.save
-          end
         end
+        set_builder
       end
     end
 
-
     job = {
       worker_queue: @build_list.worker_queue_with_priority(false),
       worker_class: @build_list.worker_queue_class,
       :worker_args  => [@build_list.abf_worker_args]
     } if @build_list
-
     render json: { job: job }.to_json
   end
 
   def statistics
     if params[:uid].present?
       RpmBuildNode.create(
-        :id           => params[:uid],
-        :user_id      => current_user.id,
-        :system       => current_user.system?,
+        id:           params[:uid],
+        user_id:      current_user.id,
+        system:       current_user.system?,
         worker_count: params[:worker_count],
         busy_workers: params[:busy_workers]
       ) rescue nil
@@ -86,4 +81,12 @@ class Api::V1::JobsController < Api::V1::BaseController
     end
   end
 
+  protected
+
+  def set_builder
+    return unless @build_list
+    @build_list.builder = current_user
+    @build_list.save
+  end
+
 end
diff --git a/app/jobs/restart_nodes_job.rb b/app/jobs/restart_nodes_job.rb
new file mode 100644
index 000000000..e158cf3a0
--- /dev/null
+++ b/app/jobs/restart_nodes_job.rb
@@ -0,0 +1,10 @@
+class RestartNodesJob
+  @queue = :hook
+
+  def self.perform
+    available_nodes = RpmBuildNode.all.map{ |n| n.user_id }.compact.uniq
+    NodeInstruction.where(status: NodeInstruction::READY).
+      where('user_id NOT IN (?)', available_nodes).find_each(&:restart)
+  end
+
+end
diff --git a/app/models/node_instruction.rb b/app/models/node_instruction.rb
index 2fcfc2d7a..24ac3509b 100644
--- a/app/models/node_instruction.rb
+++ b/app/models/node_instruction.rb
@@ -2,7 +2,6 @@ class NodeInstruction < ActiveRecord::Base
   STATUSES = [
     DISABLED    = 'disabled',
     READY       = 'ready',
-    CHECKING    = 'checking',
     RESTARTING  = 'restarting',
     FAILED      = 'failed'
   ]
@@ -22,23 +21,19 @@ class NodeInstruction < ActiveRecord::Base
 
   state_machine :status, initial: :ready do
     event :ready do
-      transition %i(ready restarting disabled failed checking) => :ready
+      transition %i(ready restarting disabled failed) => :ready
     end
 
     event :disable do
       transition ready: :disabled
     end
 
-    event :check do
-      transition ready: :checking
-    end
-
     event :restart do
-      transition checking: :restarting
+      transition ready: :restarting
     end
 
     event :fail do
-      transition %i(restarting checking) => :failed
+      transition restarting: :failed
     end
   end
 
diff --git a/app/views/projects/build_lists/show.json.jbuilder b/app/views/projects/build_lists/show.json.jbuilder
index 03a1a6d98..1f723f807 100644
--- a/app/views/projects/build_lists/show.json.jbuilder
+++ b/app/views/projects/build_lists/show.json.jbuilder
@@ -31,7 +31,7 @@ json.build_list do
   json.builder do
     json.fullname @build_list.builder.try(:fullname)
     json.path user_path(@build_list.builder)
-  end if @build_list.builder
+  end if @build_list.builder && (!@build_list.builder.system? || current_user.admin?)
 
   json.advisory do
     json.(@build_list.advisory, :description, :advisory_id)
diff --git a/config/resque_schedule.yml b/config/resque_schedule.yml
index 44b2dfac2..fbdade941 100644
--- a/config/resque_schedule.yml
+++ b/config/resque_schedule.yml
@@ -24,4 +24,11 @@ clean_api_defender_statistics:
     - '1d'
   class: 'CleanApiDefenderStatisticsJob'
   queue: clone_build
-  description: 'Cleans ApiDefender statistics'
\ No newline at end of file
+  description: 'Cleans ApiDefender statistics'
+
+restart_nodes:
+  every:
+    - '5m'
+  class: 'RestartNodesJob'
+  queue: hook
+  description: 'Restarts unavailable nodes'
\ No newline at end of file

From 44ac1056abd7bae00ebb4f3df6ba6930389cb088 Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Tue, 15 Apr 2014 22:41:06 +0400
Subject: [PATCH 03/14] #376: updated priority of queues

---
 app/jobs/build_lists_publish_task_manager_job.rb |  2 +-
 app/jobs/build_lists_queues_monitoring_job.rb    |  2 +-
 app/jobs/clean_api_defender_statistics_job.rb    |  2 +-
 app/jobs/clean_rpm_build_node_job.rb             |  2 +-
 app/jobs/restart_nodes_job.rb                    |  2 +-
 app/models/build_list.rb                         |  6 +++---
 app/models/concerns/file_store_clean.rb          |  4 ++--
 app/models/mass_build.rb                         |  8 ++++----
 app/models/platform.rb                           |  4 ++--
 app/models/project.rb                            |  2 +-
 app/models/pull_request.rb                       |  2 +-
 app/models/repository.rb                         |  9 ++++-----
 config/resque_schedule.yml                       | 10 +++++-----
 lib/recipes/resque.rb                            |  4 +++-
 14 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/app/jobs/build_lists_publish_task_manager_job.rb b/app/jobs/build_lists_publish_task_manager_job.rb
index 1fc5dca83..4a615dfec 100644
--- a/app/jobs/build_lists_publish_task_manager_job.rb
+++ b/app/jobs/build_lists_publish_task_manager_job.rb
@@ -1,5 +1,5 @@
 class BuildListsPublishTaskManagerJob
-  @queue = :hook
+  @queue = :middle
 
   def self.perform
     AbfWorker::BuildListsPublishTaskManager.new.run
diff --git a/app/jobs/build_lists_queues_monitoring_job.rb b/app/jobs/build_lists_queues_monitoring_job.rb
index 08b4db60a..ecea75cf0 100644
--- a/app/jobs/build_lists_queues_monitoring_job.rb
+++ b/app/jobs/build_lists_queues_monitoring_job.rb
@@ -1,5 +1,5 @@
 class BuildListsQueuesMonitoringJob
-  @queue = :hook
+  @queue = :middle
 
   def self.perform
     Redis.current.smembers('resque:queues').each do |key|
diff --git a/app/jobs/clean_api_defender_statistics_job.rb b/app/jobs/clean_api_defender_statistics_job.rb
index 6fea6e786..9f8a6d976 100644
--- a/app/jobs/clean_api_defender_statistics_job.rb
+++ b/app/jobs/clean_api_defender_statistics_job.rb
@@ -1,5 +1,5 @@
 class CleanApiDefenderStatisticsJob
-  @queue = :clone_build
+  @queue = :low
 
   def self.perform
     deadline = Date.today - 1.month
diff --git a/app/jobs/clean_rpm_build_node_job.rb b/app/jobs/clean_rpm_build_node_job.rb
index 67c2f212e..b31e4ec25 100644
--- a/app/jobs/clean_rpm_build_node_job.rb
+++ b/app/jobs/clean_rpm_build_node_job.rb
@@ -1,5 +1,5 @@
 class CleanRpmBuildNodeJob
-  @queue = :hook
+  @queue = :middle
 
   def self.perform
     RpmBuildNode.all.each do |n|
diff --git a/app/jobs/restart_nodes_job.rb b/app/jobs/restart_nodes_job.rb
index e158cf3a0..020062162 100644
--- a/app/jobs/restart_nodes_job.rb
+++ b/app/jobs/restart_nodes_job.rb
@@ -1,5 +1,5 @@
 class RestartNodesJob
-  @queue = :hook
+  @queue = :low
 
   def self.perform
     available_nodes = RpmBuildNode.all.map{ |n| n.user_id }.compact.uniq
diff --git a/app/models/build_list.rb b/app/models/build_list.rb
index 54fe6ca5d..aafbb8ae5 100644
--- a/app/models/build_list.rb
+++ b/app/models/build_list.rb
@@ -259,8 +259,8 @@ class BuildList < ActiveRecord::Base
     end
   end
 
-  later :publish, queue: :clone_build
-  later :add_job_to_abf_worker_queue, queue: :clone_build
+  later :publish, queue: :middle
+  later :add_job_to_abf_worker_queue, queue: :middle
 
   HUMAN_CONTAINER_STATUSES = { WAITING_FOR_RESPONSE => :waiting_for_publish,
                                BUILD_PUBLISHED => :container_published,
@@ -572,7 +572,7 @@ class BuildList < ActiveRecord::Base
   def delayed_add_job_to_abf_worker_queue(*args)
     restart_job if status == BUILD_PENDING
   end
-  later :delayed_add_job_to_abf_worker_queue, delay: 60, queue: :clone_build
+  later :delayed_add_job_to_abf_worker_queue, delay: 60, queue: :middle
 
   protected
 
diff --git a/app/models/concerns/file_store_clean.rb b/app/models/concerns/file_store_clean.rb
index f4fe70c42..c04d934a1 100644
--- a/app/models/concerns/file_store_clean.rb
+++ b/app/models/concerns/file_store_clean.rb
@@ -6,7 +6,7 @@ module FileStoreClean
       destroy_files_from_file_store if Rails.env.production?
       super
     end
-    later :destroy, queue: :clone_build
+    later :destroy, queue: :middle
 
     def sha1_of_file_store_files
       raise NotImplementedError, "You should implement this method"
@@ -31,7 +31,7 @@ module FileStoreClean
     def later_destroy_files_from_file_store(args)
       destroy_files_from_file_store(args)
     end
-    later :later_destroy_files_from_file_store, queue: :clone_build
+    later :later_destroy_files_from_file_store, queue: :middle
   end
 
   def self.file_exist_on_file_store?(sha1)
diff --git a/app/models/mass_build.rb b/app/models/mass_build.rb
index 998d2faea..a8f720f05 100644
--- a/app/models/mass_build.rb
+++ b/app/models/mass_build.rb
@@ -59,7 +59,7 @@ class MassBuild < ActiveRecord::Base
       end
     end
   end
-  later :build_all, queue: :clone_build
+  later :build_all, queue: :low
 
   def generate_failed_builds_list
     generate_list BuildList::BUILD_ERROR
@@ -75,17 +75,17 @@ class MassBuild < ActiveRecord::Base
       bl.cancel
     end
   end
-  later :cancel_all, queue: :clone_build
+  later :cancel_all, queue: :low
 
   def publish_success_builds(user)
     publish user, BuildList::SUCCESS, BuildList::FAILED_PUBLISH
   end
-  later :publish_success_builds, queue: :clone_build
+  later :publish_success_builds, queue: :low
 
   def publish_test_failed_builds(user)
     publish user, BuildList::TESTS_FAILED
   end
-  later :publish_test_failed_builds, queue: :clone_build
+  later :publish_test_failed_builds, queue: :low
 
   COUNT_STATUSES.each do |stat|
     stat_count = "#{stat}_count"
diff --git a/app/models/platform.rb b/app/models/platform.rb
index 1f043350e..ee6a6fd40 100644
--- a/app/models/platform.rb
+++ b/app/models/platform.rb
@@ -205,7 +205,7 @@ class Platform < ActiveRecord::Base
   def destroy
     with_skip {super} # avoid cascade XML RPC requests
   end
-  later :destroy, queue: :clone_build
+  later :destroy, queue: :low
 
   def default_host
     EventLog.current_controller.request.host_with_port rescue ::Rosa::Application.config.action_mailer.default_url_options[:host]
@@ -274,7 +274,7 @@ class Platform < ActiveRecord::Base
     def fs_clone(old_name = parent.name, new_name = name)
       FileUtils.cp_r "#{parent.path}/repository", path
     end
-    later :fs_clone, queue: :clone_build
+    later :fs_clone, queue: :low
 
     def freeze_platform_and_update_repos
       if released_changed? && released == true
diff --git a/app/models/project.rb b/app/models/project.rb
index 1f9f846d7..db2351fec 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -417,7 +417,7 @@ class Project < ActiveRecord::Base
     PullRequest.where(from_project_id: id).each{ |p| p.update_relations(old_name) }
     pull_requests.where('from_project_id != to_project_id').each(&:update_relations)
   end
-  later :update_path_to_project, queue: :clone_build
+  later :update_path_to_project, queue: :middle
 
   def check_default_branch
     if self.repo.branches.count > 0 && self.repo.branches.map(&:name).exclude?(self.default_branch)
diff --git a/app/models/pull_request.rb b/app/models/pull_request.rb
index 1e98c5629..b482a2e97 100644
--- a/app/models/pull_request.rb
+++ b/app/models/pull_request.rb
@@ -57,7 +57,7 @@ class PullRequest < ActiveRecord::Base
       system 'git', 'remote', 'set-url', 'head', from_project.path if cross_pull?
     end
   end
-  later :update_relations, queue: :clone_build
+  later :update_relations, queue: :middle
 
   def cross_pull?
     from_project_id != to_project_id
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 561d99902..e42ce367c 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -59,8 +59,7 @@ class Repository < ActiveRecord::Base
       from.projects.find_each {|p| self.projects << p}
     end
   end
-  later :clone_relations, loner: true, queue: :clone_build
-
+  later :clone_relations, loner: true, queue: :low
   def add_projects(list, user)
     current_ability = Ability.new(user)
     list.lines.each do |line|
@@ -75,7 +74,7 @@ class Repository < ActiveRecord::Base
       end
     end
   end
-  later :add_projects, queue: :clone_build
+  later :add_projects, queue: :middle
 
   def remove_projects(list)
     list.lines.each do |name|
@@ -87,7 +86,7 @@ class Repository < ActiveRecord::Base
       end
     end
   end
-  later :remove_projects, queue: :clone_build
+  later :remove_projects, queue: :middle
 
   def full_clone(attrs = {})
     base_clone(attrs).tap do |c|
@@ -148,7 +147,7 @@ class Repository < ActiveRecord::Base
   def destroy
     with_skip {super} # avoid cascade XML RPC requests
   end
-  later :destroy, queue: :clone_build
+  later :destroy, queue: :low
 
   def self.custom_sort(repos)
     repos.select{ |r| SORT.keys.include?(r.name) }.sort{ |a,b| SORT[a.name] <=>  SORT[b.name] } | repos.sort_by(&:name)
diff --git a/config/resque_schedule.yml b/config/resque_schedule.yml
index fbdade941..79565cbb0 100644
--- a/config/resque_schedule.yml
+++ b/config/resque_schedule.yml
@@ -2,33 +2,33 @@ clean_rpm_build_nodes:
   every:
     - '1m'
   class: 'CleanRpmBuildNodeJob'
-  queue: hook
+  queue: middle
   description: 'Cleans RPM build nodes'
 
 build_lists_publish_task_manager:
   every:
     - '3m'
   class: 'BuildListsPublishTaskManagerJob'
-  queue: hook
+  queue: middle
   description: 'Creates tasks for publishing'
 
 build_lists_queues_monitoring:
   every:
     - '1m'
   class: 'BuildListsQueuesMonitoringJob'
-  queue: hook
+  queue: middle
   description: 'Monitoring for "user/mass-build" queues'
 
 clean_api_defender_statistics:
   every:
     - '1d'
   class: 'CleanApiDefenderStatisticsJob'
-  queue: clone_build
+  queue: low
   description: 'Cleans ApiDefender statistics'
 
 restart_nodes:
   every:
     - '5m'
   class: 'RestartNodesJob'
-  queue: hook
+  queue: low
   description: 'Restarts unavailable nodes'
\ No newline at end of file
diff --git a/lib/recipes/resque.rb b/lib/recipes/resque.rb
index 3ffbb70dd..dbf72ce71 100644
--- a/lib/recipes/resque.rb
+++ b/lib/recipes/resque.rb
@@ -32,9 +32,11 @@ Capistrano::Configuration.instance(:must_exist).load do
         :fork_import,
         :hook,
         :clone_build,
+        :middle,
         :notification
       ].join(',')
-      run "cd #{fetch :current_path} && COUNT=#{workers_count} QUEUE=#{queue} #{rails_env} BACKGROUND=yes bundle exec rake resque:workers"
+      run "cd #{fetch :current_path} && COUNT=#{workers_count - 1} QUEUE=#{queue} INTERVAL=0.1 #{rails_env} BACKGROUND=yes bundle exec rake resque:workers"
+      run "cd #{fetch :current_path} && COUNT=1 QUEUE=low #{rails_env} BACKGROUND=yes bundle exec rake resque:workers"
     end
 
     def remote_file_exists?(full_path)

From aa73cb361fa96a5b3496d6c913dd275597146eba Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Tue, 15 Apr 2014 23:15:52 +0400
Subject: [PATCH 04/14] #376: added ability to restart node

---
 app/models/node_instruction.rb | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/app/models/node_instruction.rb b/app/models/node_instruction.rb
index 24ac3509b..aa001f865 100644
--- a/app/models/node_instruction.rb
+++ b/app/models/node_instruction.rb
@@ -20,6 +20,9 @@ class NodeInstruction < ActiveRecord::Base
   attr_accessible :instruction, :user_id, :output, :status
 
   state_machine :status, initial: :ready do
+
+    after_transition on: :restart, do: :perform_restart
+
     event :ready do
       transition %i(ready restarting disabled failed) => :ready
     end
@@ -37,4 +40,26 @@ class NodeInstruction < ActiveRecord::Base
     end
   end
 
+  def perform_restart
+    success = false
+    output  = ''
+    instruction.lines.each do |command|
+      next if command.blank?
+      command.chomp!; command.strip!
+      output << %x[ #{command} 2>&1 ]
+      success = $?.success?
+    end
+
+    build_lists = BuildList.where(builder_id: user_id, external_nodes: [nil, '']).
+      for_status(BuildList::BUILD_STARTED)
+    
+    build_lists.find_each do |bl|
+      bl.update_column(:status, BuildList::BUILD_PENDING)
+      bl.restart_job
+    end
+
+    success ? ready : fail
+  end
+  later :perform_restart, queue: :low
+
 end

From b37ada2ecd2b7c34b9fb6de0c1fed368378ccb60 Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Wed, 16 Apr 2014 01:26:12 +0400
Subject: [PATCH 05/14] #376: some fixes

---
 app/admin/node_instructions.rb | 2 +-
 app/models/node_instruction.rb | 4 ++--
 app/models/project.rb          | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/admin/node_instructions.rb b/app/admin/node_instructions.rb
index bfa18ea15..886eb0052 100644
--- a/app/admin/node_instructions.rb
+++ b/app/admin/node_instructions.rb
@@ -49,7 +49,7 @@ ActiveAdmin.register NodeInstruction do
 
   sidebar 'Actions', only: :show do
 
-    %w(disable ready restart fail).each do |state|
+    %w(disable ready restart restart_failed).each do |state|
       div do
         link_to state.humanize, force_admin_node_instruction_path(resource, state: state), method: :patch
       end if resource.send("can_#{state}?")
diff --git a/app/models/node_instruction.rb b/app/models/node_instruction.rb
index aa001f865..501d13924 100644
--- a/app/models/node_instruction.rb
+++ b/app/models/node_instruction.rb
@@ -35,7 +35,7 @@ class NodeInstruction < ActiveRecord::Base
       transition ready: :restarting
     end
 
-    event :fail do
+    event :restart_failed do
       transition restarting: :failed
     end
   end
@@ -58,7 +58,7 @@ class NodeInstruction < ActiveRecord::Base
       bl.restart_job
     end
 
-    success ? ready : fail
+    success ? ready : restart_failed
   end
   later :perform_restart, queue: :low
 
diff --git a/app/models/project.rb b/app/models/project.rb
index db2351fec..4b42d7e7a 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -109,7 +109,7 @@ class Project < ActiveRecord::Base
   end
 
   def init_mass_import
-    Project.perform_later :clone_build, :run_mass_import, url, srpms_list, visibility, owner, add_to_repository_id
+    Project.perform_later :low, :run_mass_import, url, srpms_list, visibility, owner, add_to_repository_id
   end
 
   def name_with_owner

From aea5c9421925775bd7bd424eb7ed34770ef0d41f Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Thu, 17 Apr 2014 23:08:22 +0400
Subject: [PATCH 06/14] #376: updated logic, UI

---
 app/admin/node_instructions.rb                | 29 ++++++++++++++++---
 app/jobs/restart_nodes_job.rb                 |  1 +
 app/models/node_instruction.rb                | 20 ++++++++++++-
 .../node_instructions_controller_spec.rb      |  5 ++++
 4 files changed, 50 insertions(+), 5 deletions(-)
 create mode 100644 spec/controllers/admin/node_instructions_controller_spec.rb

diff --git a/app/admin/node_instructions.rb b/app/admin/node_instructions.rb
index 886eb0052..9b14fb554 100644
--- a/app/admin/node_instructions.rb
+++ b/app/admin/node_instructions.rb
@@ -57,11 +57,32 @@ ActiveAdmin.register NodeInstruction do
 
   end
 
-  member_action :force, method: :patch do
-    if NodeInstruction::STATUSES.include?(params[:state])
-      resource.send(params[:state])
-      flash[:info] = 'Action added to queue successfully'
+  sidebar 'Actions', only: :index do
+    locked = NodeInstruction.all_locked?
+    span(class: "status_tag #{locked ? 'red' : 'green'}") do
+      if locked
+        link_to 'Unlock instructions', unlock_all_admin_node_instructions_path, method: :post
+      else
+        link_to 'Lock instructions', lock_all_admin_node_instructions_path, method: :post
+      end
     end
+  end
+
+  collection_action :lock_all, method: :post do
+    NodeInstruction.lock_all
+    flash[:info] = 'Locked successfully'
+    redirect_to admin_node_instructions_path
+  end
+
+  collection_action :unlock_all, method: :post do
+    NodeInstruction.unlock_all
+    flash[:info] = 'Unlocked successfully'
+    redirect_to admin_node_instructions_path
+  end
+
+  member_action :force, method: :patch do
+    resource.send(params[:state])
+    flash[:info] = 'Updated successfully'
     redirect_to admin_node_instruction_path(resource)
   end
 
diff --git a/app/jobs/restart_nodes_job.rb b/app/jobs/restart_nodes_job.rb
index 020062162..74345f056 100644
--- a/app/jobs/restart_nodes_job.rb
+++ b/app/jobs/restart_nodes_job.rb
@@ -2,6 +2,7 @@ class RestartNodesJob
   @queue = :low
 
   def self.perform
+    return if NodeInstruction.all_locked?
     available_nodes = RpmBuildNode.all.map{ |n| n.user_id }.compact.uniq
     NodeInstruction.where(status: NodeInstruction::READY).
       where('user_id NOT IN (?)', available_nodes).find_each(&:restart)
diff --git a/app/models/node_instruction.rb b/app/models/node_instruction.rb
index 501d13924..5daa15ee8 100644
--- a/app/models/node_instruction.rb
+++ b/app/models/node_instruction.rb
@@ -6,6 +6,8 @@ class NodeInstruction < ActiveRecord::Base
     FAILED      = 'failed'
   ]
 
+  LOCK_KEY = 'NodeInstruction::lock-key'
+
   belongs_to :user
 
   attr_encrypted :instruction, key: APP_CONFIG['keys']['node_instruction_secret_key']
@@ -21,7 +23,9 @@ class NodeInstruction < ActiveRecord::Base
 
   state_machine :status, initial: :ready do
 
-    after_transition on: :restart, do: :perform_restart
+    after_transition(on: :restart) do |instruction, transition|
+      instruction.perform_restart
+    end
 
     event :ready do
       transition %i(ready restarting disabled failed) => :ready
@@ -41,6 +45,8 @@ class NodeInstruction < ActiveRecord::Base
   end
 
   def perform_restart
+    restart_failed if NodeInstruction.all_locked?
+
     success = false
     output  = ''
     instruction.lines.each do |command|
@@ -62,4 +68,16 @@ class NodeInstruction < ActiveRecord::Base
   end
   later :perform_restart, queue: :low
 
+  def self.all_locked?
+    Redis.current.get(LOCK_KEY).present?
+  end
+
+  def self.lock_all
+    Redis.current.set(LOCK_KEY, 1)
+  end
+
+  def self.unlock_all
+    Redis.current.del(LOCK_KEY)
+  end
+
 end
diff --git a/spec/controllers/admin/node_instructions_controller_spec.rb b/spec/controllers/admin/node_instructions_controller_spec.rb
new file mode 100644
index 000000000..eb6e735a3
--- /dev/null
+++ b/spec/controllers/admin/node_instructions_controller_spec.rb
@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe Admin::NodeInstructionsController do
+  it_should_behave_like 'an admin controller'
+end

From 16f2bbb246751c5547b07b346660b5b3000dd945 Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Thu, 17 Apr 2014 23:15:35 +0400
Subject: [PATCH 07/14] #376: refactoring according to Alexander's comments

---
 app/admin/node_instructions.rb | 6 +++---
 app/jobs/restart_nodes_job.rb  | 2 +-
 app/models/node_instruction.rb | 6 +++++-
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/app/admin/node_instructions.rb b/app/admin/node_instructions.rb
index 9b14fb554..323568935 100644
--- a/app/admin/node_instructions.rb
+++ b/app/admin/node_instructions.rb
@@ -70,19 +70,19 @@ ActiveAdmin.register NodeInstruction do
 
   collection_action :lock_all, method: :post do
     NodeInstruction.lock_all
-    flash[:info] = 'Locked successfully'
+    flash[:notice] = 'Locked successfully'
     redirect_to admin_node_instructions_path
   end
 
   collection_action :unlock_all, method: :post do
     NodeInstruction.unlock_all
-    flash[:info] = 'Unlocked successfully'
+    flash[:notice] = 'Unlocked successfully'
     redirect_to admin_node_instructions_path
   end
 
   member_action :force, method: :patch do
     resource.send(params[:state])
-    flash[:info] = 'Updated successfully'
+    flash[:notice] = 'Updated successfully'
     redirect_to admin_node_instruction_path(resource)
   end
 
diff --git a/app/jobs/restart_nodes_job.rb b/app/jobs/restart_nodes_job.rb
index 74345f056..b0063d497 100644
--- a/app/jobs/restart_nodes_job.rb
+++ b/app/jobs/restart_nodes_job.rb
@@ -5,7 +5,7 @@ class RestartNodesJob
     return if NodeInstruction.all_locked?
     available_nodes = RpmBuildNode.all.map{ |n| n.user_id }.compact.uniq
     NodeInstruction.where(status: NodeInstruction::READY).
-      where('user_id NOT IN (?)', available_nodes).find_each(&:restart)
+      where.not(user_id: available_nodes).find_each(&:restart)
   end
 
 end
diff --git a/app/models/node_instruction.rb b/app/models/node_instruction.rb
index 5daa15ee8..b720ebda4 100644
--- a/app/models/node_instruction.rb
+++ b/app/models/node_instruction.rb
@@ -10,13 +10,17 @@ class NodeInstruction < ActiveRecord::Base
 
   belongs_to :user
 
+  scope :duplicate, -> id, user_id {
+    where.not(id: id.to_i).where(user_id: user_id, status: STATUSES - [DISABLED])
+  }
+
   attr_encrypted :instruction, key: APP_CONFIG['keys']['node_instruction_secret_key']
 
   validates :user,        presence: true
   validates :instruction, presence: true, length: { maximum: 10000 }
   validates :status,      presence: true
   validate  -> {
-    errors.add(:status, 'Can be only single active instruction for each node') if !disabled? && NodeInstruction.where('id != ?', id.to_i).where(user_id: user_id, status: STATUSES - [DISABLED]).exists?
+    errors.add(:status, 'Can be only single active instruction for each node') if !disabled? && NodeInstruction.duplicate(id.to_i, user_id).exists?
   }
 
   attr_accessible :instruction, :user_id, :output, :status

From 602df4e57a611952ddbec995d85bf1f290c610e6 Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Fri, 18 Apr 2014 00:18:24 +0400
Subject: [PATCH 08/14] #376: added specs

---
 app/jobs/restart_nodes_job.rb        |  2 +-
 spec/factories/node_instructions.rb  |  6 +++++
 spec/factories/rpm_build_nodes.rb    |  6 +++++
 spec/factories/users.rb              |  4 +++
 spec/jobs/restart_nodes_job_spec.rb  | 40 ++++++++++++++++++++++++++++
 spec/models/node_instruction_spec.rb |  9 +++++++
 6 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 spec/factories/node_instructions.rb
 create mode 100644 spec/factories/rpm_build_nodes.rb
 create mode 100644 spec/jobs/restart_nodes_job_spec.rb
 create mode 100644 spec/models/node_instruction_spec.rb

diff --git a/app/jobs/restart_nodes_job.rb b/app/jobs/restart_nodes_job.rb
index b0063d497..107b042fc 100644
--- a/app/jobs/restart_nodes_job.rb
+++ b/app/jobs/restart_nodes_job.rb
@@ -3,7 +3,7 @@ class RestartNodesJob
 
   def self.perform
     return if NodeInstruction.all_locked?
-    available_nodes = RpmBuildNode.all.map{ |n| n.user_id }.compact.uniq
+    available_nodes = RpmBuildNode.all.map{ |n| n.user_id if n.user.try(:system?) }.compact.uniq
     NodeInstruction.where(status: NodeInstruction::READY).
       where.not(user_id: available_nodes).find_each(&:restart)
   end
diff --git a/spec/factories/node_instructions.rb b/spec/factories/node_instructions.rb
new file mode 100644
index 000000000..82a8aed0b
--- /dev/null
+++ b/spec/factories/node_instructions.rb
@@ -0,0 +1,6 @@
+FactoryGirl.define do
+  factory :node_instruction do
+    association :user, factory: :system_user
+    instruction { FactoryGirl.generate(:string) }
+  end
+end
diff --git a/spec/factories/rpm_build_nodes.rb b/spec/factories/rpm_build_nodes.rb
new file mode 100644
index 000000000..6bb155698
--- /dev/null
+++ b/spec/factories/rpm_build_nodes.rb
@@ -0,0 +1,6 @@
+FactoryGirl.define do
+  factory :rpm_build_node do
+    id      { FactoryGirl.generate(:string) }
+    user_id { FactoryGirl.create(:user).id }
+  end
+end
diff --git a/spec/factories/users.rb b/spec/factories/users.rb
index 3bd5fc3a5..86af559fa 100644
--- a/spec/factories/users.rb
+++ b/spec/factories/users.rb
@@ -12,4 +12,8 @@ FactoryGirl.define do
   factory :admin, parent: :user do
     role 'admin'
   end
+
+  factory :system_user, parent: :user do
+    role 'system'
+  end
 end
diff --git a/spec/jobs/restart_nodes_job_spec.rb b/spec/jobs/restart_nodes_job_spec.rb
new file mode 100644
index 000000000..b84ebc129
--- /dev/null
+++ b/spec/jobs/restart_nodes_job_spec.rb
@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe RestartNodesJob do
+
+  it 'ensures that not raises error' do
+    lambda do
+      RestartNodesJob.perform
+    end.should_not raise_exception
+  end
+
+  it 'ensures that do nothing when all instructions disabled' do
+    NodeInstruction.lock_all
+    expect(RpmBuildNode).to_not receive(:all)
+    RestartNodesJob.perform
+  end
+
+  it 'ensures that creates tasks' do
+    allow_any_instance_of(NodeInstruction).to receive(:perform_restart)
+
+    # ABF active node
+    ni1 = FactoryGirl.create(:node_instruction)
+    FactoryGirl.create(:rpm_build_node, user_id: ni1.user_id)
+
+    # User node
+    FactoryGirl.create(:rpm_build_node)
+
+    FactoryGirl.create(:node_instruction, status: NodeInstruction::DISABLED)
+    ni2 = FactoryGirl.create(:node_instruction, status: NodeInstruction::RESTARTING)
+    FactoryGirl.create(:node_instruction, status: NodeInstruction::FAILED)
+
+    ni3 = FactoryGirl.create(:node_instruction)
+
+    RestartNodesJob.perform
+
+    NodeInstruction.where(status: NodeInstruction::RESTARTING).should have(2).items
+    NodeInstruction.where(status: NodeInstruction::RESTARTING).should include(ni2, ni3)
+    NodeInstruction.where(status: NodeInstruction::RESTARTING).should_not include(ni1)
+  end
+
+end
diff --git a/spec/models/node_instruction_spec.rb b/spec/models/node_instruction_spec.rb
new file mode 100644
index 000000000..99d675e22
--- /dev/null
+++ b/spec/models/node_instruction_spec.rb
@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe NodeInstruction do
+
+  it 'is valid given valid attributes' do
+    FactoryGirl.build(:node_instruction).should be_valid
+  end
+
+end

From 721bd49868a858cff2c121363c170ada5dc25ad7 Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Fri, 18 Apr 2014 01:07:32 +0400
Subject: [PATCH 09/14] #376: updated application.yml.travis

---
 config/application.yml.travis | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/application.yml.travis b/config/application.yml.travis
index 8e9c95e0d..a0f0509d6 100644
--- a/config/application.yml.travis
+++ b/config/application.yml.travis
@@ -14,6 +14,7 @@ common: &common
       port: 6379
   keys:
     key_pair_secret_key: 'key_pair_secret_key'
+    node_instruction_secret_key: 'node_instruction_secret_key'
     airbrake_api_key: 'airbrake_api_key'
     devise_pepper: 'e295a79fb7966e94a6e8b184ba65791a'
     secret_token: 'e295a79fb7966e94a6e8b184ba65791a'

From 8c2175d8909459bea34a83676defdce21ef8d033 Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Fri, 18 Apr 2014 01:55:37 +0400
Subject: [PATCH 10/14] #376: save output

---
 app/models/node_instruction.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/models/node_instruction.rb b/app/models/node_instruction.rb
index b720ebda4..7fef37bfe 100644
--- a/app/models/node_instruction.rb
+++ b/app/models/node_instruction.rb
@@ -52,11 +52,11 @@ class NodeInstruction < ActiveRecord::Base
     restart_failed if NodeInstruction.all_locked?
 
     success = false
-    output  = ''
+    stdout  = ''
     instruction.lines.each do |command|
       next if command.blank?
       command.chomp!; command.strip!
-      output << %x[ #{command} 2>&1 ]
+      stdout << %x[ #{command} 2>&1 ]
       success = $?.success?
     end
 
@@ -68,6 +68,7 @@ class NodeInstruction < ActiveRecord::Base
       bl.restart_job
     end
 
+    update_column(:output, stdout)
     success ? ready : restart_failed
   end
   later :perform_restart, queue: :low

From 93a0beb0d24ba5fe6bb6ef1b3bcbc3e734f68ece Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Fri, 18 Apr 2014 02:18:00 +0400
Subject: [PATCH 11/14] #376: updated output

---
 app/admin/node_instructions.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/admin/node_instructions.rb b/app/admin/node_instructions.rb
index 323568935..8644ed153 100644
--- a/app/admin/node_instructions.rb
+++ b/app/admin/node_instructions.rb
@@ -43,7 +43,9 @@ ActiveAdmin.register NodeInstruction do
       row :created_at
       row :updated_at
       row :instruction
-      row :output
+      row(:output) do |ni|
+        ni.output.to_s.lines.join('<br/>').html_safe
+      end
     end
   end
 

From 628b69e22b12ee7d518cb612b84e987ab11108bc Mon Sep 17 00:00:00 2001
From: Alexander Machehin <alexander.machehin@gmail.com>
Date: Fri, 18 Apr 2014 14:51:24 +0600
Subject: [PATCH 12/14] [#369] small highlighting changes

---
 .../projects/git/blobs/_editor.html.haml      |   2 +-
 config/environments/production.rb             |   2 +-
 .../assets/stylesheets/codemirror_editor.scss |   7 +
 vendor/assets/stylesheets/gollum/gollum.scss  | 164 +++++++++---------
 vendor/assets/stylesheets/pygments.scss       |   6 +-
 5 files changed, 95 insertions(+), 86 deletions(-)
 create mode 100644 vendor/assets/stylesheets/codemirror_editor.scss

diff --git a/app/views/projects/git/blobs/_editor.html.haml b/app/views/projects/git/blobs/_editor.html.haml
index f57533fa6..075f2bd80 100644
--- a/app/views/projects/git/blobs/_editor.html.haml
+++ b/app/views/projects/git/blobs/_editor.html.haml
@@ -1,5 +1,5 @@
 = javascript_include_tag 'codemirror_editor'
-= stylesheet_link_tag    'codemirror'
+= stylesheet_link_tag    'codemirror_editor'
 
 %h3= t("layout.projects.files_in_project")
 .files
diff --git a/config/environments/production.rb b/config/environments/production.rb
index 274242237..39258a20b 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -65,7 +65,7 @@ Rosa::Application.configure do
   # Precompile additional assets (application.js, application.css, and all non-JS/CSS are already added)
   config.assets.precompile +=
     %w(login.css login.js reg_session.css tour.css tour.js gollum/editor/langs/*.js moment/ru.js
-       codemirror_editor.js codemirror.css)
+       codemirror_editor.js codemirror_editor.css)
 
   # Compress JavaScripts and CSS.
   config.assets.js_compressor = :uglifier
diff --git a/vendor/assets/stylesheets/codemirror_editor.scss b/vendor/assets/stylesheets/codemirror_editor.scss
new file mode 100644
index 000000000..4891a344d
--- /dev/null
+++ b/vendor/assets/stylesheets/codemirror_editor.scss
@@ -0,0 +1,7 @@
+@import "codemirror";
+
+.cm-s-default span.cm-preamble,  {color: #b26818; font-weight: bold;}
+.cm-s-default span.cm-macro {color: #b218b2;}
+.cm-s-default span.cm-section {color: green; font-weight: bold;}
+.cm-s-default span.cm-script {color: red;}
+.cm-s-default span.cm-issue {color: yellow;}
diff --git a/vendor/assets/stylesheets/gollum/gollum.scss b/vendor/assets/stylesheets/gollum/gollum.scss
index f23528363..c64f8416d 100755
--- a/vendor/assets/stylesheets/gollum/gollum.scss
+++ b/vendor/assets/stylesheets/gollum/gollum.scss
@@ -47,11 +47,11 @@
     margin: 0;
     padding: 0.08em 0 0 0;
   }
-  
+
   #head ul.actions {
     float: right;
   }
-  
+
 /* @section content */
 //#wiki-content {
 //  height: 1%;
@@ -79,7 +79,7 @@
 //  .has-rightbar #wiki-body {
 //    width: 68%;
 //  }
-  
+
   /* @section rightbar */
   #wiki-rightbar {
     background-color: #f7f7f7;
@@ -88,12 +88,12 @@
     float: right;
     padding: 7px;
     width: 25%;
-    
+
     border-radius: 0.5em;
     -moz-border-radius: 0.5em;
     -webkit-border-radius: 0.5em;
   }
-  
+
     #wiki-rightbar p {
       margin: 13px 0 0;
     }
@@ -109,13 +109,13 @@
       padding: 0 0 0.5em 0;
       text-shadow: 0 1px 0 #fff;
     }
-    
+
     /* Back arrow */
     #wiki-rightbar p.parent:before {
       color: #666;
       content: "← ";
     }
-    
+
     #wiki-rightbar h3 {
       font-size: 1.2em;
       color: #333;
@@ -123,12 +123,12 @@
       padding: 0;
       text-shadow: 0 1px 0 #fff;
     }
-    
+
     #wiki-rightbar ul {
       margin: 0.5em 0 1em;
       padding: 0;
     }
-    
+
     #wiki-rightbar ul li {
       color: #bbb;
       margin: 0 0 0 1em;
@@ -136,19 +136,19 @@
       line-height: 1.75em;
       list-style-position: inside;
       list-style-type: round;
-    } 
-    
+    }
+
     #wiki-rightbar #nav ul li a {
       font-weight: bold;
       text-shadow: 0 1px 0 #fff;
     }
-    
+
   /* @section footer */
   #wiki-footer {
     clear: both;
     margin: 2em 0 5em;
   }
-  
+
     .has-rightbar #wiki-footer {
       width: 70%;
     }
@@ -160,18 +160,18 @@
       padding: 0 0 0.2em;
       text-shadow: 0 1px 0 #fff;
     }
-    
+
     #wiki-footer #footer-content p {
       margin: 0.5em 0 0;
       padding: 0;
     }
-    
+
     #wiki-footer #footer-content ul.links {
       margin: 0.5em 0 0;
       overflow: hidden;
       padding: 0;
     }
-    
+
     #wiki-footer #footer-content ul.links li {
       color: #999;
       float: left;
@@ -180,21 +180,21 @@
       padding: 0;
       margin-left: 0.75em;
     }
-    
+
     #wiki-footer #footer-content ul.links li a {
       font-weight: bold;
       text-shadow: 0 1px 0 #fff;
     }
-    
+
     #wiki-footer #footer-content ul.links li:first-child {
       list-style-type: none;
       margin: 0;
-    }  
-    
+    }
+
     .ff #wiki-footer #footer-content ul.links li:first-child {
       margin: 0 -0.75em 0 0;
     }
-  
+
   /* @section page-footer */
   .page #gollum-footer {
     border-top: 1px solid #ccc;
@@ -221,23 +221,23 @@
     margin: 2em 0;
     padding: 0;
   }
-  
+
   #wiki-history table, #wiki-history tbody {
     border-collapse: collapse;
     padding: 0;
     margin: 0;
     width: 100%;
   }
-  
+
   #wiki-history table tr {
     padding: 0;
     margin: 0;
   }
-  
+
   #wiki-history table tr {
     background-color: #ebf2f6;
   }
-  
+
   #wiki-history table tr td {
     border: 1px solid #c0dce9;
     font-size: 1.2em;
@@ -245,13 +245,13 @@
     margin: 0;
     padding: 0.3em 0.7em;
   }
-  
+
   #wiki-history table tr td.checkbox {
     min-width: 2em;
     padding: 0.3em;
     width: 24px;
   }
-  
+
   #wiki-history table tr td.checkbox input {
     cursor: pointer;
     display: block;
@@ -259,39 +259,39 @@
     padding-top: 0.4em;
     margin-right: -0.2em;
   }
-  
+
   #wiki-history table tr:nth-child(2n),
   #wiki-history table tr.alt-row {
     background-color: #f3f7fa;
   }
-  
+
   #wiki-history table tr.selected {
     background-color: #ffffea !important;
     z-index: 100;
   }
-  
+
   #wiki-history table tr td.commit-name {
     border-left: 0;
   }
-  
+
   #wiki-history table tr td.commit-name span.time-elapsed {
     color: #999;
   }
-  
+
   #wiki-history table tr td.author {
     width: 20%;
   }
-  
+
   #wiki-history table tr td.author a {
     color: #000;
     font-weight: bold;
   }
-  
+
   #wiki-history table tr td.author a span.username {
     display: block;
     padding-top: 3px;
   }
-  
+
   #wiki-history table tr td img {
     background-color: #fff;
     border: 1px solid #999;
@@ -303,7 +303,7 @@
     width: 18px;
     padding: 2px;
   }
-  
+
   #wiki-history table tr td.commit-name a {
     font-size: 0.9em;
     font-family: 'Monaco', 'Andale Mono', Consolas, 'Courier New', monospace;
@@ -338,26 +338,26 @@
     color: #000;
     font-weight: bold;
   }
-  
+
   .results #results {
     border-bottom: 1px solid #ccc;
     margin-bottom: 2em;
     padding-bottom: 2em;
   }
-  
+
   .results #results ul {
     margin: 2em 0 0 0;
     padding: 0;
     padding-left: 25px;
   }
-  
+
     .results #results ul li {
       font-size: 1.2em;
       line-height: 1.6em;
       list-style-position: outside;
       padding: 0.2em 0;
     }
-    
+
     .results #results ul li span.count {
       color: #999;
     }
@@ -367,11 +367,11 @@
     line-height: 1.6em;
     margin-top: 2em;
   }
-  
+
   .results #footer ul.actions li {
     margin: 0 1em 0 0;
   }
-  
+
 
 /* @section compare */
 .compare h1 {
@@ -383,11 +383,11 @@
     color: #000;
     font-weight: bold;
   }
-  
+
   .compare #compare-content {
     margin-top: 3em;
   }
-  
+
   .compare .data {
     border: 1px solid #ddd;
     margin-top: 1em;
@@ -398,16 +398,16 @@
   .compare .data table {
     width: 100%;
   }
-  
+
   .compare .data pre {
     margin: 0;
     padding: 0;
   }
-  
+
   .compare .data pre div {
     padding: 0 0 0 1em;
   }
-  
+
   .compare .data tr td {
     font-family: "Consolas", "Monaco", "Andale Mono", "Courier New", monospace;
     font-size: 1.2em;
@@ -415,11 +415,11 @@
     margin: 0;
     padding: 0;
   }
-  
+
   .compare .data td.line_numbers {
     background: #f7f7f7;
     border-right: 1px solid #999;
-    color: #999; 
+    color: #999;
     padding: 0 0 0 0.5em;
     width: 1%;
   }
@@ -433,7 +433,7 @@
   margin-left: 0;
   margin-right: 0.6em;
 }
-  
+
 
 /* git_access */
 //#wiki-content .url-box {
@@ -450,7 +450,7 @@
 //  padding: 10px 10px 0;
 //  width: 100%;
 //}
-//  
+//
 //#wiki-content .url-box ul.clone_urls {
 //  float: left;
 //  height: 23px;
@@ -539,30 +539,30 @@ ul.clone-urls li {
 //    padding: 3px 5px 2px;
 //    width: 400px;
 //}
-  
+
 /* @control syntax */
 .highlight  { background: #ffffff; }
-.highlight .c { color: #999988; font-style: italic } 
-.highlight .err { color: #a61717; background-color: #e3d2d2 } 
-.highlight .k { font-weight: bold } 
-.highlight .o { font-weight: bold } 
-.highlight .cm { color: #999988; font-style: italic } 
-.highlight .cp { color: #999999; font-weight: bold } 
-.highlight .c1 { color: #999988; font-style: italic } 
-.highlight .cs { color: #999999; font-weight: bold; font-style: italic } 
-.highlight .gd { color: #000000; background-color: #ffdddd } 
-.highlight .gd .x { color: #000000; background-color: #ffaaaa } 
-.highlight .ge { font-style: italic } 
-.highlight .gr { color: #aa0000 } 
-.highlight .gh { color: #999999 } 
-.highlight .gi { color: #000000; background-color: #ddffdd } 
-.highlight .gi .x { color: #000000; background-color: #aaffaa } 
+.highlight .c { color: #a50; font-style: italic; font-weight: bold; }
+.highlight .err { color: #a61717; background-color: #e3d2d2 }
+.highlight .k { font-weight: bold }
+.highlight .o { font-weight: bold }
+.highlight .cm { color: #999988; font-style: italic }
+.highlight .cp { color: #999999; font-weight: bold }
+.highlight .c1 { color: #999988; font-style: italic }
+.highlight .cs { color: #999999; font-weight: bold; font-style: italic }
+.highlight .gd { color: #000000; background-color: #ffdddd }
+.highlight .gd .x { color: #000000; background-color: #ffaaaa }
+.highlight .ge { font-style: italic }
+.highlight .gr { color: #aa0000 }
+.highlight .gh { color: #b26818; font-weight: bold; }
+.highlight .gi { color: #000000; background-color: #ddffdd }
+.highlight .gi .x { color: #000000; background-color: #aaffaa }
 .highlight .gc { color: #999; background-color: #EAF2F5 }
-.highlight .go { color: #888888 } 
-.highlight .gp { color: #555555 } 
-.highlight .gs { font-weight: bold } 
-.highlight .gu { color: #aaaaaa } 
-.highlight .gt { color: #aa0000 } 
+.highlight .go { color: #888888 }
+.highlight .gp { color: #555555 }
+.highlight .gs { font-weight: bold }
+.highlight .gu { color: #aaaaaa }
+.highlight .gt { color: #aa0000 }
 
 
 /* @control minibutton */
@@ -656,7 +656,7 @@ ul.actions {
   margin: -10% 0 0 -35%;
   position: absolute;
   width: 70%;
-  
+
   border-radius: 0.5em;
   -moz-border-radius: 0.5em;
   -webkit-border-radius: 0.5em;
@@ -689,7 +689,7 @@ ul.actions {
     background: #fff;
     border: 1px solid #d4d4d4;
     overflow: hidden;
-    
+
     border-radius: 0.3em;
     -moz-border-radius: 0.3em;
     -webkit-border-radius: 0.3em;
@@ -702,22 +702,22 @@ ul.actions {
     font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif;
     font-size: 1.2em;
     height: 1.8em;
-    
+
     -webkit-focus-ring: none;
   }
 
   .ff #head #gollum-searchbar #gollum-searchbar-fauxtext input#search-query {
     padding: 0.2em 0 0.2em 0.5em;
   }
-  
+
   .ie #head #gollum-searchbar #gollum-searchbar-fauxtext input#search-query {
     padding: 0.4em 0 0 0.5em;
   }
-  
+
   #head #gollum-searchbar #gollum-searchbar-fauxtext input#search-query.ph {
     color: #999;
   }
-  
+
   #head #gollum-searchbar #gollum-searchbar-fauxtext #search-submit {
     border: 0;
     border-left: 1px solid #d4d4d4;
@@ -726,12 +726,12 @@ ul.actions {
     padding: 0;
     float: right;
     font-size: 1.2em;
-    
+
     border-radius: 0 3px 3px 0;
     -moz-border-radius: 0 3px 3px 0;
     -webkit-border-radius: 0 3px 3px 0;
   }
-  
+
   #head #gollum-searchbar #gollum-searchbar-fauxtext #search-submit span {
     background-image: image-url('gollum/icon-sprite.png');
     background-position: -431px -1px;
@@ -742,12 +742,12 @@ ul.actions {
     text-indent: -5000px;
     width: 28px;
   }
-  
+
   .ff #head #gollum-searchbar #gollum-searchbar-fauxtext #search-submit span,
   .ie #head #gollum-searchbar #gollum-searchbar-fauxtext #search-submit span {
     height: 2.2em;
   }
-  
+
   #head #gollum-searchbar #gollum-searchbar-fauxtext #search-submit:hover span {
     background-position: -431px -28px;
     padding: 0;
diff --git a/vendor/assets/stylesheets/pygments.scss b/vendor/assets/stylesheets/pygments.scss
index 61a35b4df..4edee38eb 100644
--- a/vendor/assets/stylesheets/pygments.scss
+++ b/vendor/assets/stylesheets/pygments.scss
@@ -48,7 +48,7 @@
 .gs { font-weight: bold } /* Generic.Strong */
 .gu { color: #aaaaaa } /* Generic.Subheading */
 .gt { color: #aa0000 } /* Generic.Traceback */
-.kc { color: #000000; font-weight: bold } /* Keyword.Constant */
+.kc { color: #b218b2; font-weight: bold } /* Keyword.Constant */
 .kd { color: #000000; font-weight: bold } /* Keyword.Declaration */
 .kn { color: #000000; font-weight: bold } /* Keyword.Namespace */
 .kp { color: #000000; font-weight: bold } /* Keyword.Pseudo */
@@ -60,7 +60,7 @@
 .nb { color: #0086B3 } /* Name.Builtin */
 .nc { color: #445588; font-weight: bold } /* Name.Class */
 .no { color: #008080 } /* Name.Constant */
-.nd { color: #3c5d5d; font-weight: bold } /* Name.Decorator */
+.nd { color: green; font-weight: bold; } /* Name.Decorator */
 .ni { color: #800080 } /* Name.Entity */
 .ne { color: #990000; font-weight: bold } /* Name.Exception */
 .nf { color: #990000; font-weight: bold } /* Name.Function */
@@ -90,3 +90,5 @@
 .vg { color: #008080 } /* Name.Variable.Global */
 .vi { color: #008080 } /* Name.Variable.Instance */
 .il { color: #009999 } /* Literal.Number.Integer.Long */
+
+.c { color: #a50; } /* Comment */

From 5b895bbb2559b48450f7c7ef7a304c473680b514 Mon Sep 17 00:00:00 2001
From: Alexander Machehin <alexander.machehin@gmail.com>
Date: Fri, 18 Apr 2014 14:52:54 +0600
Subject: [PATCH 13/14] [#369] fix assets precompile list

---
 config/environments/staging.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/environments/staging.rb b/config/environments/staging.rb
index 831d485b1..730cc8e7b 100644
--- a/config/environments/staging.rb
+++ b/config/environments/staging.rb
@@ -64,7 +64,7 @@ Rosa::Application.configure do
   # Precompile additional assets (application.js, application.css, and all non-JS/CSS are already added)
   config.assets.precompile +=
     %w(login.css login.js reg_session.css tour.css tour.js gollum/editor/langs/*.js moment/ru.js
-       codemirror_editor.js codemirror.css)
+       codemirror_editor.js codemirror_editor.css)
 
   # Compress JavaScripts and CSS.
   config.assets.js_compressor = :uglifier

From 559e41d94b2d654def1636b5c4cfc82b3d89f422 Mon Sep 17 00:00:00 2001
From: Vokhmin Alexey V <avokhmin@gmail.com>
Date: Fri, 18 Apr 2014 21:17:17 +0400
Subject: [PATCH 14/14] Fixed: undefined method admin?

---
 app/views/projects/build_lists/show.json.jbuilder | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/projects/build_lists/show.json.jbuilder b/app/views/projects/build_lists/show.json.jbuilder
index 1f723f807..95a492056 100644
--- a/app/views/projects/build_lists/show.json.jbuilder
+++ b/app/views/projects/build_lists/show.json.jbuilder
@@ -31,7 +31,7 @@ json.build_list do
   json.builder do
     json.fullname @build_list.builder.try(:fullname)
     json.path user_path(@build_list.builder)
-  end if @build_list.builder && (!@build_list.builder.system? || current_user.admin?)
+  end if @build_list.builder && (!@build_list.builder.system? || current_user.try(:admin?))
 
   json.advisory do
     json.(@build_list.advisory, :description, :advisory_id)