From d06e452935fe85a9b4dab53ba8c9c9bc2d0c7970 Mon Sep 17 00:00:00 2001 From: Alexander Machehin Date: Tue, 3 Apr 2012 01:27:36 +0600 Subject: [PATCH 1/9] [refs #263] fixed feed with no repo --- app/models/activity_feed_observer.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/activity_feed_observer.rb b/app/models/activity_feed_observer.rb index dbbe8a189..b6d08342b 100644 --- a/app/models/activity_feed_observer.rb +++ b/app/models/activity_feed_observer.rb @@ -70,6 +70,7 @@ class ActivityFeedObserver < ActiveRecord::Observer end when 'GitHook' + return unless record.project change_type = record.change_type branch_name = record.refname.split('/').last From 5a384c6846e040b65ade6c92aeb8c31bf2b8d550 Mon Sep 17 00:00:00 2001 From: Alexander Machehin Date: Tue, 3 Apr 2012 01:29:06 +0600 Subject: [PATCH 2/9] [refs #263] added dj queues --- app/models/project.rb | 10 +++++----- lib/tasks/hook.rake | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index 8f52595aa..fa3ae08db 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -36,12 +36,12 @@ class Project < ActiveRecord::Base scope :addable_to_repository, lambda { |repository_id| where("projects.id NOT IN (SELECT project_to_repositories.project_id FROM project_to_repositories WHERE (project_to_repositories.repository_id = #{ repository_id }))") } after_create :attach_to_personal_repository - after_create :create_git_repo + after_create {|p| p.delay(:queue => 'fork').create_git_repo} after_save :create_wiki after_destroy :destroy_git_repo after_destroy :destroy_wiki - after_save {|p| p.delay.import_attached_srpm if p.srpm?} # should be after create_git_repo + after_save {|p| p.delay(:queue => 'import').import_attached_srpm if p.srpm?} # should be after create_git_repo # after_rollback lambda { destroy_git_repo rescue true if new_record? } has_ancestry @@ -214,8 +214,8 @@ class Project < ActiveRecord::Base end def create_git_repo - is_root? ? Grit::Repo.init_bare(path) : parent.git_repository.repo.delay.fork_bare(path) - write_hook.delay + is_root? ? Grit::Repo.init_bare(path) : (dummy = Grit::Repo.new(path) rescue parent.git_repository.repo.fork_bare(path)) + write_hook end def destroy_git_repo @@ -248,7 +248,7 @@ class Project < ActiveRecord::Base hook = File.join(::Rails.root.to_s, 'tmp', "post-receive-hook") FileUtils.cp(File.join(::Rails.root.to_s, 'bin', "post-receive-hook.partial"), hook) File.open(hook, 'a') do |f| - s = "\n /bin/bash -l -c \"cd #{is_production ? '/srv/rosa_build/current' : Rails.root.to_s} && #{is_production ? 'RAILS_ENV=production' : ''} bundle exec rails runner 'Project.delay.process_hook(\\\"$owner\\\", \\\"$reponame\\\", \\\"$newrev\\\", \\\"$oldrev\\\", \\\"$ref\\\", \\\"$newrev_type\\\", \\\"$oldrev_type\\\")'\"" + s = "\n /bin/bash -l -c \"cd #{is_production ? '/srv/rosa_build/current' : Rails.root.to_s} && #{is_production ? 'RAILS_ENV=production' : ''} bundle exec rails runner 'Project.delay(:queue => \\\"hook\\\").process_hook(\\\"$owner\\\", \\\"$reponame\\\", \\\"$newrev\\\", \\\"$oldrev\\\", \\\"$ref\\\", \\\"$newrev_type\\\", \\\"$oldrev_type\\\")'\"" s << " > /dev/null 2>&1" if is_production s << "\ndone\n" f.write(s) diff --git a/lib/tasks/hook.rake b/lib/tasks/hook.rake index 911666636..b543c2413 100644 --- a/lib/tasks/hook.rake +++ b/lib/tasks/hook.rake @@ -6,7 +6,7 @@ namespace :hook do hook = File.join(::Rails.root.to_s, 'tmp', "post-receive-hook") FileUtils.cp(File.join(::Rails.root.to_s, 'bin', "post-receive-hook.partial"), hook) File.open(hook, 'a') do |f| - s = "\n /bin/bash -l -c \"cd #{is_production ? '/srv/rosa_build/current' : Rails.root.to_s} && #{is_production ? 'RAILS_ENV=production' : ''} bundle exec rails runner 'Project.delay.process_hook(\\\"$owner\\\", \\\"$reponame\\\", \\\"$newrev\\\", \\\"$oldrev\\\", \\\"$ref\\\", \\\"$newrev_type\\\", \\\"$oldrev_type\\\")'\"" + s = "\n /bin/bash -l -c \"cd #{is_production ? '/srv/rosa_build/current' : Rails.root.to_s} && #{is_production ? 'RAILS_ENV=production' : ''} bundle exec rails runner 'Project.delay(:queue => \\\"hook\\\").process_hook(\\\"$owner\\\", \\\"$reponame\\\", \\\"$newrev\\\", \\\"$oldrev\\\", \\\"$ref\\\", \\\"$newrev_type\\\", \\\"$oldrev_type\\\")'\"" s << " > /dev/null 2>&1" if is_production s << "\ndone\n" f.write(s) From f4b3a4275a79cbec4b4f25709ee7bc332a0dfe46 Mon Sep 17 00:00:00 2001 From: Alexander Machehin Date: Tue, 3 Apr 2012 01:30:14 +0600 Subject: [PATCH 3/9] [refs #263] some hook task improvement --- lib/tasks/hook.rake | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/tasks/hook.rake b/lib/tasks/hook.rake index b543c2413..cd3841fba 100644 --- a/lib/tasks/hook.rake +++ b/lib/tasks/hook.rake @@ -18,8 +18,14 @@ namespace :hook do projects = ENV['project_id'] ? Project.where(:id => eval(ENV['project_id'])) : Project projects.where('created_at >= ?', Time.now.ago(ENV['period'] ? eval(ENV['period']) : 100.years)).each do |project| hook_file = File.join(project.path, 'hooks', 'post-receive') - FileUtils.copy_entry(hook, hook_file, false, false, true) - count = count + 1 + begin + FileUtils.copy_entry(hook, hook_file, false, false, true) + count = count + 1 + rescue Exception => e + say "----\nCatching exception with project #{project.id}" + say e.message + say '----' + end end say "Writing to #{count.to_s} repo(s)" say "Removing temporary file" From 1fb63cf1a7b5490f6d8cc2ddd961f636bc354da5 Mon Sep 17 00:00:00 2001 From: Alexander Machehin Date: Tue, 3 Apr 2012 17:32:00 +0600 Subject: [PATCH 4/9] [refs #263] division init bare & fork --- app/models/project.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index fa3ae08db..297114d00 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -36,7 +36,8 @@ class Project < ActiveRecord::Base scope :addable_to_repository, lambda { |repository_id| where("projects.id NOT IN (SELECT project_to_repositories.project_id FROM project_to_repositories WHERE (project_to_repositories.repository_id = #{ repository_id }))") } after_create :attach_to_personal_repository - after_create {|p| p.delay(:queue => 'fork').create_git_repo} + after_create :create_git_repo + after_create {|p| p.delay(:queue => 'fork').fork_git_repo unless root?} after_save :create_wiki after_destroy :destroy_git_repo @@ -214,7 +215,14 @@ class Project < ActiveRecord::Base end def create_git_repo - is_root? ? Grit::Repo.init_bare(path) : (dummy = Grit::Repo.new(path) rescue parent.git_repository.repo.fork_bare(path)) + if root? + Grit::Repo.init_bare(path) + write_hook.delay(:queue => 'fork') + end + end + + def fork_git_repo + dummy = Grit::Repo.new(path) rescue parent.git_repository.repo.fork_bare(path) write_hook end From 06cd8a3c4548c0288e685ff29a1c7c7143fe0a79 Mon Sep 17 00:00:00 2001 From: Alexander Machehin Date: Tue, 3 Apr 2012 21:12:11 +0600 Subject: [PATCH 5/9] [refs #295] 4 workers to dj --- config/deploy.rb | 9 +++++---- config/production.pill | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 8f4b921c2..6742b24da 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -34,6 +34,7 @@ set :keep_releases, 3 set :scm, :git set :repository, "git@github.com:warpc/rosa-build.git" set :deploy_via, :remote_cache +set :delayed_job_args, "-n 4" require 'lib/recipes/nginx' require 'lib/recipes/unicorn' @@ -49,11 +50,11 @@ namespace :deploy do task :symlink_all, :roles => :app do run "mkdir -p #{fetch :shared_path}/config" - + # Setup DB run "cp -n #{fetch :release_path}/config/database.yml.sample #{fetch :shared_path}/config/database.yml" run "ln -nfs #{fetch :shared_path}/config/database.yml #{fetch :release_path}/config/database.yml" - + # Setup application run "cp -n #{fetch :release_path}/config/deploy/application.#{fetch :stage}.yml #{fetch :shared_path}/config/application.yml" run "ln -nfs #{fetch :shared_path}/config/application.yml #{fetch :release_path}/config/application.yml" @@ -66,7 +67,7 @@ namespace :deploy do task :symlink_pids, :roles => :app do run "cd #{fetch :shared_path}/tmp && ln -nfs ../pids pids" end - + # Speed up precompile (http://www.bencurtis.com/2011/12/skipping-asset-compilation-with-capistrano ) # namespace :assets do # task :precompile, :roles => :web, :except => { :no_release => true } do @@ -77,7 +78,7 @@ namespace :deploy do # logger.info "Skipping asset pre-compilation because there were no asset changes" # end # end - # end + # end end after "deploy:finalize_update", "deploy:symlink_all" diff --git a/config/production.pill b/config/production.pill index 1d9f23340..7613914c7 100644 --- a/config/production.pill +++ b/config/production.pill @@ -9,11 +9,11 @@ Bluepill.application(app_name) do |app| process.stop_grace_time = 10.seconds process.restart_grace_time = 10.seconds - process.start_command = "/usr/bin/env ruby script/delayed_job start" + process.start_command = "/usr/bin/env ruby script/delayed_job -n 4 start" process.stop_command = "/usr/bin/env ruby script/delayed_job stop" process.pid_file = File.join(app.working_dir, 'tmp', 'pids', 'delayed_job.pid') end - + app.process("newrelic") do |process| process.start_grace_time = 10.seconds process.stop_grace_time = 10.seconds From 302179eff8e972548f8c84e9342beb0e726351d3 Mon Sep 17 00:00:00 2001 From: Alexander Machehin Date: Wed, 4 Apr 2012 18:13:10 +0600 Subject: [PATCH 6/9] [refs #295] set priority to delayed jobs --- app/models/project.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index 297114d00..be1cecc0a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -20,7 +20,7 @@ class Project < ActiveRecord::Base validates :name, :uniqueness => {:scope => [:owner_id, :owner_type], :case_sensitive => false}, :presence => true, :format => {:with => /^[a-zA-Z0-9_\-\+\.]+$/} validates :owner, :presence => true validate { errors.add(:base, :can_have_less_or_equal, :count => MAX_OWN_PROJECTS) if owner.projects.size >= MAX_OWN_PROJECTS } - + validates_attachment_size :srpm, :less_than => 500.megabytes validates_attachment_content_type :srpm, :content_type => ['application/octet-stream', "application/x-rpm", "application/x-redhat-package-manager"], :message => I18n.t('layout.invalid_content_type') @@ -37,12 +37,12 @@ class Project < ActiveRecord::Base after_create :attach_to_personal_repository after_create :create_git_repo - after_create {|p| p.delay(:queue => 'fork').fork_git_repo unless root?} + after_create {|p| p.delay(:queue => 'fork', :priority => 20).fork_git_repo unless root?} after_save :create_wiki after_destroy :destroy_git_repo after_destroy :destroy_wiki - after_save {|p| p.delay(:queue => 'import').import_attached_srpm if p.srpm?} # should be after create_git_repo + after_save {|p| p.delay(:queue => 'import', :priority => 10).import_attached_srpm if p.srpm?} # should be after create_git_repo # after_rollback lambda { destroy_git_repo rescue true if new_record? } has_ancestry @@ -217,7 +217,7 @@ class Project < ActiveRecord::Base def create_git_repo if root? Grit::Repo.init_bare(path) - write_hook.delay(:queue => 'fork') + write_hook.delay(:queue => 'fork', :priority => 15) end end From 349e173f605db701f5e0a34cc683728eca322fe8 Mon Sep 17 00:00:00 2001 From: Alexander Machehin Date: Wed, 4 Apr 2012 22:50:05 +0600 Subject: [PATCH 7/9] [refs #295] deploy recipes for dj --- config/deploy.rb | 3 +- config/production.pill | 16 +++++---- .../20120404134602_modify_default_queue.rb | 11 ++++++ lib/recipes/bluepill.rb | 6 ++-- lib/recipes/delayed_job.rb | 35 +++++++++++++++++++ 5 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 db/migrate/20120404134602_modify_default_queue.rb create mode 100644 lib/recipes/delayed_job.rb diff --git a/config/deploy.rb b/config/deploy.rb index 6742b24da..d6aa18778 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -11,7 +11,6 @@ set :default_environment, { require 'rvm/capistrano' require 'bundler/capistrano' -require 'delayed/recipes' require 'airbrake/capistrano' set :whenever_command, "bundle exec whenever" @@ -34,11 +33,11 @@ set :keep_releases, 3 set :scm, :git set :repository, "git@github.com:warpc/rosa-build.git" set :deploy_via, :remote_cache -set :delayed_job_args, "-n 4" require 'lib/recipes/nginx' require 'lib/recipes/unicorn' require 'lib/recipes/bluepill' +require 'lib/recipes/delayed_job' namespace :deploy do task :stub_xml_rpc do diff --git a/config/production.pill b/config/production.pill index 7613914c7..8cee2adf1 100644 --- a/config/production.pill +++ b/config/production.pill @@ -4,14 +4,16 @@ app_name = ENV['APP_NAME'] || 'rosa_build' Bluepill.application(app_name) do |app| app.uid = app.gid = 'rosa' app.working_dir = "/srv/#{app_name}/current" - app.process("delayed_job") do |process| - process.start_grace_time = 10.seconds - process.stop_grace_time = 10.seconds - process.restart_grace_time = 10.seconds + %w(fork import hook default).each do |queue| + app.process("delayed_job_#{queue}_queue") do |process| + process.start_grace_time = 10.seconds + process.stop_grace_time = 10.seconds + process.restart_grace_time = 10.seconds - process.start_command = "/usr/bin/env ruby script/delayed_job -n 4 start" - process.stop_command = "/usr/bin/env ruby script/delayed_job stop" - process.pid_file = File.join(app.working_dir, 'tmp', 'pids', 'delayed_job.pid') + process.start_command = "/usr/bin/env ruby script/delayed_job --queue=#{queue} -p #{queue} --pid-dir=/srv/#{app_name}/current/tmp/#{queue}_pids start" + process.stop_command = "/usr/bin/env ruby script/delayed_job --pid-dir=/srv/#{app_name}/current/tmp/#{queue}_pids stop" + process.pid_file = File.join(app.working_dir, 'tmp', "#{queue}_pids", 'delayed_job.pid') + end end app.process("newrelic") do |process| diff --git a/db/migrate/20120404134602_modify_default_queue.rb b/db/migrate/20120404134602_modify_default_queue.rb new file mode 100644 index 000000000..e8aebf214 --- /dev/null +++ b/db/migrate/20120404134602_modify_default_queue.rb @@ -0,0 +1,11 @@ +class ModifyDefaultQueue < ActiveRecord::Migration + def up + change_column :delayed_jobs, :queue, :string, :default => 'default' + execute "UPDATE delayed_jobs SET queue = 'default'" + end + + def down + change_column :delayed_jobs, :queue, :string, :default => nil + execute "UPDATE delayed_jobs SET queue = null" + end +end diff --git a/lib/recipes/bluepill.rb b/lib/recipes/bluepill.rb index 5b23601f2..f09965eb8 100644 --- a/lib/recipes/bluepill.rb +++ b/lib/recipes/bluepill.rb @@ -24,9 +24,11 @@ Capistrano::Configuration.instance(:must_exist).load do run "cd #{fetch :current_path} && #{try_sudo} #{bluepill_binary} #{fetch :application} status" end - desc "Restart DJ process" + desc "Restart DJ processes" task :restart_dj, :roles => [:app] do - run "cd #{fetch :current_path} && #{try_sudo} #{bluepill_binary} #{fetch :application} restart delayed_job" + %w(fork import hook default).each do |queue| + run "cd #{fetch :current_path} && #{try_sudo} #{bluepill_binary} #{fetch :application} restart delayed_job_#{queue}_queue" + end end end diff --git a/lib/recipes/delayed_job.rb b/lib/recipes/delayed_job.rb new file mode 100644 index 000000000..c1fb6a7c4 --- /dev/null +++ b/lib/recipes/delayed_job.rb @@ -0,0 +1,35 @@ +# -*- encoding : utf-8 -*- +Capistrano::Configuration.instance(:must_exist).load do + namespace :delayed_job do + def dj_queues + %w(fork import hook default) + end + + def rails_env + fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : '' + end + + def roles + fetch(:delayed_job_server_role, :app) + end + + desc "Stop the delayed_job process" + task :stop, :roles => lambda { roles } do + dj_queues.each do |queue| + run "cd #{current_path};#{rails_env} script/delayed_job --pid-dir=#{current_path}/tmp/#{queue}_pids stop" + end + end + + desc "Start the delayed_job process" + task :start, :roles => lambda { roles } do + dj_queues.each do |queue| + run "cd #{current_path};#{rails_env} script/delayed_job --queue=#{queue} -p #{queue} --pid-dir=#{current_path}/tmp/#{queue}_pids start" + end + end + + desc "Restart the delayed_job process" + task :restart, :roles => lambda { roles } do + run "cd #{current_path};#{rails_env} script/delayed_job --queue=#{queue} -p #{queue} --pid-dir=#{current_path}/tmp/#{queue}_pids restart" + end + end +end From 6d5cee2c04425dbf300f8b2908b53e2811376cab Mon Sep 17 00:00:00 2001 From: Alexander Machehin Date: Thu, 5 Apr 2012 17:36:53 +0600 Subject: [PATCH 8/9] [refs #383] added ability to destroy project --- app/models/ability.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/ability.rb b/app/models/ability.rb index c7c89c70d..6820d6d97 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -58,6 +58,7 @@ class Ability can([:update, :sections, :manage_collaborators], Project) {|project| local_admin? project} can(:fork, Project) {|project| can? :read, project} can(:destroy, Project) {|project| owner? project} + can(:destroy, Project) {|project| project.owner_type == 'Group' and project.owner.objects.exists?(:object_type => 'User', :object_id => user.id, :role => 'admin')} can :remove_user, Project can [:read, :owned], BuildList, :user_id => user.id From 01976015a6ae197b046c51dd88f1bd731aa118ea Mon Sep 17 00:00:00 2001 From: Alexander Machehin Date: Thu, 5 Apr 2012 22:52:21 +0600 Subject: [PATCH 9/9] [refs #295] fix broken create/fork project --- app/models/project.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index be1cecc0a..b1cd682f8 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -37,7 +37,7 @@ class Project < ActiveRecord::Base after_create :attach_to_personal_repository after_create :create_git_repo - after_create {|p| p.delay(:queue => 'fork', :priority => 20).fork_git_repo unless root?} + after_create {|p| p.delay(:queue => 'fork', :priority => 20).fork_git_repo unless is_root?} after_save :create_wiki after_destroy :destroy_git_repo @@ -215,7 +215,7 @@ class Project < ActiveRecord::Base end def create_git_repo - if root? + if is_root? Grit::Repo.init_bare(path) write_hook.delay(:queue => 'fork', :priority => 15) end