Merge pull request #376 from warpc/263-dj
[Refs #295] Customized DJ jobs and workers: * set priority to delayed jobs; * 4 workers to dj; * division init bare & fork.
This commit is contained in:
Vladimir Sharshov
commit
4bb8a38fbb
|
|
@ -70,6 +70,7 @@ class ActivityFeedObserver < ActiveRecord::Observer
|
|||
end
|
||||
|
||||
when 'GitHook'
|
||||
return unless record.project
|
||||
change_type = record.change_type
|
||||
branch_name = record.refname.split('/').last
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class Project < ActiveRecord::Base
|
|||
validates :name, :uniqueness => {:scope => [:owner_id, :owner_type], :case_sensitive => false}, :presence => true, :format => {:with => /^[a-zA-Z0-9_\-\+\.]+$/}
|
||||
validates :owner, :presence => true
|
||||
validate { errors.add(:base, :can_have_less_or_equal, :count => MAX_OWN_PROJECTS) if owner.projects.size >= MAX_OWN_PROJECTS }
|
||||
|
||||
|
||||
validates_attachment_size :srpm, :less_than => 500.megabytes
|
||||
validates_attachment_content_type :srpm, :content_type => ['application/octet-stream', "application/x-rpm", "application/x-redhat-package-manager"], :message => I18n.t('layout.invalid_content_type')
|
||||
|
||||
|
|
@ -37,11 +37,12 @@ class Project < ActiveRecord::Base
|
|||
|
||||
after_create :attach_to_personal_repository
|
||||
after_create :create_git_repo
|
||||
after_create {|p| p.delay(:queue => 'fork', :priority => 20).fork_git_repo unless root?}
|
||||
after_save :create_wiki
|
||||
|
||||
after_destroy :destroy_git_repo
|
||||
after_destroy :destroy_wiki
|
||||
after_save {|p| p.delay.import_attached_srpm if p.srpm?} # should be after create_git_repo
|
||||
after_save {|p| p.delay(:queue => 'import', :priority => 10).import_attached_srpm if p.srpm?} # should be after create_git_repo
|
||||
# after_rollback lambda { destroy_git_repo rescue true if new_record? }
|
||||
|
||||
has_ancestry
|
||||
|
|
@ -214,8 +215,15 @@ class Project < ActiveRecord::Base
|
|||
end
|
||||
|
||||
def create_git_repo
|
||||
is_root? ? Grit::Repo.init_bare(path) : parent.git_repository.repo.delay.fork_bare(path)
|
||||
write_hook.delay
|
||||
if root?
|
||||
Grit::Repo.init_bare(path)
|
||||
write_hook.delay(:queue => 'fork', :priority => 15)
|
||||
end
|
||||
end
|
||||
|
||||
def fork_git_repo
|
||||
dummy = Grit::Repo.new(path) rescue parent.git_repository.repo.fork_bare(path)
|
||||
write_hook
|
||||
end
|
||||
|
||||
def destroy_git_repo
|
||||
|
|
@ -248,7 +256,7 @@ class Project < ActiveRecord::Base
|
|||
hook = File.join(::Rails.root.to_s, 'tmp', "post-receive-hook")
|
||||
FileUtils.cp(File.join(::Rails.root.to_s, 'bin', "post-receive-hook.partial"), hook)
|
||||
File.open(hook, 'a') do |f|
|
||||
s = "\n /bin/bash -l -c \"cd #{is_production ? '/srv/rosa_build/current' : Rails.root.to_s} && #{is_production ? 'RAILS_ENV=production' : ''} bundle exec rails runner 'Project.delay.process_hook(\\\"$owner\\\", \\\"$reponame\\\", \\\"$newrev\\\", \\\"$oldrev\\\", \\\"$ref\\\", \\\"$newrev_type\\\", \\\"$oldrev_type\\\")'\""
|
||||
s = "\n /bin/bash -l -c \"cd #{is_production ? '/srv/rosa_build/current' : Rails.root.to_s} && #{is_production ? 'RAILS_ENV=production' : ''} bundle exec rails runner 'Project.delay(:queue => \\\"hook\\\").process_hook(\\\"$owner\\\", \\\"$reponame\\\", \\\"$newrev\\\", \\\"$oldrev\\\", \\\"$ref\\\", \\\"$newrev_type\\\", \\\"$oldrev_type\\\")'\""
|
||||
s << " > /dev/null 2>&1" if is_production
|
||||
s << "\ndone\n"
|
||||
f.write(s)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ set :default_environment, {
|
|||
|
||||
require 'rvm/capistrano'
|
||||
require 'bundler/capistrano'
|
||||
require 'delayed/recipes'
|
||||
require 'airbrake/capistrano'
|
||||
|
||||
set :whenever_command, "bundle exec whenever"
|
||||
|
|
@ -38,6 +37,7 @@ set :deploy_via, :remote_cache
|
|||
require 'lib/recipes/nginx'
|
||||
require 'lib/recipes/unicorn'
|
||||
require 'lib/recipes/bluepill'
|
||||
require 'lib/recipes/delayed_job'
|
||||
|
||||
namespace :deploy do
|
||||
task :stub_xml_rpc do
|
||||
|
|
@ -49,11 +49,11 @@ namespace :deploy do
|
|||
|
||||
task :symlink_all, :roles => :app do
|
||||
run "mkdir -p #{fetch :shared_path}/config"
|
||||
|
||||
|
||||
# Setup DB
|
||||
run "cp -n #{fetch :release_path}/config/database.yml.sample #{fetch :shared_path}/config/database.yml"
|
||||
run "ln -nfs #{fetch :shared_path}/config/database.yml #{fetch :release_path}/config/database.yml"
|
||||
|
||||
|
||||
# Setup application
|
||||
run "cp -n #{fetch :release_path}/config/deploy/application.#{fetch :stage}.yml #{fetch :shared_path}/config/application.yml"
|
||||
run "ln -nfs #{fetch :shared_path}/config/application.yml #{fetch :release_path}/config/application.yml"
|
||||
|
|
@ -66,7 +66,7 @@ namespace :deploy do
|
|||
task :symlink_pids, :roles => :app do
|
||||
run "cd #{fetch :shared_path}/tmp && ln -nfs ../pids pids"
|
||||
end
|
||||
|
||||
|
||||
# Speed up precompile (http://www.bencurtis.com/2011/12/skipping-asset-compilation-with-capistrano )
|
||||
# namespace :assets do
|
||||
# task :precompile, :roles => :web, :except => { :no_release => true } do
|
||||
|
|
@ -77,7 +77,7 @@ namespace :deploy do
|
|||
# logger.info "Skipping asset pre-compilation because there were no asset changes"
|
||||
# end
|
||||
# end
|
||||
# end
|
||||
# end
|
||||
end
|
||||
|
||||
after "deploy:finalize_update", "deploy:symlink_all"
|
||||
|
|
|
|||
|
|
@ -4,16 +4,18 @@ app_name = ENV['APP_NAME'] || 'rosa_build'
|
|||
Bluepill.application(app_name) do |app|
|
||||
app.uid = app.gid = 'rosa'
|
||||
app.working_dir = "/srv/#{app_name}/current"
|
||||
app.process("delayed_job") do |process|
|
||||
process.start_grace_time = 10.seconds
|
||||
process.stop_grace_time = 10.seconds
|
||||
process.restart_grace_time = 10.seconds
|
||||
%w(fork import hook default).each do |queue|
|
||||
app.process("delayed_job_#{queue}_queue") do |process|
|
||||
process.start_grace_time = 10.seconds
|
||||
process.stop_grace_time = 10.seconds
|
||||
process.restart_grace_time = 10.seconds
|
||||
|
||||
process.start_command = "/usr/bin/env ruby script/delayed_job start"
|
||||
process.stop_command = "/usr/bin/env ruby script/delayed_job stop"
|
||||
process.pid_file = File.join(app.working_dir, 'tmp', 'pids', 'delayed_job.pid')
|
||||
process.start_command = "/usr/bin/env ruby script/delayed_job --queue=#{queue} -p #{queue} --pid-dir=/srv/#{app_name}/current/tmp/#{queue}_pids start"
|
||||
process.stop_command = "/usr/bin/env ruby script/delayed_job --pid-dir=/srv/#{app_name}/current/tmp/#{queue}_pids stop"
|
||||
process.pid_file = File.join(app.working_dir, 'tmp', "#{queue}_pids", 'delayed_job.pid')
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
app.process("newrelic") do |process|
|
||||
process.start_grace_time = 10.seconds
|
||||
process.stop_grace_time = 10.seconds
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
class ModifyDefaultQueue < ActiveRecord::Migration
|
||||
def up
|
||||
change_column :delayed_jobs, :queue, :string, :default => 'default'
|
||||
execute "UPDATE delayed_jobs SET queue = 'default'"
|
||||
end
|
||||
|
||||
def down
|
||||
change_column :delayed_jobs, :queue, :string, :default => nil
|
||||
execute "UPDATE delayed_jobs SET queue = null"
|
||||
end
|
||||
end
|
||||
|
|
@ -24,9 +24,11 @@ Capistrano::Configuration.instance(:must_exist).load do
|
|||
run "cd #{fetch :current_path} && #{try_sudo} #{bluepill_binary} #{fetch :application} status"
|
||||
end
|
||||
|
||||
desc "Restart DJ process"
|
||||
desc "Restart DJ processes"
|
||||
task :restart_dj, :roles => [:app] do
|
||||
run "cd #{fetch :current_path} && #{try_sudo} #{bluepill_binary} #{fetch :application} restart delayed_job"
|
||||
%w(fork import hook default).each do |queue|
|
||||
run "cd #{fetch :current_path} && #{try_sudo} #{bluepill_binary} #{fetch :application} restart delayed_job_#{queue}_queue"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
# -*- encoding : utf-8 -*-
|
||||
Capistrano::Configuration.instance(:must_exist).load do
|
||||
namespace :delayed_job do
|
||||
def dj_queues
|
||||
%w(fork import hook default)
|
||||
end
|
||||
|
||||
def rails_env
|
||||
fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : ''
|
||||
end
|
||||
|
||||
def roles
|
||||
fetch(:delayed_job_server_role, :app)
|
||||
end
|
||||
|
||||
desc "Stop the delayed_job process"
|
||||
task :stop, :roles => lambda { roles } do
|
||||
dj_queues.each do |queue|
|
||||
run "cd #{current_path};#{rails_env} script/delayed_job --pid-dir=#{current_path}/tmp/#{queue}_pids stop"
|
||||
end
|
||||
end
|
||||
|
||||
desc "Start the delayed_job process"
|
||||
task :start, :roles => lambda { roles } do
|
||||
dj_queues.each do |queue|
|
||||
run "cd #{current_path};#{rails_env} script/delayed_job --queue=#{queue} -p #{queue} --pid-dir=#{current_path}/tmp/#{queue}_pids start"
|
||||
end
|
||||
end
|
||||
|
||||
desc "Restart the delayed_job process"
|
||||
task :restart, :roles => lambda { roles } do
|
||||
run "cd #{current_path};#{rails_env} script/delayed_job --queue=#{queue} -p #{queue} --pid-dir=#{current_path}/tmp/#{queue}_pids restart"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -6,7 +6,7 @@ namespace :hook do
|
|||
hook = File.join(::Rails.root.to_s, 'tmp', "post-receive-hook")
|
||||
FileUtils.cp(File.join(::Rails.root.to_s, 'bin', "post-receive-hook.partial"), hook)
|
||||
File.open(hook, 'a') do |f|
|
||||
s = "\n /bin/bash -l -c \"cd #{is_production ? '/srv/rosa_build/current' : Rails.root.to_s} && #{is_production ? 'RAILS_ENV=production' : ''} bundle exec rails runner 'Project.delay.process_hook(\\\"$owner\\\", \\\"$reponame\\\", \\\"$newrev\\\", \\\"$oldrev\\\", \\\"$ref\\\", \\\"$newrev_type\\\", \\\"$oldrev_type\\\")'\""
|
||||
s = "\n /bin/bash -l -c \"cd #{is_production ? '/srv/rosa_build/current' : Rails.root.to_s} && #{is_production ? 'RAILS_ENV=production' : ''} bundle exec rails runner 'Project.delay(:queue => \\\"hook\\\").process_hook(\\\"$owner\\\", \\\"$reponame\\\", \\\"$newrev\\\", \\\"$oldrev\\\", \\\"$ref\\\", \\\"$newrev_type\\\", \\\"$oldrev_type\\\")'\""
|
||||
s << " > /dev/null 2>&1" if is_production
|
||||
s << "\ndone\n"
|
||||
f.write(s)
|
||||
|
|
@ -18,8 +18,14 @@ namespace :hook do
|
|||
projects = ENV['project_id'] ? Project.where(:id => eval(ENV['project_id'])) : Project
|
||||
projects.where('created_at >= ?', Time.now.ago(ENV['period'] ? eval(ENV['period']) : 100.years)).each do |project|
|
||||
hook_file = File.join(project.path, 'hooks', 'post-receive')
|
||||
FileUtils.copy_entry(hook, hook_file, false, false, true)
|
||||
count = count + 1
|
||||
begin
|
||||
FileUtils.copy_entry(hook, hook_file, false, false, true)
|
||||
count = count + 1
|
||||
rescue Exception => e
|
||||
say "----\nCatching exception with project #{project.id}"
|
||||
say e.message
|
||||
say '----'
|
||||
end
|
||||
end
|
||||
say "Writing to #{count.to_s} repo(s)"
|
||||
say "Removing temporary file"
|
||||
|
|
|
|||
Loading…
Reference in New Issue