diff --git a/Gemfile b/Gemfile index 7e170aab8..eca994107 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'http://rubygems.org' -gem 'rails', '3.2.8' #, :git => 'git://github.com/rails/rails.git' +gem 'rails', '3.2.11' #, :git => 'git://github.com/rails/rails.git' gem 'pg', '~> 0.14.0' # gem 'silent-postgres', :git => 'git://github.com/dolzenko/silent-postgres.git' #'~> 0.1.1' diff --git a/Gemfile.lock b/Gemfile.lock index 09222ab96..1e85c7f5a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -25,31 +25,31 @@ GEM remote: http://rubygems.org/ specs: RedCloth (4.2.9) - actionmailer (3.2.8) - actionpack (= 3.2.8) + actionmailer (3.2.11) + actionpack (= 3.2.11) mail (~> 2.4.4) - actionpack (3.2.8) - activemodel (= 3.2.8) - activesupport (= 3.2.8) + actionpack (3.2.11) + activemodel (= 3.2.11) + activesupport (= 3.2.11) builder (~> 3.0.0) erubis (~> 2.7.0) journey (~> 1.0.4) rack (~> 1.4.0) rack-cache (~> 1.2) rack-test (~> 0.6.1) - sprockets (~> 2.1.3) - activemodel (3.2.8) - activesupport (= 3.2.8) + sprockets (~> 2.2.1) + activemodel (3.2.11) + activesupport (= 3.2.11) builder (~> 3.0.0) - activerecord (3.2.8) - activemodel (= 3.2.8) - activesupport (= 3.2.8) + activerecord (3.2.11) + activemodel (= 3.2.11) + activesupport (= 3.2.11) arel (~> 3.0.2) tzinfo (~> 0.3.29) - activeresource (3.2.8) - activemodel (= 3.2.8) - activesupport (= 3.2.8) - activesupport (3.2.8) + activeresource (3.2.11) + activemodel (= 3.2.11) + activesupport (= 3.2.11) + activesupport (3.2.11) i18n (~> 0.6) multi_json (~> 1.0) airbrake (3.1.2) @@ -65,7 +65,7 @@ GEM daemons (~> 1.1.4, <= 1.1.6) i18n (>= 0.5.0) state_machine (~> 1.1.0) - builder (3.0.0) + builder (3.0.4) cancan (1.6.7) cape (1.4.0) capistrano (2.12.0) @@ -144,14 +144,14 @@ GEM highline (1.6.13) hike (1.2.1) hirb (0.7.0) - i18n (0.6.0) + i18n (0.6.1) jbuilder (0.8.2) activesupport (>= 3.0.0) journey (1.0.4) jquery-rails (2.0.2) railties (>= 3.2.0, < 5.0) thor (~> 0.14) - json (1.7.4) + json (1.7.6) kgio (2.7.4) libv8 (3.3.10.4) macaddr (1.6.1) @@ -173,7 +173,7 @@ GEM actionpack mime-types (1.19) mock_redis (0.6.2) - multi_json (1.3.6) + multi_json (1.5.0) mustache (0.99.4) net-scp (1.0.4) net-ssh (>= 1.99.1) @@ -206,7 +206,7 @@ GEM posix-spawn (0.3.6) pygments.rb (0.2.13) rubypython (~> 0.5.3) - rack (1.4.1) + rack (1.4.3) rack-cache (1.2) rack (>= 0.4) rack-openid (1.3.1) @@ -216,18 +216,18 @@ GEM rack rack-ssl (1.3.2) rack - rack-test (0.6.1) + rack-test (0.6.2) rack (>= 1.0) rack-throttle (0.3.0) rack (>= 1.0.0) - rails (3.2.8) - actionmailer (= 3.2.8) - actionpack (= 3.2.8) - activerecord (= 3.2.8) - activeresource (= 3.2.8) - activesupport (= 3.2.8) + rails (3.2.11) + actionmailer (= 3.2.11) + actionpack (= 3.2.11) + activerecord (= 3.2.11) + activeresource (= 3.2.11) + activesupport (= 3.2.11) bundler (~> 1.0) - railties (= 3.2.8) + railties (= 3.2.11) rails-backbone (0.7.2) coffee-script (~> 2.2.0) ejs (~> 1.0.0) @@ -236,15 +236,15 @@ GEM railties (>= 3.0.0) rails3-jquery-autocomplete (1.0.7) rails (~> 3.0) - railties (3.2.8) - actionpack (= 3.2.8) - activesupport (= 3.2.8) + railties (3.2.11) + actionpack (= 3.2.11) + activesupport (= 3.2.11) rack-ssl (~> 1.3.2) rake (>= 0.8.7) rdoc (~> 3.4) thor (>= 0.14.6, < 2.0) raindrops (0.10.0) - rake (0.9.2.2) + rake (10.0.3) rdiscount (1.6.8) rdoc (3.12) json (~> 1.4) @@ -315,8 +315,9 @@ GEM skinny (0.2.1) eventmachine (~> 0.12) thin (~> 1.2) - sprockets (2.1.3) + sprockets (2.2.2) hike (~> 1.2) + multi_json (~> 1.0) rack (~> 1.0) tilt (~> 1.1, != 1.3.0) sqlite3 (1.3.6) @@ -331,10 +332,10 @@ GEM rack (>= 1.0.0) thor (0.16.0) tilt (1.3.3) - treetop (1.4.10) + treetop (1.4.12) polyglot polyglot (>= 0.3.1) - tzinfo (0.3.33) + tzinfo (0.3.35) uglifier (1.2.7) execjs (>= 0.3.0) multi_json (~> 1.3) @@ -395,7 +396,7 @@ DEPENDENCIES perform_later (~> 1.3.0) pg (~> 0.14.0) rack-throttle - rails (= 3.2.8) + rails (= 3.2.11) rails-backbone (~> 0.7.2) rails3-generators rails3-jquery-autocomplete (~> 1.0.7) diff --git a/app/models/build_list.rb b/app/models/build_list.rb index 6f2867c76..4f237eb93 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -273,28 +273,37 @@ class BuildList < ActiveRecord::Base def publish_container type = build_for_platform.distrib_type - archive = results.select{ |r| r['file_name'] =~ /.*\.tar\.gz$/}[0] + archive = results.find{ |r| r['file_name'] =~ /.*\.tar\.gz$/ } - platform_path = "#{APP_CONFIG['root_path']}/platforms/#{save_to_platform.name}/repository" + platform_path = "#{save_to_platform.path}/repository" if save_to_platform.personal? platform_path << '/' platform_path << build_for_platform.name Dir.mkdir(platform_path) unless File.exists?(platform_path) end + packages = last_published.includes(:packages).limit(5).map{ |bl| bl.packages }.flatten + sources = packages.map{ |p| p.fullname if p.package_type == 'source' }.compact + binaries = packages.map{ |p| p.fullname if p.package_type == 'binary' }.compact + Resque.push( - "publish_build_list_container_#{type}_worker", - 'class' => "AbfWorker::PublishBuildListContainer#{type.capitalize}Worker", + worker_queue_with_priority("publish_#{type}_worker"), + 'class' => worker_queue_class("AbfWorker::Publish#{type.capitalize}Worker"), 'args' => [{ :id => id, :arch => arch.name, :distrib_type => type, :container_sha1 => archive['sha1'], + :packages => { :sources => sources, :binaries => binaries }, :platform => { :platform_path => platform_path, :released => save_to_platform.released }, - :repository_name => save_to_repository.name, + :repository => { + :name => save_to_repository.name, + :id => save_to_repository.id + }, + :type => :publish, :time_living => 2400 # 40 min }] ) @@ -416,6 +425,14 @@ class BuildList < ActiveRecord::Base protected + def abf_worker_priority + mass_build_id ? '' : 'default' + end + + def abf_worker_base_queue + 'rpm_worker' + end + def abf_worker_args include_repos_hash = {}.tap do |h| include_repos.each do |r| diff --git a/app/models/product_build_list.rb b/app/models/product_build_list.rb index 7b28899f3..7142d3f7f 100644 --- a/app/models/product_build_list.rb +++ b/app/models/product_build_list.rb @@ -129,6 +129,14 @@ class ProductBuildList < ActiveRecord::Base protected + def abf_worker_priority + '' + end + + def abf_worker_base_queue + 'iso_worker' + end + def abf_worker_args file_name = "#{project.owner.uname}-#{project.name}-#{commit_hash}" opts = {:host => ActionMailer::Base.default_url_options[:host]} diff --git a/app/views/projects/build_lists/index.html.haml b/app/views/projects/build_lists/index.html.haml index 67d8a0622..27fa5c909 100644 --- a/app/views/projects/build_lists/index.html.haml +++ b/app/views/projects/build_lists/index.html.haml @@ -1,5 +1,6 @@ -set_meta_tags :title => t('.title') += hidden_field_tag :servertime, Time.now.utc.to_i %div.reloader = label_tag :autoreload do = check_box_tag :autoreload, true, true @@ -46,7 +47,7 @@ .replace("%d", formatNumber(minutes)); } // TODO Very, very ugly method. - var now = (new Date).getTime(); + var now = $('#servertime').val() * 1000; $("time.js-relative-date").each(function() { var time = parseIso8601($(this).attr('datetime')).getTime(); $(this).text(strTimeBetween(time, now)); diff --git a/lib/abf_worker/model_helper.rb b/lib/abf_worker/model_helper.rb index 44994e49e..2d240f01f 100644 --- a/lib/abf_worker/model_helper.rb +++ b/lib/abf_worker/model_helper.rb @@ -5,12 +5,12 @@ module AbfWorker # - #build_canceled def abf_worker_log - Resque.redis.get("abfworker::#{worker_queue('-')}-#{id}") || I18n.t('layout.build_lists.log.not_available') + Resque.redis.get(service_queue) || I18n.t('layout.build_lists.log.not_available') end def add_job_to_abf_worker_queue Resque.push( - worker_queue, + worker_queue_with_priority, 'class' => worker_queue_class, 'args' => [abf_worker_args] ) @@ -18,7 +18,7 @@ module AbfWorker def cancel_job deleted = Resque::Job.destroy( - worker_queue, + worker_queue_with_priority, worker_queue_class, abf_worker_args ) @@ -30,34 +30,31 @@ module AbfWorker true end + protected + + def worker_queue_with_priority(queue = nil) + queue ||= abf_worker_base_queue + queue << '_' << abf_worker_priority if abf_worker_priority.present? + queue + end + + def worker_queue_class(queue_class = nil) + queue_class ||= "AbfWorker::#{abf_worker_base_queue.classify}" + queue_class << abf_worker_priority.capitalize + end + private def send_stop_signal Resque.redis.setex( - live_inspector_queue, + "#{service_queue}::live-inspector", 240, # Data will be removed from Redis after 240 sec. 'USR1' # Immediately kill child but don't exit ) end - def live_inspector_queue - q = 'abfworker::' - q << worker_queue('-') - q << '-' - q << id.to_s - q << '::live-inspector' - q - end - - def worker_queue(delimiter = '_') - a = [] - a << (is_a?(BuildList) ? 'rpm' : 'iso') - a << 'worker' - a.join(delimiter) - end - - def worker_queue_class - is_a?(BuildList) ? 'AbfWorker::RpmWorker' : 'AbfWorker::IsoWorker' + def service_queue + "abfworker::#{abf_worker_base_queue.gsub(/\_/, '-')}-#{id}" end end diff --git a/lib/abf_worker/publish_build_list_container_observer.rb b/lib/abf_worker/publish_observer.rb similarity index 79% rename from lib/abf_worker/publish_build_list_container_observer.rb rename to lib/abf_worker/publish_observer.rb index a06012950..59bf86866 100644 --- a/lib/abf_worker/publish_build_list_container_observer.rb +++ b/lib/abf_worker/publish_observer.rb @@ -1,6 +1,6 @@ module AbfWorker - class PublishBuildListContainerObserver < AbfWorker::BaseObserver - @queue = :publish_build_list_container_observer + class PublishObserver < AbfWorker::BaseObserver + @queue = :publish_observer def self.perform(options) bl = BuildList.find options['id'] @@ -20,7 +20,7 @@ module AbfWorker def self.update_results(subject, options) results = (subject.results || []). - map{ |r| r if r['file_name'] !~ /^abfworker\:\:publish\-build\-list\-container\-worker.*\.log$/ }. + map{ |r| r if r['file_name'] !~ /^abfworker\:\:publish\-worker.*\.log$/ }. compact results += options['results'] sort_results_and_save(subject, results) diff --git a/lib/recipes/resque.rb b/lib/recipes/resque.rb index 3bac324f5..41c2254c9 100644 --- a/lib/recipes/resque.rb +++ b/lib/recipes/resque.rb @@ -26,7 +26,7 @@ Capistrano::Configuration.instance(:must_exist).load do end def start_workers - run "cd #{fetch :current_path} && COUNT=#{workers_count} QUEUE=fork_import,hook,clone_build,notification,iso_worker_observer,rpm_worker_observer,publish_build_list_container_observer #{rails_env} BACKGROUND=yes bundle exec rake resque:workers" + run "cd #{fetch :current_path} && COUNT=#{workers_count} QUEUE=fork_import,hook,clone_build,notification,iso_worker_observer,rpm_worker_observer,publish_observer #{rails_env} BACKGROUND=yes bundle exec rake resque:workers" end end end diff --git a/lib/tasks/new_core.rake b/lib/tasks/new_core.rake index f61689b96..953e1b251 100644 --- a/lib/tasks/new_core.rake +++ b/lib/tasks/new_core.rake @@ -14,4 +14,44 @@ namespace :new_core do say "[#{Time.zone.now}] done" end + + desc 'Publish mass-build 73' + task :publish_mass_build_73 => :environment do + say "[#{Time.zone.now}] Starting to publish mass-build 317..." + + bl = BuildList.where(:mass_build_id => 73).first + platform_repository_folder = "#{bl.save_to_platform.path}/repository" + BuildList.where(:mass_build_id => 73). + where(:status => [ + BuildServer::SUCCESS, + BuildList::FAILED_PUBLISH + ]). + order(:id). + find_in_batches(:batch_size => 1) do | bls | + + bl = bls.first + puts "[#{Time.zone.now}] - where build_lists.id #{bl.id}" + + sha1 = bl.results.find{ |r| r['file_name'] =~ /.*\.tar\.gz$/ }['sha1'] + + system "cd #{platform_repository_folder} && curl -L -O http://file-store.rosalinux.ru/api/v1/file_stores/#{sha1}" + system "cd #{platform_repository_folder} && tar -xzf #{sha1}" + system "rm -f #{platform_repository_folder}/#{sha1}" + + archive_folder = "#{platform_repository_folder}/archives" + system "sudo chown root:root #{archive_folder}/SRC_RPM/*" + system "sudo chmod 0666 #{archive_folder}/SRC_RPM/*" + system "sudo chown root:root #{archive_folder}/RPM/*" + system "sudo chmod 0666 #{archive_folder}/RPM/*" + + system "sudo mv #{archive_folder}/SRC_RPM/* #{platform_repository_folder}/SRPMS/main/release/" + system "sudo mv #{archive_folder}/RPM/* #{platform_repository_folder}/#{bl.arch.name}/main/release/" + + system "sudo rm -rf #{archive_folder}" + bl.update_column(:status, BuildList::BUILD_PUBLISH) + end + + say "[#{Time.zone.now}] done" + end + end \ No newline at end of file diff --git a/spec/controllers/platforms/product_build_lists_controller_spec.rb b/spec/controllers/platforms/product_build_lists_controller_spec.rb index 0a0330aa7..afecf80fa 100644 --- a/spec/controllers/platforms/product_build_lists_controller_spec.rb +++ b/spec/controllers/platforms/product_build_lists_controller_spec.rb @@ -84,6 +84,7 @@ end describe Platforms::ProductBuildListsController do before(:each) do stub_symlink_methods + stub_redis end context 'crud' do diff --git a/spec/controllers/projects/git/git_trees_controller_spec.rb b/spec/controllers/projects/git/git_trees_controller_spec.rb index 2150e244d..397e78cee 100644 --- a/spec/controllers/projects/git/git_trees_controller_spec.rb +++ b/spec/controllers/projects/git/git_trees_controller_spec.rb @@ -15,6 +15,7 @@ describe Projects::Git::TreesController do context 'for guest' do it 'should be able to perform archive action with anonymous acccess', :anonymous_access => true do + stub(controller).render fill_project @project get :archive, @params.merge(:format => 'tar.gz') response.should be_success @@ -49,6 +50,7 @@ describe Projects::Git::TreesController do end it 'should be able to perform archive action' do + stub(controller).render fill_project @project @user = FactoryGirl.create(:user) set_session_for(@user) diff --git a/spec/factories/build_lists.rb b/spec/factories/build_lists.rb index daadeee0e..233f19567 100644 --- a/spec/factories/build_lists.rb +++ b/spec/factories/build_lists.rb @@ -4,7 +4,11 @@ FactoryGirl.define do association :user #association :project association :save_to_platform, :factory => :platform_with_repos - project { |bl| FactoryGirl.create(:project_with_commit, :repositories => [bl.save_to_platform.repositories.first]) } + project { |bl| + pr = FactoryGirl.create(:project_with_commit) + bl.save_to_platform.repositories.first.projects << pr + pr + } association :arch build_for_platform {|bl| bl.save_to_platform} save_to_repository {|bl| bl.save_to_platform.repositories.first} @@ -16,7 +20,7 @@ FactoryGirl.define do factory :build_list_with_attaching_project, :parent => :build_list do before(:create) { |bl| attach_project_to_build_list bl } - end + end factory :build_list_core, :parent => :build_list do bs_id { FactoryGirl.generate(:integer) } @@ -27,7 +31,11 @@ FactoryGirl.define do end factory :build_list_by_group_project, :parent => :build_list_core do - project { |bl| FactoryGirl.create(:group_project_with_commit, :repositories => [bl.save_to_platform.repositories.first]) } + project { |bl| + pr = FactoryGirl.create(:group_project_with_commit) + bl.save_to_platform.repositories.first.projects << pr + pr + } end factory :build_list_package, :class => BuildList::Package do @@ -45,4 +53,4 @@ end def attach_project_to_build_list(bl) bl.save_to_platform ||= FactoryGirl.create(:platform_with_repos) bl.project.repositories << bl.save_to_platform.repositories.first -end \ No newline at end of file +end