diff --git a/app/controllers/projects/projects_controller.rb b/app/controllers/projects/projects_controller.rb index cd92f47fd..1ec982a6e 100644 --- a/app/controllers/projects/projects_controller.rb +++ b/app/controllers/projects/projects_controller.rb @@ -3,6 +3,7 @@ class Projects::ProjectsController < Projects::BaseController include ProjectsHelper before_filter :authenticate_user! load_and_authorize_resource :id_param => :project_name, :collection => :possible_forks # to force member actions load + before_filter :who_owns, :only => [:new, :create, :mass_import, :run_mass_import] def index @projects = Project.accessible_by(current_ability, :membered) @@ -24,7 +25,27 @@ class Projects::ProjectsController < Projects::BaseController def new @project = Project.new - @who_owns = :me + end + + def mass_import + @project = Project.new(:mass_import => true) + end + + def run_mass_import + @project = Project.new params[:project] + @project.owner = choose_owner + authorize! :write, @project.owner if @project.owner.class == Group + authorize! :add_project, Repository.find(params[:project][:add_to_repository_id]) + @project.valid? + @project.errors.messages.slice! :url + if @project.errors.messages.blank? # We need only url validation + @project.init_mass_import + flash[:notice] = t('flash.project.mass_import_added_to_queue') + redirect_to projects_path + else + flash[:warning] = @project.errors.full_messages.join('. ') + render :mass_import + end end def edit @@ -33,7 +54,6 @@ class Projects::ProjectsController < Projects::BaseController def create @project = Project.new params[:project] @project.owner = choose_owner - @who_owns = (@project.owner_type == 'User' ? :me : :group) authorize! :write, @project.owner if @project.owner.class == Group if @project.save @@ -120,6 +140,10 @@ class Projects::ProjectsController < Projects::BaseController protected + def who_owns + @who_owns = (@project.try(:owner_type) == 'User' ? :me : :group) + end + def prepare_list(projects, groups, owners) res = {} diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 1c837c144..8e1cd9082 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -20,6 +20,15 @@ module ProjectsHelper end.sort_by{ |f| f[:uname] } end + def repositories_grouped_by_platform + groups = {} + Platform.accessible_by(current_ability, :related).order(:name).each do |platform| + next unless can?(:local_admin_manage, platform) + groups[platform.name] = Repository.custom_sort(platform.repositories).map{ |r| [r.name, r.id] } + end + groups + end + def git_repo_url(name) if current_user "#{request.protocol}#{current_user.uname}@#{request.host_with_port}/#{name}.git" diff --git a/app/models/ability.rb b/app/models/ability.rb index 518bd556c..30cde50a3 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -60,6 +60,7 @@ class Ability can :remove_user, Group can :create, Project + can([:mass_import, :run_mass_import], Project) if user.platforms.main.find{ |p| local_admin?(p) }.present? can :read, Project, :visibility => 'open' can [:read, :archive, :membered, :get_id], Project, :owner_type => 'User', :owner_id => user.id can [:read, :archive, :membered, :get_id], Project, :owner_type => 'Group', :owner_id => user.group_ids diff --git a/app/models/build_list.rb b/app/models/build_list.rb index f4a3260a4..5bf4216c4 100644 --- a/app/models/build_list.rb +++ b/app/models/build_list.rb @@ -427,12 +427,12 @@ class BuildList < ActiveRecord::Base new_core? ? abf_worker_log : I18n.t('layout.build_lists.log.not_available') end - def last_published + def last_published(testing = false) BuildList.where(:project_id => self.project_id, :save_to_repository_id => self.save_to_repository_id) .for_platform(self.build_for_platform_id) .scoped_to_arch(self.arch_id) - .for_status(BUILD_PUBLISHED) + .for_status(testing ? BUILD_PUBLISHED_INTO_TESTING : BUILD_PUBLISHED) .recent end diff --git a/app/models/key_pair.rb b/app/models/key_pair.rb index 17fb6c708..4c95c0d14 100644 --- a/app/models/key_pair.rb +++ b/app/models/key_pair.rb @@ -19,7 +19,7 @@ class KeyPair < ActiveRecord::Base protected def check_keys - dir = Dir.mktmpdir('keys-', '/tmp') + dir = Dir.mktmpdir 'keys-', APP_CONFIG['tmpfs_path'] begin %w(pubring secring).each do |kind| filename = "#{dir}/#{kind}" diff --git a/app/models/project.rb b/app/models/project.rb index 3b2be1721..019e9fb23 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -29,8 +29,11 @@ class Project < ActiveRecord::Base validates :name, :uniqueness => {:scope => [:owner_id, :owner_type], :case_sensitive => false}, :presence => true, - :format => {:with => /\A#{NAME_REGEXP}\z/, :message => I18n.t("activerecord.errors.project.uname")} + :format => {:with => /\A#{NAME_REGEXP}\z/, + :message => I18n.t("activerecord.errors.project.uname")} validates :maintainer_id, :presence => true, :unless => :new_record? + validates :url, :presence => true, :format => {:with => /\Ahttps?:\/\/[\S]+\z/}, :if => :mass_import + validates :add_to_repository_id, :presence => true, :if => :mass_import validates :visibility, :presence => true, :inclusion => {:in => VISIBILITIES} validate { errors.add(:base, :can_have_less_or_equal, :count => MAX_OWN_PROJECTS) if owner.projects.size >= MAX_OWN_PROJECTS } validate :check_default_branch @@ -43,7 +46,9 @@ class Project < ActiveRecord::Base errors.delete :project_to_repositories end - attr_accessible :name, :description, :visibility, :srpm, :is_package, :default_branch, :has_issues, :has_wiki, :maintainer_id, :publish_i686_into_x86_64 + attr_accessible :name, :description, :visibility, :srpm, :is_package, :default_branch, + :has_issues, :has_wiki, :maintainer_id, :publish_i686_into_x86_64, + :url, :srpms_list, :mass_import, :add_to_repository_id attr_readonly :owner_id, :owner_type scope :recent, order("lower(#{table_name}.name) ASC") @@ -82,6 +87,8 @@ class Project < ActiveRecord::Base has_ancestry :orphan_strategy => :rootify #:adopt not available yet + attr_accessor :url, :srpms_list, :mass_import, :add_to_repository_id + include Modules::Models::Owner include Modules::Models::Git include Modules::Models::Wiki @@ -98,6 +105,10 @@ class Project < ActiveRecord::Base end end + def init_mass_import + Project.perform_later :clone_build, :run_mass_import, url, srpms_list, visibility, owner, add_to_repository_id + end + def name_with_owner "#{owner_uname || owner.uname}/#{name}" end diff --git a/app/views/projects/projects/_filters.html.haml b/app/views/projects/projects/_filters.html.haml index 12577ac68..6b578a3b1 100644 --- a/app/views/projects/projects/_filters.html.haml +++ b/app/views/projects/projects/_filters.html.haml @@ -6,6 +6,10 @@ - if can?(:create, Project) .bordered.bpadding20 = link_to t('layout.projects.new'), new_project_path, :class => 'button' + - if can?(:mass_import, Project) + .both + %br + = link_to t('layout.projects.mass_import'), mass_import_projects_path, :class => 'button' .bordered.bpadding20 %h3=t('layout.relations.filters') - options_for_filters(@all_projects, @groups, @owners).each do |options| diff --git a/app/views/projects/projects/_form.html.haml b/app/views/projects/projects/_form.html.haml index c009d642f..04b3f4cb9 100644 --- a/app/views/projects/projects/_form.html.haml +++ b/app/views/projects/projects/_form.html.haml @@ -7,22 +7,10 @@ .rightlist= f.text_area :description, :class => 'text_field', :cols => 80 .both - if [:new, :create].include? act - .leftlist= f.label :owner - .rightlist - = label_tag t("activerecord.attributes.project.who_owns.me") - - if Group.can_own_project(current_user).count > 0 - = radio_button_tag :who_owns, 'me', @who_owns == :me #{}.merge( (@who_owns == :me) ? {:checked => 'checked'} : {} ) - = label_tag t("activerecord.attributes.project.who_owns.group") - = radio_button_tag :who_owns, 'group', @who_owns == :group #{}.merge( (@who_owns == :group) ? {:checked => 'checked'} : {} ) - -# TODO: Make our own select_box helper with new design, blackjack and bitches! - = select_tag :owner_id, options_from_collection_for_select( Group.can_own_project(current_user), :id, :name ) - - else - = hidden_field_tag :who_owns, :me - .both + = render 'owner', :f => f .leftlist= f.label :visibility .rightlist - =# f.select :visibility, Project::VISIBILITIES - Project::VISIBILITIES.each do |visibility| = f.radio_button :visibility, visibility, :class => 'niceRadio' - if visibility == 'open' diff --git a/app/views/projects/projects/_owner.html.haml b/app/views/projects/projects/_owner.html.haml new file mode 100644 index 000000000..3d1d19653 --- /dev/null +++ b/app/views/projects/projects/_owner.html.haml @@ -0,0 +1,12 @@ +.leftlist= f.label :owner +.rightlist + = label_tag t("activerecord.attributes.project.who_owns.me") + - if Group.can_own_project(current_user).count > 0 + = radio_button_tag :who_owns, 'me', @who_owns == :me #{}.merge( (@who_owns == :me) ? {:checked => 'checked'} : {} ) + = label_tag t("activerecord.attributes.project.who_owns.group") + = radio_button_tag :who_owns, 'group', @who_owns == :group #{}.merge( (@who_owns == :group) ? {:checked => 'checked'} : {} ) + -# TODO: Make our own select_box helper with new design, blackjack and bitches! + = select_tag :owner_id, options_from_collection_for_select( Group.can_own_project(current_user), :id, :name ) + - else + = hidden_field_tag :who_owns, :me +.both \ No newline at end of file diff --git a/app/views/projects/projects/mass_import.html.haml b/app/views/projects/projects/mass_import.html.haml new file mode 100644 index 000000000..796ca274a --- /dev/null +++ b/app/views/projects/projects/mass_import.html.haml @@ -0,0 +1,27 @@ +%h3.bpadding10= title t("layout.projects.mass_import") += form_for @project, :url => run_mass_import_projects_path, :html => { :class => :form } do |f| + = f.hidden_field :mass_import + .leftlist= f.label :url + .rightlist= f.text_field :url + .both + .leftlist= f.label :srpms_list + .rightlist= f.text_area :srpms_list + .both + .leftlist= f.label :add_to_repository_id + .rightlist= f.select :add_to_repository_id, repositories_grouped_by_platform + .both + = render 'owner', :f => f + + .leftlist= f.label :visibility + .rightlist + - Project::VISIBILITIES.each do |visibility| + = f.radio_button :visibility, visibility, :class => 'niceRadio' + - if visibility == 'open' + = image_tag("unlock.png") + - else + = image_tag("lock.png") + = t("activerecord.attributes.project.visibilities.#{visibility}") + .both + .hr + .button_block + = f.submit t('layout.add'), :data => {'disable-with' => t('layout.saving')} \ No newline at end of file diff --git a/config/application.yml.sample b/config/application.yml.sample index d20d851d3..f2824c609 100644 --- a/config/application.yml.sample +++ b/config/application.yml.sample @@ -57,6 +57,7 @@ development: <<: *common root_path: /var/rosa git_path: /var/rosa + tmpfs_path: /dev/shm do-not-reply-email: do-not-reply@localhost github_services: ip: 127.0.0.1 @@ -66,6 +67,7 @@ production: <<: *common root_path: /share git_path: /mnt/gitstore + tmpfs_path: /dev/shm do-not-reply-email: do-not-reply@abf.rosalinux.ru mailer_https_url: false github_services: @@ -74,6 +76,7 @@ production: test: <<: *common + tmpfs_path: "use Rails.root/tmp/test_root in spec" root_path: "use Rails.root/tmp/test_root in spec" git_path: "use Rails.root/tmp/test_root in spec" do-not-reply-email: do-not-reply@localhost diff --git a/config/locales/models/project.en.yml b/config/locales/models/project.en.yml index d616729d1..586f0c5fb 100644 --- a/config/locales/models/project.en.yml +++ b/config/locales/models/project.en.yml @@ -1,6 +1,7 @@ en: layout: projects: + mass_import: Mass import branches: Branches delete_branch: Delete branch restore_branch: Restore branch @@ -76,6 +77,7 @@ en: flash: project: + mass_import_added_to_queue: Mass import added to queue saved: Project saved save_error: Unable to save project save_warning_ssh_key: Project owner must provide a SSH key in his profile @@ -89,6 +91,9 @@ en: project: Project attributes: project: + url: URL + add_to_repository_id: Add to repository + srpms_list: SRPMs list name: Name description: Descripton owner: Owner diff --git a/config/locales/models/project.ru.yml b/config/locales/models/project.ru.yml index 5360e237c..12821d64c 100644 --- a/config/locales/models/project.ru.yml +++ b/config/locales/models/project.ru.yml @@ -1,6 +1,7 @@ ru: layout: projects: + mass_import: Массовый импорт branches: Ветки delete_branch: Удалить ветку restore_branch: Восстановить ветку @@ -76,6 +77,7 @@ ru: flash: project: + mass_import_added_to_queue: Массовый импорт добавлен в очередь saved: Проект успешно сохранен save_error: Не удалось сохранить проект save_warning_ssh_key: Владельцу проекта необходимо указать в профиле свой SSH ключ @@ -89,6 +91,9 @@ ru: project: Проект attributes: project: + url: URL + add_to_repository_id: Добавить в репозиторий + srpms_list: Список SRPMs name: Название description: Описание owner: Владелец diff --git a/config/routes.rb b/config/routes.rb index cdfc771e6..a5833f914 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -284,7 +284,12 @@ Rosa::Application.routes.draw do end end - resources :projects, :only => [:index, :new, :create] + resources :projects, :only => [:index, :new, :create] do + collection do + post :run_mass_import + get :mass_import + end + end scope ':owner_name/:project_name', :constraints => {:project_name => Project::NAME_REGEXP} do # project scope :as => 'project' do resources :wiki do diff --git a/lib/abf_worker/build_lists_publish_task_manager.rb b/lib/abf_worker/build_lists_publish_task_manager.rb index c98d6fcc5..2e4996695 100644 --- a/lib/abf_worker/build_lists_publish_task_manager.rb +++ b/lib/abf_worker/build_lists_publish_task_manager.rb @@ -132,7 +132,7 @@ module AbfWorker old_packages = packages_structure build_lists_for_cleanup.each do |bl| - bl.last_published.includes(:packages).limit(2).each{ |old_bl| + bl.last_published(testing).includes(:packages).limit(2).each{ |old_bl| fill_packages(old_bl, old_packages, :fullname) } end @@ -309,7 +309,7 @@ module AbfWorker # remove duplicates of sources for different arches bl.packages.by_package_type('source').each{ |s| new_sources["#{s.fullname}"] = s.sha1 } self.class.fill_packages(bl, packages) - bl.last_published.includes(:packages).limit(2).each{ |old_bl| + bl.last_published(testing).includes(:packages).limit(2).each{ |old_bl| self.class.fill_packages(old_bl, old_packages, :fullname) } build_list_ids << bl.id diff --git a/lib/modules/models/git.rb b/lib/modules/models/git.rb index 246652074..c60ca5869 100644 --- a/lib/modules/models/git.rb +++ b/lib/modules/models/git.rb @@ -1,4 +1,8 @@ # -*- encoding : utf-8 -*- +require 'nokogiri' +require 'open-uri' +require 'iconv' + module Modules module Models module Git @@ -173,10 +177,70 @@ module Modules end module ClassMethods + MAX_SRC_SIZE = 1024*1024*256 + def process_hook(owner_uname, repo, newrev, oldrev, ref, newrev_type, user = nil, message = nil) rec = GitHook.new(owner_uname, repo, newrev, oldrev, ref, newrev_type, user, message) Modules::Observers::ActivityFeed::Git.create_notifications rec end + + def run_mass_import(url, srpms_list, visibility, owner, add_to_repository_id) + doc = Nokogiri::HTML(open(url)) + links = doc.css("a[href$='.src.rpm']") + return if links.count == 0 + filter = srpms_list.lines.map(&:chomp).map(&:strip).select(&:present?) + + repository = Repository.find add_to_repository_id + platform = repository.platform + dir = Dir.mktmpdir 'mass-import-', APP_CONFIG['tmpfs_path'] + links.each do |link| + begin + package = link.attributes['href'].value + package.chomp!; package.strip! + + next if package.size == 0 || package !~ /^[\w\.\-]+$/ + next if filter.present? && !filter.include?(package) + + uri = URI "#{url}/#{package}" + srpm_file = "#{dir}/#{package}" + Net::HTTP.start(uri.host) do |http| + if http.request_head(uri.path)['content-length'].to_i < MAX_SRC_SIZE + f = open(srpm_file, 'wb') + http.request_get(uri.path) do |resp| + resp.read_body{ |segment| f.write(segment) } + end + f.close + end + end + if name = `rpm -q --qf '[%{Name}]' -p #{srpm_file}` and $?.success? and name.present? + next if owner.projects.exists?(:name => name) + description = ::Iconv.conv('UTF-8//IGNORE', 'UTF-8', `rpm -q --qf '[%{Description}]' -p #{srpm_file}`) + project = owner.projects.build( + :name => name, + :description => description, + :visibility => visibility, + :is_package => false # See: Hook for #attach_to_personal_repository + ) + project.owner = owner + if project.save + repository.projects << project rescue nil + project.update_attributes(:is_package => true) + project.import_srpm srpm_file, platform.name + end + end + rescue => e + f.close if defined?(f) + Airbrake.notify_or_ignore(e, :link => link.to_s, :url => url, :owner => owner) + ensure + File.delete srpm_file if srpm_file + end + end + rescue => e + Airbrake.notify_or_ignore(e, :url => url, :owner => owner) + ensure + FileUtils.remove_entry_secure dir if dir + end + end end end diff --git a/spec/models/cancan_spec.rb b/spec/models/cancan_spec.rb index 3e6a8377a..ec1d16d33 100644 --- a/spec/models/cancan_spec.rb +++ b/spec/models/cancan_spec.rb @@ -65,6 +65,12 @@ describe CanCan do end end + [:mass_import, :run_mass_import].each do |action| + it "should not be able to #{ action } project" do + @ability.should_not be_able_to(action, Project) + end + end + it 'should not be able to update register request' do @ability.should_not be_able_to(:update, register_request) end @@ -93,6 +99,12 @@ describe CanCan do end end + [:mass_import, :run_mass_import].each do |action| + it "should not be able to #{ action } project" do + @ability.should_not be_able_to(action, Project) + end + end + it "shoud be able to show user profile" do @ability.should be_able_to(:show, User) end @@ -260,6 +272,13 @@ describe CanCan do before(:each) do @platform.owner = @user @platform.save + @ability = Ability.new(@user) + end + + [:mass_import, :run_mass_import].each do |action| + it "should be able to #{ action } project" do + @ability.should be_able_to(action, Project) + end end [:read, :update, :destroy, :change_visibility].each do |action| @@ -272,6 +291,13 @@ describe CanCan do context 'with read rights' do before(:each) do @platform.relations.create!(:actor_id => @user.id, :actor_type => 'User', :role => 'reader') + @ability = Ability.new(@user) + end + + [:mass_import, :run_mass_import].each do |action| + it "should not be able to #{ action } project" do + @ability.should_not be_able_to(action, Project) + end end it "should be able to read platform" do diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 25d0c2e0e..64e0e3e73 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -169,4 +169,17 @@ describe Project do end end + it '#run_mass_import' do + owner = FactoryGirl.create(:user) + repository = FactoryGirl.create(:repository) + url = 'http://abf-downloads.rosalinux.ru/abf_personal/repository/test-mass-import' + visibility = 'open' + + Project.run_mass_import(url, "abf-worker-service-1-3.src.rpm\nredir-2.2.1-7.res6.src.rpm\n", visibility, owner, repository.id) + + Project.count.should == 2 + repository.projects.should have(2).items + owner.projects.should have(2).items + end + end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ea7b659ae..8d78688e2 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -50,8 +50,9 @@ def stub_symlink_methods end Resque.inline = true -APP_CONFIG['root_path'] = "#{Rails.root}/tmp/test_root" -APP_CONFIG['git_path'] = "#{Rails.root}/tmp/test_root" +APP_CONFIG['root_path'] = "#{Rails.root}/tmp/test_root" +APP_CONFIG['git_path'] = "#{Rails.root}/tmp/test_root" +APP_CONFIG['tmpfs_path'] = "#{Rails.root}/tmp/test_root" def init_test_root clear_test_root