Merge pull request #323 from abf/rosa-build:321-add-ability-to-mass-importing-projects

#321: Add ability to mass importing projects to the group/user

app/controllers/projects/projects_controller.rb

@@ -3,6 +3,7 @@ class Projects::ProjectsController < Projects::BaseController
   include ProjectsHelper
   before_filter :authenticate_user!
   load_and_authorize_resource :id_param => :project_name # to force member actions load
+  before_filter :who_owns, :only => [:new, :create, :mass_import, :run_mass_import]
 
   def index
     @projects = Project.accessible_by(current_ability, :membered)
@@ -24,7 +25,27 @@ class Projects::ProjectsController < Projects::BaseController
 
   def new
     @project = Project.new
-    @who_owns = :me
+  end
+
+  def mass_import
+    @project = Project.new(:mass_import => true)
+  end
+
+  def run_mass_import
+    @project = Project.new params[:project]
+    @project.owner = choose_owner
+    authorize! :write, @project.owner if @project.owner.class == Group
+    authorize! :add_project, Repository.find(params[:project][:add_to_repository_id])
+    @project.valid?
+    @project.errors.messages.slice! :url
+    if @project.errors.messages.blank? # We need only url validation
+      @project.init_mass_import
+      flash[:notice] = t('flash.project.mass_import_added_to_queue')
+      redirect_to projects_path
+    else
+      flash[:warning] = @project.errors.full_messages.join('. ')
+      render :mass_import
+    end
   end
 
   def edit
@@ -33,7 +54,6 @@ class Projects::ProjectsController < Projects::BaseController
   def create
     @project = Project.new params[:project]
     @project.owner = choose_owner
-    @who_owns = (@project.owner_type == 'User' ? :me : :group)
     authorize! :write, @project.owner if @project.owner.class == Group
 
     if @project.save
@@ -115,6 +135,10 @@ class Projects::ProjectsController < Projects::BaseController
 
   protected
 
+  def who_owns
+    @who_owns = (@project.try(:owner_type) == 'User' ? :me : :group)
+  end
+
   def prepare_list(projects, groups, owners)
     res = {}
 

app/helpers/projects_helper.rb

@@ -20,6 +20,15 @@ module ProjectsHelper
     end.sort_by{ |f| f[:uname] }
   end
 
+  def repositories_grouped_by_platform
+    groups = {}
+    Platform.accessible_by(current_ability, :related).order(:name).each do |platform|
+      next unless can?(:local_admin_manage, platform)
+      groups[platform.name] = Repository.custom_sort(platform.repositories).map{ |r| [r.name, r.id] }
+    end
+    groups
+  end
+
   def git_repo_url(name)
     if current_user
       "#{request.protocol}#{current_user.uname}@#{request.host_with_port}/#{name}.git"

app/models/ability.rb

@@ -60,6 +60,7 @@ class Ability
         can :remove_user, Group
 
         can :create, Project
+        can([:mass_import, :run_mass_import], Project) if user.platforms.main.find{ |p| local_admin?(p) }.present?
         can :read, Project, :visibility => 'open'
         can [:read, :archive, :membered, :get_id], Project, :owner_type => 'User', :owner_id => user.id
         can [:read, :archive, :membered, :get_id], Project, :owner_type => 'Group', :owner_id => user.group_ids

app/models/key_pair.rb

@@ -19,7 +19,7 @@ class KeyPair < ActiveRecord::Base
   protected
 
     def check_keys
-      dir = Dir.mktmpdir('keys-', '/tmp')
+      dir = Dir.mktmpdir 'keys-', APP_CONFIG['tmpfs_path']
       begin
         %w(pubring secring).each do |kind|
           filename = "#{dir}/#{kind}"

app/models/project.rb

@@ -29,8 +29,11 @@ class Project < ActiveRecord::Base
 
   validates :name, :uniqueness => {:scope => [:owner_id, :owner_type], :case_sensitive => false},
                    :presence => true,
-                   :format => {:with => /\A#{NAME_REGEXP}\z/, :message => I18n.t("activerecord.errors.project.uname")}
+                   :format => {:with => /\A#{NAME_REGEXP}\z/,
+                   :message => I18n.t("activerecord.errors.project.uname")}
   validates :maintainer_id, :presence => true, :unless => :new_record?
+  validates :url, :presence => true, :format => {:with => /\Ahttps?:\/\/[\S]+\z/}, :if => :mass_import
+  validates :add_to_repository_id, :presence => true, :if => :mass_import
   validates :visibility, :presence => true, :inclusion => {:in => VISIBILITIES}
   validate { errors.add(:base, :can_have_less_or_equal, :count => MAX_OWN_PROJECTS) if owner.projects.size >= MAX_OWN_PROJECTS }
   validate :check_default_branch
@@ -43,7 +46,9 @@ class Project < ActiveRecord::Base
     errors.delete :project_to_repositories
   end
 
-  attr_accessible :name, :description, :visibility, :srpm, :is_package, :default_branch, :has_issues, :has_wiki, :maintainer_id, :publish_i686_into_x86_64
+  attr_accessible :name, :description, :visibility, :srpm, :is_package, :default_branch,
+                  :has_issues, :has_wiki, :maintainer_id, :publish_i686_into_x86_64,
+                  :url, :srpms_list, :mass_import, :add_to_repository_id
   attr_readonly :owner_id, :owner_type
 
   scope :recent, order("lower(#{table_name}.name) ASC")
@@ -82,6 +87,8 @@ class Project < ActiveRecord::Base
 
   has_ancestry :orphan_strategy => :rootify #:adopt not available yet
 
+  attr_accessor :url, :srpms_list, :mass_import, :add_to_repository_id
+
   include Modules::Models::Owner
   include Modules::Models::Git
   include Modules::Models::Wiki
@@ -98,6 +105,10 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def init_mass_import
+    Project.perform_later :clone_build, :run_mass_import, url, srpms_list, visibility, owner, add_to_repository_id
+  end
+
   def name_with_owner
     "#{owner_uname || owner.uname}/#{name}"
   end

app/views/projects/projects/_filters.html.haml

@@ -6,6 +6,10 @@
       - if can?(:create, Project)
         .bordered.bpadding20
           = link_to t('layout.projects.new'), new_project_path, :class => 'button'
+          - if can?(:mass_import, Project)
+            .both
+            %br
+            = link_to t('layout.projects.mass_import'), mass_import_projects_path, :class => 'button'
       .bordered.bpadding20
         %h3=t('layout.relations.filters')
         - options_for_filters(@all_projects, @groups, @owners).each do |options|

app/views/projects/projects/_form.html.haml

@@ -7,22 +7,10 @@
   .rightlist= f.text_area :description, :class => 'text_field', :cols => 80
   .both
   - if [:new, :create].include? act
-    .leftlist= f.label :owner
+    = render 'owner', :f => f
-    .rightlist
-      = label_tag t("activerecord.attributes.project.who_owns.me")
-      - if Group.can_own_project(current_user).count > 0
-        = radio_button_tag :who_owns, 'me', @who_owns == :me #{}.merge( (@who_owns == :me) ? {:checked => 'checked'} : {} )
-        = label_tag t("activerecord.attributes.project.who_owns.group")
-        = radio_button_tag :who_owns, 'group', @who_owns == :group #{}.merge( (@who_owns == :group) ? {:checked => 'checked'} : {} )
-        -# TODO: Make our own select_box helper with new design, blackjack and bitches!
-        = select_tag :owner_id, options_from_collection_for_select( Group.can_own_project(current_user), :id, :name )
-      - else
-        = hidden_field_tag :who_owns, :me
-    .both
 
   .leftlist= f.label :visibility
   .rightlist
-    =# f.select :visibility, Project::VISIBILITIES
     - Project::VISIBILITIES.each do |visibility|
       = f.radio_button :visibility, visibility, :class => 'niceRadio'
       - if visibility == 'open'

app/views/projects/projects/_owner.html.haml

@@ -0,0 +1,12 @@
+.leftlist= f.label :owner
+.rightlist
+  = label_tag t("activerecord.attributes.project.who_owns.me")
+  - if Group.can_own_project(current_user).count > 0
+    = radio_button_tag :who_owns, 'me', @who_owns == :me #{}.merge( (@who_owns == :me) ? {:checked => 'checked'} : {} )
+    = label_tag t("activerecord.attributes.project.who_owns.group")
+    = radio_button_tag :who_owns, 'group', @who_owns == :group #{}.merge( (@who_owns == :group) ? {:checked => 'checked'} : {} )
+    -# TODO: Make our own select_box helper with new design, blackjack and bitches!
+    = select_tag :owner_id, options_from_collection_for_select( Group.can_own_project(current_user), :id, :name )
+  - else
+    = hidden_field_tag :who_owns, :me
+.both

app/views/projects/projects/mass_import.html.haml

@@ -0,0 +1,27 @@
+%h3.bpadding10= title t("layout.projects.mass_import")
+= form_for @project, :url => run_mass_import_projects_path, :html => { :class => :form } do |f|
+  = f.hidden_field :mass_import
+  .leftlist= f.label :url
+  .rightlist= f.text_field :url
+  .both
+  .leftlist= f.label :srpms_list
+  .rightlist= f.text_area :srpms_list
+  .both
+  .leftlist= f.label :add_to_repository_id
+  .rightlist= f.select :add_to_repository_id, repositories_grouped_by_platform
+  .both
+  = render 'owner', :f => f
+
+  .leftlist= f.label :visibility
+  .rightlist
+    - Project::VISIBILITIES.each do |visibility|
+      = f.radio_button :visibility, visibility, :class => 'niceRadio'
+      - if visibility == 'open'
+        = image_tag("unlock.png")
+      - else
+        = image_tag("lock.png")
+      = t("activerecord.attributes.project.visibilities.#{visibility}")
+  .both
+  .hr
+  .button_block
+    = f.submit t('layout.add'), :data => {'disable-with' => t('layout.saving')}

config/application.yml.sample

@@ -57,6 +57,7 @@ development:
   <<: *common
   root_path: /var/rosa
   git_path: /var/rosa
+  tmpfs_path: /dev/shm
   do-not-reply-email: do-not-reply@localhost
   github_services:
     ip: 127.0.0.1
@@ -66,6 +67,7 @@ production:
   <<: *common
   root_path: /share
   git_path: /mnt/gitstore
+  tmpfs_path: /dev/shm
   do-not-reply-email: do-not-reply@abf.rosalinux.ru
   mailer_https_url: false
   github_services:
@@ -74,6 +76,7 @@ production:
 
 test:
   <<: *common
+  tmpfs_path: "use Rails.root/tmp/test_root in spec"
   root_path: "use Rails.root/tmp/test_root in spec"
   git_path: "use Rails.root/tmp/test_root in spec"
   do-not-reply-email: do-not-reply@localhost

config/locales/models/project.en.yml

@@ -1,6 +1,7 @@
 en:
   layout:
     projects:
+      mass_import: Mass import
       branches: Branches
       delete_branch: Delete branch
       restore_branch: Restore branch
@@ -76,6 +77,7 @@ en:
 
   flash:
     project:
+      mass_import_added_to_queue: Mass import added to queue
       saved: Project saved
       save_error: Unable to save project
       save_warning_ssh_key: Project owner must provide a SSH key in his profile
@@ -89,6 +91,9 @@ en:
       project: Project
     attributes:
       project:
+        url: URL
+        add_to_repository_id: Add to repository
+        srpms_list: SRPMs list
         name: Name
         description: Descripton
         owner: Owner

config/locales/models/project.ru.yml

@@ -1,6 +1,7 @@
 ru:
   layout:
     projects:
+      mass_import: Массовый импорт
       branches: Ветки
       delete_branch: Удалить ветку
       restore_branch: Восстановить ветку
@@ -76,6 +77,7 @@ ru:
 
   flash:
     project:
+      mass_import_added_to_queue: Массовый импорт добавлен в очередь
       saved: Проект успешно сохранен
       save_error: Не удалось сохранить проект
       save_warning_ssh_key: Владельцу проекта необходимо указать в профиле свой SSH ключ
@@ -89,6 +91,9 @@ ru:
       project: Проект
     attributes:
       project:
+        url: URL
+        add_to_repository_id: Добавить в репозиторий
+        srpms_list: Список SRPMs
         name: Название
         description: Описание
         owner: Владелец

config/routes.rb

@@ -284,7 +284,12 @@ Rosa::Application.routes.draw do
       end
     end
 
-    resources :projects, :only => [:index, :new, :create]
+    resources :projects, :only => [:index, :new, :create] do
+      collection do
+        post  :run_mass_import
+        get   :mass_import
+      end
+    end
     scope ':owner_name/:project_name', :constraints => {:project_name => Project::NAME_REGEXP} do # project
       scope :as => 'project' do
         resources :wiki do

lib/modules/models/git.rb

@@ -1,4 +1,8 @@
 # -*- encoding : utf-8 -*-
+require 'nokogiri'
+require 'open-uri'
+require 'iconv'
+
 module Modules
   module Models
     module Git
@@ -173,10 +177,70 @@ module Modules
       end
 
       module ClassMethods
+        MAX_SRC_SIZE = 1024*1024*256
+
         def process_hook(owner_uname, repo, newrev, oldrev, ref, newrev_type, user = nil, message = nil)
           rec = GitHook.new(owner_uname, repo, newrev, oldrev, ref, newrev_type, user, message)
           Modules::Observers::ActivityFeed::Git.create_notifications rec
         end
+
+        def run_mass_import(url, srpms_list, visibility, owner, add_to_repository_id)
+          doc = Nokogiri::HTML(open(url))
+          links = doc.css("a[href$='.src.rpm']")
+          return if links.count == 0
+          filter = srpms_list.lines.map(&:chomp).map(&:strip).select(&:present?)
+          
+          repository = Repository.find add_to_repository_id
+          platform = repository.platform
+          dir = Dir.mktmpdir 'mass-import-', APP_CONFIG['tmpfs_path']
+          links.each do |link|
+            begin
+              package = link.attributes['href'].value
+              package.chomp!; package.strip!
+
+              next if package.size == 0 || package !~ /^[\w\.\-]+$/
+              next if filter.present? && !filter.include?(package)
+
+              uri = URI "#{url}/#{package}"
+              srpm_file = "#{dir}/#{package}"
+              Net::HTTP.start(uri.host) do |http|
+                if http.request_head(uri.path)['content-length'].to_i < MAX_SRC_SIZE
+                  f = open(srpm_file, 'wb')
+                  http.request_get(uri.path) do |resp|
+                    resp.read_body{ |segment| f.write(segment) }
+                  end
+                  f.close
+                end
+              end
+              if name = `rpm -q --qf '[%{Name}]' -p #{srpm_file}` and $?.success? and name.present?
+                next if owner.projects.exists?(:name => name)
+                description = ::Iconv.conv('UTF-8//IGNORE', 'UTF-8', `rpm -q --qf '[%{Description}]' -p #{srpm_file}`)
+                project = owner.projects.build(
+                  :name         => name,
+                  :description  => description,
+                  :visibility   => visibility,
+                  :is_package   => false # See: Hook for #attach_to_personal_repository
+                )
+                project.owner = owner
+                if project.save
+                  repository.projects << project rescue nil
+                  project.update_attributes(:is_package => true)
+                  project.import_srpm srpm_file, platform.name
+                end
+              end
+            rescue => e
+              f.close if defined?(f)
+              Airbrake.notify_or_ignore(e, :link => link.to_s, :url => url, :owner => owner)
+            ensure
+              File.delete srpm_file if srpm_file
+            end
+          end
+        rescue => e
+          Airbrake.notify_or_ignore(e, :url => url, :owner => owner)
+        ensure
+          FileUtils.remove_entry_secure dir if dir
+        end
+
       end
     end
   end

spec/models/cancan_spec.rb

@@ -65,6 +65,12 @@ describe CanCan do
       end
     end
 
+    [:mass_import, :run_mass_import].each do |action|
+      it "should not be able to #{ action } project" do
+        @ability.should_not be_able_to(action, Project)
+      end
+    end
+
     it 'should not be able to update register request' do
       @ability.should_not be_able_to(:update, register_request)
     end
@@ -93,6 +99,12 @@ describe CanCan do
       end
     end
 
+    [:mass_import, :run_mass_import].each do |action|
+      it "should not be able to #{ action } project" do
+        @ability.should_not be_able_to(action, Project)
+      end
+    end
+
     it "shoud be able to show user profile" do
       @ability.should be_able_to(:show, User)
     end
@@ -260,6 +272,13 @@ describe CanCan do
         before(:each) do
           @platform.owner = @user
           @platform.save
+          @ability = Ability.new(@user)
+        end
+
+        [:mass_import, :run_mass_import].each do |action|
+          it "should be able to #{ action } project" do
+            @ability.should be_able_to(action, Project)
+          end
         end
 
         [:read, :update, :destroy, :change_visibility].each do |action|
@@ -272,6 +291,13 @@ describe CanCan do
       context 'with read rights' do
         before(:each) do
           @platform.relations.create!(:actor_id => @user.id, :actor_type => 'User', :role => 'reader')
+          @ability = Ability.new(@user)
+        end
+
+        [:mass_import, :run_mass_import].each do |action|
+          it "should not be able to #{ action } project" do
+            @ability.should_not be_able_to(action, Project)
+          end
         end
 
         it "should be able to read platform" do

spec/models/project_spec.rb

@@ -169,4 +169,17 @@ describe Project do
     end
   end
 
+  it '#run_mass_import' do
+    owner = FactoryGirl.create(:user)
+    repository = FactoryGirl.create(:repository)
+    url = 'http://abf-downloads.rosalinux.ru/abf_personal/repository/test-mass-import'
+    visibility = 'open'
+    
+    Project.run_mass_import(url, "abf-worker-service-1-3.src.rpm\nredir-2.2.1-7.res6.src.rpm\n", visibility, owner, repository.id)
+
+    Project.count.should == 2
+    repository.projects.should have(2).items
+    owner.projects.should have(2).items
+  end
+
 end

spec/spec_helper.rb

@@ -50,8 +50,9 @@ def stub_symlink_methods
 end
 
 Resque.inline = true
-APP_CONFIG['root_path'] = "#{Rails.root}/tmp/test_root"
+APP_CONFIG['root_path']   = "#{Rails.root}/tmp/test_root"
-APP_CONFIG['git_path']  = "#{Rails.root}/tmp/test_root"
+APP_CONFIG['git_path']    = "#{Rails.root}/tmp/test_root"
+APP_CONFIG['tmpfs_path']  = "#{Rails.root}/tmp/test_root"
 
 def init_test_root
   clear_test_root