Merge branch 'master' into 90-fix_orphan_pulls

Conflicts:
	app/models/pull_request.rb

Gemfile

@@ -54,6 +54,7 @@ gem 'ruby-haml-js', '~> 0.0.3'
 gem 'rails-backbone', '~> 0.7.2'
 
 gem 'rack-throttle'
+gem 'rest-client', '~> 1.6.6'
 
 group :assets do
   gem 'sass-rails', '~> 3.2.5'

Gemfile.lock

@@ -265,6 +265,8 @@ GEM
       uuid (~> 2.3)
     resque_mailer (2.1.0)
       actionmailer (~> 3.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     rr (1.0.4)
     rspec (2.11.0)
       rspec-core (~> 2.11.0)
@@ -401,6 +403,7 @@ DEPENDENCIES
   resque (~> 1.21.0)
   resque-status (~> 0.3.3)
   resque_mailer (~> 2.1.0)
+  rest-client (~> 1.6.6)
   rr (~> 1.0.4)
   rspec-rails (~> 2.11.0)
   ruby-haml-js (~> 0.0.3)

app/controllers/projects/git/commits_controller.rb

@@ -10,11 +10,11 @@ class Projects::Git::CommitsController < Projects::Git::BaseController
 
   def show
     @commit = @commentable = @project.repo.commit(params[:id]) || raise(ActiveRecord::RecordNotFound)
-    @comments, @diff = Comment.for_commit(@commit), @commit.diffs
+    @comments = Comment.for_commit(@commit)
 
     respond_to do |format|
       format.html
-      format.diff  { render :text => (@diff.map(&:diff).join("\n") rescue ''), :content_type => "text/plain" }
+      format.diff  { render :text => (@commit.diffs.map(&:diff).join("\n") rescue ''), :content_type => "text/plain" }
       format.patch { render :text => (@commit.to_patch rescue ''), :content_type => "text/plain" }
     end
   end

app/controllers/projects/pull_requests_controller.rb

@@ -132,7 +132,7 @@ class Projects::PullRequestsController < Projects::BaseController
     @total_commits = @commits.count
     @commits = @commits.last(100)
 
-    @diff, @stats = @pull.diff, @pull.diff_stats
+    @stats = @pull.diff_stats
     @comments, @commentable = @issue.comments, @issue
   end
 

app/helpers/comments_helper.rb

@@ -25,6 +25,6 @@ module CommentsHelper
   end
 
   def comment_anchor c
-    "#{(c.data.present? && c.actual_inline_comment?(@diff)) ? 'diff-' : ''}comment#{c.id}"
+    "#{(c.data.present? && c.actual_inline_comment?) ? 'diff-' : ''}comment#{c.id}"
   end
 end

app/helpers/diff_helper.rb

@@ -232,7 +232,7 @@ module DiffHelper
   def render_line_comments
     unless @in_wiki || @in_discussion
       comments = @line_comments.select do |c|
-        c.data.try('[]', :line) == @num_line.to_s && c.actual_inline_comment?(@diff)
+        c.data.try('[]', :line) == @num_line.to_s && c.actual_inline_comment?
       end
       tr_line_comments(comments) if comments.count > 0
     end

app/models/comment.rb

@@ -54,10 +54,11 @@ class Comment < ActiveRecord::Base
     User.find(subscribe.user).notifier.new_comment && User.find(subscribe.user).notifier.can_notify
   end
 
-  def actual_inline_comment?(diff, force = false)
+  def actual_inline_comment?(diff = nil, force = false)
     unless force
       raise "This is not inline comment!" if data.blank? # for debug
       return data[:actual] unless data[:actual].nil?
+      return false if diff.nil?
     end
     filepath, line_number = data[:path], data[:line]
     diff_path = (diff || commentable.diffs ).select {|d| d.a_path == data[:path]}

app/models/group.rb

@@ -13,7 +13,7 @@ class Group < Avatar
   has_many :own_platforms, :as => :owner, :class_name => 'Platform', :dependent => :destroy
 
   validates :owner, :presence => true
-  validates :uname, :presence => true, :uniqueness => {:case_sensitive => false}, :format => {:with => /^[a-z0-9_]+$/}, :reserved_name => true
+  validates :uname, :presence => true, :uniqueness => {:case_sensitive => false}, :format => {:with => /\A[a-z0-9_]+\z/}, :reserved_name => true
   validate { errors.add(:uname, :taken) if User.by_uname(uname).present? }
 
   scope :opened, where('1=1')

app/models/platform.rb

@@ -22,7 +22,7 @@ class Platform < ActiveRecord::Base
   validates :description, :presence => true
   validates :owner, :presence => true
   validates :visibility, :presence => true, :inclusion => {:in => VISIBILITIES}
-  validates :name, :uniqueness => {:case_sensitive => false}, :presence => true, :format => { :with => /^[a-zA-Z0-9_\-\.]+$/ }
+  validates :name, :uniqueness => {:case_sensitive => false}, :presence => true, :format => { :with => /\A[a-zA-Z0-9_\-\.]+\z/ }
   validates :distrib_type, :presence => true, :inclusion => {:in => APP_CONFIG['distr_types']}
   validate lambda {
     if released_was && !released

app/models/project.rb

@@ -25,7 +25,7 @@ class Project < ActiveRecord::Base
 
   validates :name, :uniqueness => {:scope => [:owner_id, :owner_type], :case_sensitive => false},
                    :presence => true,
-                   :format => {:with => /^#{NAME_REGEXP}$/, :message => I18n.t("activerecord.errors.project.uname")}
+                   :format => {:with => /\A#{NAME_REGEXP}\z/, :message => I18n.t("activerecord.errors.project.uname")}
   validates :owner, :presence => true
   validates :maintainer_id, :presence => true, :unless => :new_record?
   validates :visibility, :presence => true, :inclusion => {:in => VISIBILITIES}
@@ -116,10 +116,10 @@ class Project < ActiveRecord::Base
   end
 
   def git_project_address
-    #host ||= EventLog.current_controller.request.host_with_port rescue ::Rosa::Application.config.action_mailer.default_url_options[:host]
-    #protocol ||= EventLog.current_controller.request.protocol rescue "http"
-    #Rails.application.routes.url_helpers.project_url(self.owner.uname, self.name, :host => host, :protocol => protocol) + ".git"
-    path
+    host ||= EventLog.current_controller.request.host_with_port rescue ::Rosa::Application.config.action_mailer.default_url_options[:host]
+    protocol = APP_CONFIG['mailer_https_url'] ? "https" : "http" rescue "http"
+    Rails.application.routes.url_helpers.project_url(self.owner.uname, self.name, :host => host, :protocol => protocol) + ".git"
+    #path #share by NFS
   end
 
   def build_for(platform, repository_id, user, arch = 'i586', auto_publish = false, mass_build_id = nil, priority = 0)

app/models/pull_request.rb

@@ -85,7 +85,7 @@ class PullRequest < ActiveRecord::Base
 
   def path
     filename = [id, from_project_owner_uname, from_project_name].compact.join('-')
-    File.join(APP_CONFIG['root_path'], 'pull_requests', to_project.owner.uname, to_project.name, filename)
+    File.join(APP_CONFIG['git_path'], 'pull_requests', to_project.owner.uname, to_project.name, filename)
   end
 
   def from_branch

app/models/repository.rb

@@ -12,7 +12,7 @@ class Repository < ActiveRecord::Base
   has_one  :key_pair, :dependent => :destroy
 
   validates :description, :presence => true
-  validates :name, :uniqueness => {:scope => :platform_id, :case_sensitive => false}, :presence => true, :format => {:with => /^[a-z0-9_\-]+$/}
+  validates :name, :uniqueness => {:scope => :platform_id, :case_sensitive => false}, :presence => true, :format => {:with => /\A[a-z0-9_\-]+\z/}
 
   scope :recent, order("name ASC")
 

app/models/user.rb

@@ -29,7 +29,7 @@ class User < Avatar
 
   has_many :key_pairs
 
-  validates :uname, :presence => true, :uniqueness => {:case_sensitive => false}, :format => {:with => /^[a-z0-9_]+$/}, :reserved_name => true
+  validates :uname, :presence => true, :uniqueness => {:case_sensitive => false}, :format => {:with => /\A[a-z0-9_]+\z/}, :reserved_name => true
   validate { errors.add(:uname, :taken) if Group.by_uname(uname).present? }
   validates :role, :inclusion => {:in => ROLES}, :allow_blank => true
   validates :language, :inclusion => {:in => LANGUAGES}, :allow_blank => true

app/views/projects/git/commits/_show.html.haml

@@ -13,6 +13,6 @@
 
 -begin
   = render_commit_stats(stats)
-  = render :partial => 'commit_diff', :collection => @diff
+  = render :partial => 'commit_diff', :collection => @commit.diffs
 - rescue Grit::Git::GitTimeout
   %p= t 'layout.git.repositories.commit_diff_too_big'

app/views/projects/pull_requests/_diff_commits_tabs.html.haml

@@ -9,10 +9,12 @@
   .both
   -begin
     = render_diff_stats(@stats)
-    = render :partial => 'pull_diff', :collection => @diff
-  - rescue Grit::Git::GitTimeout
-    %p= t 'layout.git.repositories.commit_diff_too_big'
-
+    = render :partial => 'pull_diff', :collection => @pull.diff
+  - rescue => ex
+    -if ex.try(:message) == 'Grit::Git::GitTimeout'
+      %p= t 'layout.git.repositories.commit_diff_too_big'
+    -else
+      -raise ex
 #commits.tab-pane
   - if @total_commits > @commits.count
     %div= t("projects.pull_requests.is_big", :count => @commits.count)

app/views/projects/pull_requests/_discussion_comments.html.haml

@@ -2,7 +2,7 @@
 .file
   .top
     .l=comment.data[:view_path]
-    -if comment.actual_inline_comment? @diff
+    -if comment.actual_inline_comment? @pull.diff
       .r=link_to t("layout.pull_requests.view_full_changes"),
         "#{project_commentable_path(@project, @commentable)}##{comment_anchor(comment)}",
         :class => 'link_to_full_changes'

bin/file-store.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+require 'json'
+require 'rest-client'
+
+abf_yml, new_sources = '.abf.yml', []
+old_sources = if File.exist? abf_yml
+                File.read(abf_yml).split("\n").reject {|line| line =~ /sources/}
+              else
+                []
+              end
+MAX_SIZE = 2 * 1024 * 1024 # 2.megabytes
+url = 'http://file-store.rosalinux.ru/api/v1/file_stores.json'
+#url = 'http://localhost:3001/api/v1/file_stores.json'
+rclient = RestClient::Resource.new(url, :user => ARGF.argv[0]) # user auth token
+
+Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each do |file|
+  begin
+    #puts " work with file \"#{file}\""
+    next if File.size(file) < MAX_SIZE
+
+    sha1 = Digest::SHA1.file(file).hexdigest
+    resp = JSON(RestClient.get url, :params => {:hash => sha1})
+    if resp[0].respond_to?('[]') && resp[0]['file_name'] && resp[0]['sha1_hash']
+      # file already exists at file-store
+      new_sources << "  \"#{file}\": #{sha1}"
+      FileUtils.rm_rf file
+      puts " file \"#{file}\" already exists in the file-store"
+    elsif resp == []
+      # try to put file at file-store
+      resp = JSON(rclient.post :file_store => {:file => File.new(file, 'rb')})
+      unless resp['sha1_hash'].nil?
+        new_sources << "  \"#{file}\": #{sha1}"
+        FileUtils.rm_rf file
+         puts " upload file \"#{file}\" to the file-store"
+      end
+    else
+      raise "Response unknown!\n #{resp}"
+    end
+
+  #rescue => e
+  #  e.response
+  end
+end
+sources = (old_sources | new_sources)
+unless sources.empty?
+  Dir.chdir(ARGF.argv[1]) do # Need if in filter-branch
+    File.open(abf_yml, 'w') do |abf|
+      abf.puts 'sources:'
+      (old_sources | new_sources).sort.each {|line| abf.puts line}
+    end
+  end
+end

bin/import_srpm.sh

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 # import_srpm.sh: Import SRPM packages to git repo
-
+pwd=$PWD
 # Input data
 srpm_path=$1
 git_path=$2
 git_branch=$3
+token=$4
 name=$(rpm -q --qf '[%{Name}]' -p $srpm_path)
 version=$(rpm -q --qf '[%{Version}-%{Release}]' -p $srpm_path)
 tmp_dir=/tmp/$name-$version-$RANDOM
@@ -30,6 +31,9 @@ rpm2cpio $srpm_path > srpm.cpio
 cpio -idv < srpm.cpio
 rm -f srpm.cpio
 
+# Remove archives
+$pwd/file-store.rb $token $PWD
+
 # Commit and push changes
 git add -A .
 git commit -m "Automatic import for version $version"

bin/remove_archives.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+# git_dir_projects[0] clone_path[1] owner[2] project_name[3]
+
+require 'fileutils'
+require 'digest'
+
+token = '[CENSORED]'
+
+owners = ARGF.argv[2] || '[a-z0-9_]*'
+project_names = ARGF.argv[3] || '[a-zA-Z0-9_\-\+\.]*'
+
+begin_time = Time.now
+pr_count = total_count = 0
+
+Dir.chdir ARGF.argv[0]
+Dir.glob(owners).each do |owner|
+  Dir.chdir "#{ARGF.argv[0]}/#{owner}"
+  Dir.glob(project_names).each do |project|
+    name_with_owner = "#{owner}/#{project.chomp('.git')}"
+    project_path = "#{ARGF.argv[0]}/#{name_with_owner}.git"
+    time, total_count = Time.now, total_count + 1
+    Dir.chdir project_path
+    project_stats = "#{name_with_owner}: #{total_count}"
+    if system('git log -n 1 --oneline > /dev/null 2>&1') == false
+      p "Skipping empty project #{project_stats}"
+    else
+      p "Start working with #{project_stats}"
+      path = "#{ARGF.argv[1].chomp('/')}/repos/#{name_with_owner}"
+      FileUtils.rm_rf path
+      #-- hack for refs/heads (else git branch return only master)
+      system "git clone --mirror #{project_path} #{path}/.git"
+      system "cd #{path} && git config --bool core.bare false && git checkout -f HEAD"
+      #--
+      Dir.chdir(path)
+      archives_exists = false
+      %w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z).each do |ext|
+        archives_exists=true and break unless `git log --all --format='%H' -- *.#{ext}`.empty?
+      end
+
+      if archives_exists
+        system "git filter-branch -d /dev/shm/git_task --tree-filter \"/home/rosa/git_task/file-store.rb #{token} #{path}\" --prune-empty --tag-name-filter cat -- --all"
+        #####
+        # This is dangerous !!!
+        system "rm -rf #{project_path} && git clone --bare #{path} #{project_path}"
+        #####
+
+        p "Worked with #{name_with_owner}: #{(Time.now - time).truncate} sec."
+        pr_count +=1
+      else
+        p "Skipping project with no archives #{project_stats}"
+      end
+      `rm -rf #{path}`
+    end
+    p '-------------'
+  end
+end
+p '======================='
+p "Total count of projects are #{total_count}"
+p "Finished work with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}"

config/initializers/setup.rb

@@ -6,7 +6,7 @@ require 'modules'
 require 'plugins'
 
 # Setup Smart HTTP GRack
-config = {:project_root => File.join(APP_CONFIG['root_path'], 'git_projects'), :git_path => 'git', :upload_pack => true, :receive_pack => true}
+config = {:project_root => File.join(APP_CONFIG['git_path'], 'git_projects'), :git_path => 'git', :upload_pack => true, :receive_pack => true}
 # Rosa::Application.config.middleware.insert_before 0, ::Grack::Handler, config
 Rosa::Application.config.middleware.insert_after ::Rails::Rack::Logger, ::Grack::Handler, config
 Rosa::Application.config.middleware.insert_before ::Grack::Handler, ::Grack::Auth

config/locales/en.yml

@@ -116,7 +116,7 @@ en:
         empty: "Repository is empty. You need to wait some time if you have forked project or imported package"
         source: Source
         commits: Commits
-        commit_diff_too_big: Sorry, commit too big!
+        commit_diff_too_big: Sorry, diff too big!
         tags: Tags
         branches: Branches
         project_versions: Versions

config/locales/ru.yml

@@ -116,7 +116,7 @@ ru:
         empty: "Репозиторий пуст. Если вы клонировали(Fork) проект или импортировали пакет, данные скоро появятся"
         source: Source
         commits: Коммиты
-        commit_diff_too_big: Извините, коммит слишком большой!
+        commit_diff_too_big: Извините, изменений слишком много!
         tags: Теги
         branches: Ветки
         project_versions: Версии

config/schedule.rb

@@ -7,9 +7,9 @@
 #  runner "Download.parse_and_remove_nginx_log"
 #end
 
-every 1.day, :at => '4:00 am' do
-  rake "import:sync:all", :output => 'log/sync.log'
-end
+#every 1.day, :at => '4:00 am' do
+#  rake "import:sync:all", :output => 'log/sync.log'
+#end
 
 every 1.day, :at => '3:50 am' do
   rake "buildlist:clear:outdated", :output => 'log/build_list_clear.log'

db/seeds.rb

@@ -3,3 +3,6 @@ ARCHES = %w(i386 i586 x86_64)
 ARCHES.each do |arch|
   Arch.find_or_create_by_name arch
 end
+
+user = User.new uname: 'rosa_system', email: 'rosa_system@rosalinux.ru', password: SecureRandom.base64
+user.confirmed_at = Time.now.utc; user.save

lib/ext/git/grit.rb

@@ -32,16 +32,16 @@ module Grit
     # def file_mime_type
     #   @file_mime_type ||= data.file_type(:mime_type)
     # end
-    # 
+    #
     # def text?
     #   file_mime_type =~ /^text\// # not binary?
     # end
-    # 
+    #
     # def binary?
     #   not text? # file_mime_type !~ /^text\//
     #   # s = data.split(//); ((s.size - s.grep(" ".."~").size) / s.size.to_f) > 0.30 # works only for latin chars
     # end
-    # 
+    #
     # def image?
     #   mime_type.match(/image/)
     # end

lib/modules/models/git.rb

@@ -92,13 +92,14 @@ module Modules
       end
 
       def import_srpm(srpm_path = srpm.path, branch_name = 'import')
-        system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} >> /dev/null 2>&1")
+        token = User.find_by_uname('rosa_system').token_authenticatable
+        system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} #{token} >> /dev/null 2>&1")
       end
 
       protected
 
       def build_path(dir)
-        File.join(APP_CONFIG['root_path'], 'git_projects', "#{dir}.git")
+        File.join(APP_CONFIG['git_path'], 'git_projects', "#{dir}.git")
       end
 
       def import_attached_srpm

spec/models/group_spec.rb

@@ -61,11 +61,11 @@ describe Group do
 
   context 'for group owner' do
     before(:each) do
-      @user = FactoryGirl.create(:user) 
-      
+      @user = FactoryGirl.create(:user)
+
       @group.owner = @user
       @group.save
-      
+
       @group.actors.create(:actor_type => 'User', :actor_id => @user.id, :role => 'admin')
       @ability = Ability.new(@user)
     end
@@ -96,4 +96,6 @@ describe Group do
       end
     end
   end
+
+  it {should_not allow_value("How do you do...\nmy_group").for(:uname)}
 end

spec/models/platform_spec.rb

@@ -15,30 +15,32 @@ describe Platform do
   it { should have_many(:members)}
   it { should have_many(:repositories)}
   it { should have_many(:products)}
-  
+
   it { should validate_presence_of(:name)}
   it { should validate_uniqueness_of(:name).case_insensitive }
   it { should validate_format_of(:name).with('Basic_platform-name-1234') }
   it { should validate_format_of(:name).not_with('.!') }
   it { should validate_presence_of(:description) }
   it { should validate_presence_of(:distrib_type) }
-  it { should validate_presence_of(:visibility) }  
-  
+  it { should validate_presence_of(:visibility) }
+
   Platform::VISIBILITIES.each do |value|
     it {should allow_value(value).for(:visibility)}
   end
   it {should_not allow_value('custom_status').for(:visibility)}
-  
+
   it { should have_readonly_attribute(:name) }
   it { should have_readonly_attribute(:distrib_type) }
   it { should have_readonly_attribute(:parent_platform_id) }
   it { should have_readonly_attribute(:platform_type) }
-  
+
   it { should_not allow_mass_assignment_of(:repositories) }
   it { should_not allow_mass_assignment_of(:products) }
   it { should_not allow_mass_assignment_of(:members) }
   it { should_not allow_mass_assignment_of(:parent) }
-  
+
+  it {should_not allow_value("How do you do...\nmy_platform").for(:name)}
+
   after(:all) do
     Platform.delete_all
     User.delete_all

spec/models/project_spec.rb

@@ -72,11 +72,18 @@ describe Project do
     it 'ensures that validation passed' do
       project.valid?.should be_true
     end
-    
+
     it 'ensures that name has been truncated' do
       project.valid?
       project.name.should == 'test_name'
     end
   end
 
+  context 'Validate project name' do
+    let!(:project) { FactoryGirl.build(:project, :name => '  test_name  ') }
+
+    it "'hacked' uname should not pass" do
+      lambda {FactoryGirl.create(:project, :name => "...\nbeatiful_name\n for project")}.should raise_error(ActiveRecord::RecordInvalid)
+    end
+  end
 end

spec/models/repository_spec.rb

@@ -15,7 +15,7 @@ describe Repository do
       @platform.repositories.count.should eql(1)
     end
   end
-  
+
   before(:all) do
     stub_symlink_methods
     Platform.delete_all
@@ -25,29 +25,30 @@ describe Repository do
     # Need for validate_uniqueness_of check
     FactoryGirl.create(:repository)
   end
-  
+
   it { should belong_to(:platform) }
   it { should have_many(:project_to_repositories)}
   it { should have_many(:projects).through(:project_to_repositories)}
-  
+
   it { should validate_presence_of(:name)}
   it { should validate_uniqueness_of(:name).case_insensitive.scoped_to(:platform_id) }
   it { should validate_format_of(:name).with('basic_repository-name-1234') }
   it { should validate_format_of(:name).not_with('.!') }
   it { should validate_format_of(:name).not_with('Main') }
+  it { should validate_format_of(:name).not_with("!!\nbang_bang\n!!") }
   it { should validate_presence_of(:description) }
-  
+
   it { should have_readonly_attribute(:name) }
   it { should have_readonly_attribute(:platform_id) }
-  
+
   it { should_not allow_mass_assignment_of(:platform) }
   it { should_not allow_mass_assignment_of(:platform_id) }
-  
+
   after(:all) do
     Platform.delete_all
     User.delete_all
     Repository.delete_all
     FileUtils.rm_rf(APP_CONFIG['root_path'])
   end
-  
+
 end

spec/models/user_spec.rb

@@ -45,7 +45,7 @@ describe User do
   context 'for group project' do
     before(:each) do
       @project.relations.destroy_all
-      
+
       @project.owner = @group
       @project.save
       @project.relations.create :actor_id => @project.owner.id, :actor_type => @project.owner.class.to_s, :role => 'admin'
@@ -92,4 +92,5 @@ describe User do
     end
   end
 
+  it {should_not allow_value("new_user\nHello World!").for(:uname)}
 end