merge master into the branch

app/controllers/projects/pull_requests_controller.rb

@@ -151,5 +151,7 @@ class Projects::PullRequestsController < Projects::BaseController
     @pull.from_project = @project
     @pull.to_ref = (pull_params[:to_ref].presence if pull_params) || @pull.to_project.default_branch
     @pull.from_ref = params[:treeish].presence || (pull_params[:from_ref].presence if pull_params) || @pull.from_project.default_branch
+    @pull.from_project_owner_uname = @pull.from_project.owner.uname
+    @pull.from_project_name = @pull.from_project.name
   end
 end

app/helpers/pull_request_helper.rb

@@ -27,20 +27,27 @@ module PullRequestHelper
    #{show_ref pull, 'from'}</span> \
    #{t 'into'} <span class='label-bootstrap label-info font14'> \
    #{show_ref pull, 'to'}</span>"
-    str << " #{t 'by'} #{link_to pull.user.uname, user_path(pull.user)}" if pull.persisted?
+    str << " #{t 'by'} #{link_to pull.user.uname, user_path(pull.user)}" if pull.user# pull.persisted?
     str.html_safe
   end
 
   #helper for helpers
   def show_ref pull, which, limit = 30
     project, ref = pull.send("#{which}_project"), pull.send("#{which}_ref")
-    link_to "#{project.owner.uname.truncate limit}/#{project.name.truncate limit}: #{ref.truncate limit}", ref_path(project, ref)
+    fullname = if which == 'to'
+                 "#{project.owner.uname.truncate limit}/#{project.name.truncate limit}"
+               elsif which == 'from'
+                 "#{pull.from_project_owner_uname.truncate limit}/#{pull.from_project_name.truncate limit}"
+               end
+    link_to "#{fullname}: #{ref.truncate limit}", ref_path(project, ref)
   end
 
   def ref_path project, ref
-    return tree_path(project, ref) if project.repo.branches_and_tags.map(&:name).include? ref
+    if project && project.repo.branches_and_tags.map(&:name).include?(ref)
-    return commit_path(project, ref) if project.repo.commit ref
+      tree_path(project, ref)
-    '#'
+    else
+      '#'
+    end
   end
 
   def ref_selector_options(project, current)

app/models/pull_request.rb

@@ -6,12 +6,14 @@ class PullRequest < ActiveRecord::Base
   delegate :user, :user_id, :title, :body, :serial_id, :assignee, :status, :to_param,
     :created_at, :updated_at, :comments, :status=, :to => :issue, :allow_nil => true
 
-  validate :uniq_merge
+  validates :from_project, :to_project, :presence => true
+  validate :uniq_merge, :if => Proc.new { |pull| pull.to_project.present? }
   validates_each :from_ref, :to_ref do |record, attr, value|
     check_ref record, attr, value
   end
 
   before_create :clean_dir
+  before_create :set_add_data
   after_destroy :clean_dir
 
   accepts_nested_attributes_for :issue
@@ -82,12 +84,12 @@ class PullRequest < ActiveRecord::Base
   end
 
   def path
-    filename = [id, from_project.owner.uname, from_project.name].compact.join('-')
+    filename = [id, from_project_owner_uname, from_project_name].compact.join('-')
     File.join(APP_CONFIG['git_path'], 'pull_requests', to_project.owner.uname, to_project.name, filename)
   end
 
   def from_branch
-    if to_project != from_project
+    if to_project_id != from_project_id
       "head_#{from_ref}"
     else
       from_ref
@@ -138,11 +140,19 @@ class PullRequest < ActiveRecord::Base
 
   def self.check_ref(record, attr, value)
     project = attr == :from_ref ? record.from_project : record.to_project
-    record.errors.add attr, I18n.t('projects.pull_requests.wrong_ref') unless project.repo.branches_and_tags.map(&:name).include?(value)
+    return if project.blank?
+    if record.to_project.repo.commits.count > 0
+      record.errors.add attr, I18n.t('projects.pull_requests.wrong_ref') unless project.repo.branches_and_tags.map(&:name).include?(value)
+    else
+      record.errors.add attr, I18n.t('projects.pull_requests.empty_repo')
+    end
   end
 
   def uniq_merge
-    if to_project.pull_requests.needed_checking.where(:from_project_id => from_project, :to_ref => to_ref, :from_ref => from_ref).where('pull_requests.id <> :id or :id is null', :id => id).count > 0
+    if to_project && to_project.pull_requests.needed_checking
+         .where(:from_project_id => from_project_id,
+                :to_ref => to_ref, :from_ref => from_ref)
+         .where('pull_requests.id <> :id or :id is null', :id => id).count > 0
       errors.add(:base_branch, I18n.t('projects.pull_requests.duplicate', :from_ref => from_ref))
     end
   end
@@ -160,7 +170,7 @@ class PullRequest < ActiveRecord::Base
 
   def merge
     clone
-    message = "Merge pull request ##{serial_id} from #{from_project.name_with_owner}:#{from_ref}\r\n #{title}"
+    message = "Merge pull request ##{serial_id} from #{from_project_owner_uname}/#{from_project_name}:#{from_ref}\r\n #{title}"
     %x(cd #{path} && git checkout #{to_ref} && git merge --no-ff #{from_branch} -m '#{message}')
   end
 
@@ -183,7 +193,7 @@ class PullRequest < ActiveRecord::Base
     Dir.chdir(path) do
       system 'git', 'checkout', to_ref
       system 'git', 'pull',  'origin', to_ref
-      if to_project == from_project
+      if to_project_id == from_project_id
         system 'git', 'checkout', from_ref
         system 'git', 'pull', 'origin', from_ref
       else
@@ -219,4 +229,9 @@ class PullRequest < ActiveRecord::Base
       end
     end
   end
+
+  def set_add_data
+    self.from_project_owner_uname = from_project.owner.uname
+    self.from_project_name = from_project.name
+  end
 end

app/views/projects/pull_requests/show.html.haml

@@ -1,5 +1,5 @@
 -ar = 'activerecord.attributes.pull_requests'
--set_meta_tags :title => [title_object(@project), t('.title', :name => @pull.title.truncate(40), :user => @pull.user.uname)]
+-set_meta_tags :title => [title_object(@project), t('.title', :name => @pull.title.truncate(40), :user => @pull.user.try(:uname))]
 = render :partial => 'submenu'
 %h3.bpadding10
   =pull_status_label @pull

bin/calc_sha

@@ -1,5 +0,0 @@
-#!/bin/bash
-pwd=`pwd`
-cd $1
-git filter-branch --tree-filter "$pwd/bin/file-store.rb $2" --prune-empty --tag-name-filter cat -- --all
-cd ..

bin/file-store.rb

@@ -2,21 +2,20 @@
 require 'json'
 require 'rest-client'
 
-abf_yml, new_sources = 'abf.yml', []
+abf_yml, new_sources = '.abf.yml', []
 old_sources = if File.exist? abf_yml
                 File.read(abf_yml).split("\n").reject {|line| line =~ /sources/}
               else
                 []
               end
-MAX_SIZE = 2 * 1024 * 1024 # 2.megabytes
+#MAX_SIZE = 2 * 1024 * 1024 # 2.megabytes
 url = 'http://file-store.rosalinux.ru/api/v1/file_stores.json'
 #url = 'http://localhost:3001/api/v1/file_stores.json'
 rclient = RestClient::Resource.new(url, :user => ARGF.argv[0]) # user auth token
 
-Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each do |file|
+Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z,tar\.xz}").uniq.sort.each do |file|
   begin
-    puts " work with file \"#{file}\""
+    #next if File.size(file) < MAX_SIZE
-    next if File.size(file) < MAX_SIZE
 
     sha1 = Digest::SHA1.file(file).hexdigest
     resp = JSON(RestClient.get url, :params => {:hash => sha1})
@@ -24,12 +23,16 @@ Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each
       # file already exists at file-store
       new_sources << "  \"#{file}\": #{sha1}"
       FileUtils.rm_rf file
+      puts " file \"#{file}\" already exists in the file-store"
     elsif resp == []
       # try to put file at file-store
       resp = JSON(rclient.post :file_store => {:file => File.new(file, 'rb')})
       unless resp['sha1_hash'].nil?
-        new_sources << "  \"#{file}\" #{sha1}"
+        new_sources << "  \"#{file}\": #{sha1}"
         FileUtils.rm_rf file
+         p " upload file \"#{file}\" to the file-store"
+      else
+        p " !Failed to upload file \"#{file}\" to the file-store!"
       end
     else
       raise "Response unknown!\n #{resp}"
@@ -39,8 +42,10 @@ Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each
   #  e.response
   end
 end
-
+sources = (old_sources | new_sources)
-File.open(abf_yml, 'w') do |abf|
+unless sources.empty?
-  abf.puts 'sources:'
+  File.open(abf_yml, 'w') do |abf|
-  (old_sources | new_sources).sort.each {|line| abf.puts line}
+    abf.puts 'sources:'
+    (old_sources | new_sources).sort.each {|line| abf.puts line}
+  end
 end

bin/import_srpm.sh

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 # import_srpm.sh: Import SRPM packages to git repo
-
+pwd=$PWD
 # Input data
 srpm_path=$1
 git_path=$2
 git_branch=$3
+token=$4
 name=$(rpm -q --qf '[%{Name}]' -p $srpm_path)
 version=$(rpm -q --qf '[%{Version}-%{Release}]' -p $srpm_path)
 tmp_dir=/tmp/$name-$version-$RANDOM
@@ -30,6 +31,9 @@ rpm2cpio $srpm_path > srpm.cpio
 cpio -idv < srpm.cpio
 rm -f srpm.cpio
 
+# Remove archives
+$pwd/file-store.rb $token $PWD
+
 # Commit and push changes
 git add -A .
 git commit -m "Automatic import for version $version"

bin/remove_archives.rb

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+# git_dir_projects[0] dest_git_path[1] clone_path[2] owner[3] project_name[4]
+
+require 'fileutils'
+require 'digest'
+
+token = '[CENSORED]'
+
+owners = ARGF.argv[3] || '[a-z0-9_]*'
+project_names = ARGF.argv[4] || '[a-zA-Z0-9_\-\+\.]*'
+
+begin_time = Time.now
+pr_count = total_count = 0
+
+Dir.chdir ARGF.argv[0]
+Dir.glob(owners).each do |owner|
+  Dir.chdir "#{ARGF.argv[0]}/#{owner}"
+  Dir.glob(project_names).each do |project|
+    name_with_owner = "#{owner}/#{project.chomp('.git')}"
+    project_path = "#{ARGF.argv[0]}/#{name_with_owner}.git"
+    dest_project_path = "#{ARGF.argv[1]}/#{name_with_owner}.git"
+    time, total_count = Time.now, total_count + 1
+    Dir.chdir project_path
+    project_stats = "#{name_with_owner}: #{total_count}"
+    if system('git log -n 1 --oneline > /dev/null 2>&1') == false
+      p "Skipping empty project #{project_stats}"
+    else
+      p "Start working with #{project_stats}"
+      path = "#{ARGF.argv[2].chomp('/')}/repos/#{name_with_owner}"
+      FileUtils.rm_rf path
+      #-- hack for refs/heads (else git branch return only master)
+      system "git clone --mirror #{project_path} #{path}/.git"
+      system "cd #{path} && git config --bool core.bare false && git checkout -f HEAD"
+      #--
+      Dir.chdir(path)
+      archives_exists = false
+      %w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z tar.xz).each do |ext|
+        archives_exists=true and break unless `git log --all --format='%H' -- *.#{ext}`.empty?
+      end
+
+      if archives_exists
+        system "git filter-branch -d /dev/shm/git_task --tree-filter \"/home/rosa/git_task/file-store.rb #{token} #{path}\" --prune-empty --tag-name-filter cat -- --all"
+        #####
+        # This is dangerous !!!
+        system "rm -rf #{dest_project_path} && git clone --bare #{path} #{dest_project_path}"
+        #####
+
+        p "Worked with #{name_with_owner}: #{(Time.now - time).truncate} sec."
+        pr_count +=1
+      else
+        p "Skipping project with no archives #{project_stats}"
+      end
+      `rm -rf #{path} && cd #{dest_project_path} && git gc --prune=now`
+    end
+    p '-------------'
+  end
+end
+p '======================='
+p "Total count of projects are #{total_count}"
+p "Finished work with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}"

config/locales/models/pull_request.en.yml

@@ -15,6 +15,7 @@ en:
       duplicate: 'There is already a pull request for %{from_ref}'
       up_to_date: 'The %{to_ref} branch is already up-to-date with %{from_ref}'
       wrong_ref: Wrong branch or tag
+      empty_repo: Empty git repository
       blocked: This pull request cannot be automatically merged.
       ready: This pull request can be automatically merged.
       merged: |

config/locales/models/pull_request.ru.yml

@@ -15,6 +15,8 @@ ru:
       duplicate: 'Уже существует пул реквест %{from_ref}'
       up_to_date: 'Ветка %{to_ref} на данный момент уже содержит последние изменения %{from_ref}'
       wrong_ref: Неправильная ветка или тег
+      empty_repo: Пустой гит-репозиторий
+
       blocked: Невозможно автоматически смержить данный пул реквест.
       ready: Данный пул реквест можно смержить автоматически.
       merged: |

db/migrate/20121027084602_add_helpers_columns_to_pull_request.rb

@@ -0,0 +1,18 @@
+class AddHelpersColumnsToPullRequest < ActiveRecord::Migration
+  def up
+    add_column :pull_requests, :from_project_owner_uname, :string
+    add_column :pull_requests, :from_project_name, :string
+    # includes generate error "undefined method `repo' for nil:NilClass"
+    # update not orphan pulls. For other need execute a task project:fix_orphan_pulls
+    PullRequest.joins(:from_project).each do |pull|
+      pull.from_project_name = pull.from_project.name
+      pull.from_project_owner_uname = pull.from_project.owner.uname
+      pull.save
+    end
+  end
+
+  def down
+    remove_column :pull_requests, :from_project_owner_uname
+    remove_column :pull_requests, :from_project_name
+  end
+end

db/schema.rb

@@ -348,11 +348,13 @@ ActiveRecord::Schema.define(:version => 20121106113338) do
   add_index "projects", ["owner_id", "name", "owner_type"], :name => "index_projects_on_name_and_owner_id_and_owner_type", :unique => true, :case_sensitive => false
 
   create_table "pull_requests", :force => true do |t|
-    t.integer "issue_id",        :null => false
+    t.integer "issue_id",                 :null => false
-    t.integer "to_project_id",   :null => false
+    t.integer "to_project_id",            :null => false
-    t.integer "from_project_id", :null => false
+    t.integer "from_project_id",          :null => false
-    t.string  "to_ref",          :null => false
+    t.string  "to_ref",                   :null => false
-    t.string  "from_ref",        :null => false
+    t.string  "from_ref",                 :null => false
+    t.string  "from_project_owner_uname"
+    t.string  "from_project_name"
   end
 
   add_index "pull_requests", ["from_project_id"], :name => "index_pull_requests_on_head_project_id"

lib/modules/models/git.rb

@@ -92,7 +92,8 @@ module Modules
       end
 
       def import_srpm(srpm_path = srpm.path, branch_name = 'import')
-        system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} >> /dev/null 2>&1")
+        token = User.find_by_uname('rosa_system').authentication_token
+        system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} #{token} >> /dev/null 2>&1")
       end
 
       protected

lib/tasks/fix_orphan_pulls.rake

@@ -0,0 +1,26 @@
+namespace :project do
+  desc 'Fix pull requests where was delete the "from project"'
+  task :fix_orphan_pulls => :environment do
+    projects = Project.where('ancestry IS NOT NULL')
+    say "Pull requests total count is #{PullRequest.count}"
+    PullRequest.all.each_with_index do |pull, ind|
+      say "Check pull with id:#{pull.id} (#{ind+1}/#{PullRequest.count})"
+      if pull.from_project.present?
+        print ' updating...'
+        pull.from_project_name = pull.from_project.name
+        pull.from_project_owner_uname = pull.from_project.owner.uname
+        say pull.save(:validate => false) ? 'success' : 'fail!'
+      else
+        print '  its orphan! updating...'
+        parent_path = File.join(APP_CONFIG['git_path'], 'pull_requests', pull.to_project.owner.uname, pull.to_project.name)
+        Dir.chdir(parent_path) do
+          # Get an owner and project name from the pull dir
+          elements = Dir["#{pull.id}-*"].first.split '-' rescue []
+          pull.from_project_owner_uname, pull.from_project_name = elements[1], elements[2]
+          say pull.save(:validate => false) ? 'success' : 'fail!'
+        end
+      end
+    end
+  end
+end
+

lib/tasks/import.rake

@@ -81,9 +81,9 @@ namespace :import do
       # system("bundle exec rake import:sync:run RELEASE=official/2011 PLATFORM=mandriva2011 REPOSITORY=main")
       # system("bundle exec rake import:sync:run RELEASE=official/2011 PLATFORM=mandriva2011 REPOSITORY=contrib")
       # system("bundle exec rake import:sync:run RELEASE=official/2011 PLATFORM=mandriva2011 REPOSITORY=non-free")
-      system("bundle exec rake import:sync:run RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=main")
+      #system("bundle exec rake import:sync:run RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=main")
-      system("bundle exec rake import:sync:run RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=contrib")
+      #system("bundle exec rake import:sync:run RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=contrib")
-      system("bundle exec rake import:sync:run RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=non-free")
+      #system("bundle exec rake import:sync:run RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=non-free")
       system("bundle exec rake import:sync:run SOURCE=rsync://mirror.yandex.ru/fedora-epel/6/SRPMS/ DESTINATION=#{File.join(APP_CONFIG['root_path'], 'mirror.yandex.ru', 'fedora-epel', '6', 'SRPMS')} PLATFORM=server_personal REPOSITORY=main OWNER=server BRANCH=import")
       system("bundle exec rake import:sync:run SOURCE=rsync://rh-mirror.redhat.com/redhat/linux/enterprise/6Server/en/os/SRPMS/ DESTINATION=#{File.join(APP_CONFIG['root_path'], 'rh-mirror.redhat.com', 'redhat', 'linux', 'enterprise', '6Server', 'en', 'os', 'SRPMS')} PLATFORM=server_personal REPOSITORY=main OWNER=server BRANCH=import")
     end

lib/tasks/remove_archives_from_git.rake

@@ -1,73 +0,0 @@
-namespace :project do
-  desc "Truncate blobs from git repo"
-  task :remove_archives => :environment do
-    #raise 'Need set GIT_PROJECTS_DIR' if ENV['GIT_PROJECTS_DIR'].blank?
-    raise 'Need set CLONE_PATH' if ENV['CLONE_PATH'].blank?
-    raise 'Need special "rosa_system" user' unless User.where(:uname => 'rosa_system').exists?
-    token = User.find_by_uname('rosa_system').authentication_token
-
-    abf_existing_log = File.open "#{ENV['CLONE_PATH']}/projects_with_abf_yml.log", 'w'
-    projects = if uname = ENV['OWNER']
-                 owner = User.find_by_uname(uname) || Group.find_by_uname(uname)
-                 owner.projects
-               else
-                 Project.scoped
-               end
-
-    projects = projects.where :id => eval(ENV['PROJECT_ID']) if ENV['PROJECT_ID']
-    total_count = projects.count
-    #FileUtils.mkdir_p "#{ENV['CLONE_PATH']}/archives" if total_count > 0
-    begin_time = Time.now
-    pr_count = 0
-    projects.each_with_index do |project, ind|
-      project_stats = "#{project.name_with_owner}: #{ind+1}/#{total_count}"
-      if project.repo.commits.count == 0
-        say "Skipping empty project #{project_stats}"
-      else
-        say "Start working with #{project_stats}"
-        time = Time.now
-        path = "#{ENV['CLONE_PATH'].chomp('/')}/repos/#{project.name_with_owner}"
-        FileUtils.rm_rf path
-        project_path = project.path#"#{ENV['GIT_PROJECTS_DIR']}/#{project.name_with_owner}/.git"
-        archives_exists = false
-        Dir.chdir(project_path) do
-          %w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z).each do |ext|
-            archives_exists=true and break if `git log --all --format='%H' -- *.#{ext}`.present?
-          end
-        end
-        #if `cd #{project_path} && git log --all -- abf.yml`.present? # already have abf.yml in repo?
-        #  message="Skipping project with abf.yml file #{project_stats}"
-        #  say message
-        #  abf_existing_log.puts message
-        #  `rm -rf #{path}`
-        #elsif archives_exists.present?
-        if archives_exists.present?
-          #-- hack for refs/heads (else git branch return only master)
-          system "git clone --mirror #{project_path} #{path}/.git"
-          system "cd #{path} && git config --bool core.bare false && git checkout -f HEAD"
-          system "bin/calc_sha #{path} #{token}" # FIXME change filename
-          #--
-
-          #####
-          # This is dangerous !!!
-          #system "rm -rf #{project_path} && git clone --bare #{path} #{project_path} && rm -rf #{path}"
-          system "rm -rf #{path}"
-          #####
-
-          say "Worked with #{project.name_with_owner}: #{(Time.now - time).truncate} sec."
-          pr_count +=1
-        else
-          message="Skipping project with no archives #{project_stats}"
-          say message
-          abf_existing_log.puts message
-          `rm -rf #{path}`
-        end
-        say '-------------'
-      end
-    end
-    say '======================='
-    say "Total count of projects are #{total_count}"
-    say "Finished with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}"
-    abf_existing_log.close
-  end
-end