Merge pull request #132 from warpc/112-sync_projects

Auto sync for mandriva repo

* Apply rsync for mandriva repo
* Parse and import changes from official repo
* Fix attachment types
* Redo srpm upload wrapper to use it without paperclip
* Add cron task through whenever

app/models/group.rb

@@ -21,6 +21,7 @@ class Group < ActiveRecord::Base
   delegate :ssh_key, :to => :owner
 
   after_create :add_owner_to_members
+  after_initialize lambda {|r| r.name ||= r.uname } # default
 
   include Modules::Models::PersonalRepository
 #  include Modules::Models::Owner

app/models/project.rb

@@ -8,7 +8,7 @@ class Project < ActiveRecord::Base
   has_many :build_lists, :dependent => :destroy
   has_many :auto_build_lists, :dependent => :destroy
 
-  # has_many :project_imports, :dependent => :destroy
+  has_many :project_imports, :dependent => :destroy
   has_many :project_to_repositories, :dependent => :destroy
   has_many :repositories, :through => :project_to_repositories
 
@@ -20,7 +20,7 @@ class Project < ActiveRecord::Base
   validates :owner, :presence => true
   # validate {errors.add(:base, I18n.t('flash.project.save_warning_ssh_key')) if owner.ssh_key.blank?}
   validates_attachment_size :srpm, :less_than => 500.megabytes
-  validates_attachment_content_type :srpm, :content_type => ['application/octet-stream'], :message => I18n.t('layout.invalid_content_type') # "application/x-rpm", "application/x-redhat-package-manager" ?
+  validates_attachment_content_type :srpm, :content_type => ['application/octet-stream', "application/x-rpm", "application/x-redhat-package-manager"], :message => I18n.t('layout.invalid_content_type')
 
   #attr_accessible :category_id, :name, :description, :visibility
   attr_readonly :name
@@ -34,7 +34,7 @@ class Project < ActiveRecord::Base
   after_create :attach_to_personal_repository
   after_create :create_git_repo
   after_destroy :destroy_git_repo
-  after_save {|p| p.delay.import_srpm if p.srpm?} # should be after create_git_repo
+  after_save {|p| p.delay.import_attached_srpm if p.srpm?} # should be after create_git_repo
   # after_rollback lambda { destroy_git_repo rescue true if new_record? }
 
   has_ancestry
@@ -143,11 +143,8 @@ class Project < ActiveRecord::Base
     @platforms ||= repositories.map(&:platform).uniq
   end
 
-  def import_srpm(branch_name = 'import')
-    if srpm?
-      system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm.path} #{path} #{branch_name} >> /dev/null 2>&1")
-      self.srpm = nil; save # clear srpm
-    end
+  def import_srpm(srpm_path = srpm.path, branch_name = 'import')
+    system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} >> /dev/null 2>&1")
   end
 
   class << self
@@ -159,19 +156,26 @@ class Project < ActiveRecord::Base
 
   protected
 
-    def build_path(dir)
-      File.join(APP_CONFIG['root_path'], 'git_projects', "#{dir}.git")
-    end
+  def build_path(dir)
+    File.join(APP_CONFIG['root_path'], 'git_projects', "#{dir}.git")
+  end
 
-    def attach_to_personal_repository
-      repositories << self.owner.personal_repository if !repositories.exists?(:id => self.owner.personal_repository)
-    end
+  def attach_to_personal_repository
+    repositories << self.owner.personal_repository if !repositories.exists?(:id => self.owner.personal_repository)
+  end
 
-    def create_git_repo
-      is_root? ? Grit::Repo.init_bare(path) : parent.git_repository.repo.delay.fork_bare(path)
-    end
+  def create_git_repo
+    is_root? ? Grit::Repo.init_bare(path) : parent.git_repository.repo.delay.fork_bare(path)
+  end
 
-    def destroy_git_repo
-      FileUtils.rm_rf path
+  def destroy_git_repo
+    FileUtils.rm_rf path
+  end
+
+  def import_attached_srpm
+    if srpm?
+      import_srpm # srpm.path
+      self.srpm = nil; save # clear srpm
     end
+  end
 end

app/models/project_import.rb

@@ -0,0 +1,5 @@
+class ProjectImport < ActiveRecord::Base
+  belongs_to :project
+
+  after_initialize lambda {|r| r.file_mtime ||= Time.current - 10.years } # default
+end

config/schedule.rb

@@ -11,3 +11,12 @@ every 5.minutes do
   runner "Download.rotate_nginx_log"
   runner "Download.parse_and_remove_nginx_log"
 end
+
+every 1.day, :at => '4:00 am' do
+ rake "import:sync:all" # RELEASE=official/2011 PLATFORM=mandriva2011 REPOSITORY=main
+ rake "import:sync:all REPOSITORY=contrib" # RELEASE=official/2011 PLATFORM=mandriva2011
+ rake "import:sync:all REPOSITORY=non-free" # RELEASE=official/2011 PLATFORM=mandriva2011
+ rake "import:sync:all RELEASE=devel/cooker PLATFORM=cooker" # REPOSITORY=main
+ rake "import:sync:all RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=contrib"
+ rake "import:sync:all RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=non-free"
+end

db/migrate/20120117210132_create_project_imports.rb

@@ -0,0 +1,16 @@
+class CreateProjectImports < ActiveRecord::Migration
+  def self.up
+    create_table :project_imports do |t|
+      t.references :project
+      t.string :name
+      t.string :version
+      t.datetime :file_mtime
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :project_imports
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20120117110723) do
+ActiveRecord::Schema.define(:version => 20120124101727) do
 
   create_table "arches", :force => true do |t|
     t.string   "name",       :null => false
@@ -167,13 +167,6 @@ ActiveRecord::Schema.define(:version => 20120117110723) do
 
   add_index "issues", ["project_id", "serial_id"], :name => "index_issues_on_project_id_and_serial_id", :unique => true
 
-  create_table "permissions", :force => true do |t|
-    t.integer  "right_id"
-    t.integer  "role_id"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-  end
-
   create_table "platforms", :force => true do |t|
     t.string   "description"
     t.string   "name"
@@ -228,6 +221,15 @@ ActiveRecord::Schema.define(:version => 20120117110723) do
     t.boolean  "use_cron",         :default => false
   end
 
+  create_table "project_imports", :force => true do |t|
+    t.integer  "project_id"
+    t.string   "name"
+    t.string   "version"
+    t.datetime "file_mtime"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
   create_table "project_to_repositories", :force => true do |t|
     t.integer  "project_id"
     t.integer  "repository_id"
@@ -241,11 +243,15 @@ ActiveRecord::Schema.define(:version => 20120117110723) do
     t.datetime "updated_at"
     t.integer  "owner_id"
     t.string   "owner_type"
-    t.string   "visibility",  :default => "open"
+    t.string   "visibility",        :default => "open"
     t.integer  "category_id"
     t.text     "description"
     t.string   "ancestry"
-    t.boolean  "has_issues",  :default => true
+    t.boolean  "has_issues",        :default => true
+    t.string   "srpm_content_type"
+    t.datetime "srpm_updated_at"
+    t.integer  "srpm_file_size"
+    t.string   "srpm_file_name"
   end
 
   add_index "projects", ["category_id"], :name => "index_projects_on_category_id"
@@ -270,14 +276,6 @@ ActiveRecord::Schema.define(:version => 20120117110723) do
     t.string   "owner_type"
   end
 
-  create_table "rights", :force => true do |t|
-    t.string   "name",       :null => false
-    t.string   "controller", :null => false
-    t.string   "action",     :null => false
-    t.datetime "created_at"
-    t.datetime "updated_at"
-  end
-
   create_table "rpms", :force => true do |t|
     t.string   "name",       :null => false
     t.integer  "arch_id",    :null => false

lib/tasks/import.rake

@@ -25,4 +25,73 @@ namespace :import do
     end
     say 'DONE'
   end
+
+  namespace :sync do
+    task :all => [:rsync, :parse]
+
+    desc "Rsync with mirror.yandex.ru"
+    task :rsync => :environment do
+      release = ENV['RELEASE'] || 'official/2011'
+      repository = ENV['REPOSITORY'] || 'main'
+      source = "rsync://mirror.yandex.ru/mandriva/#{release}/SRPMS/#{repository}/"
+      destination = ENV['DESTINATION'] || File.join(APP_CONFIG['root_path'], 'mirror.yandex.ru', 'mandriva', release, 'SRPMS', repository)
+      say "START rsync projects (*.src.rpm) from '#{source}' to '#{destination}'"
+      if system "rsync -rtv --delete #{source} #{destination}" # TODO --include='*.src.rpm' --exclude='*'
+        say 'Rsync ok!'
+      else
+        say 'Rsync failed!'
+      end
+      say 'DONE'
+    end
+
+    desc "Parse repository for changes"
+    task :parse => :environment do
+      release = ENV['RELEASE'] || 'official/2011'
+      platform = Platform.find_by_name(ENV['PLATFORM'] || "mandriva2011")
+      repository = platform.repositories.find_by_name(ENV['REPOSITORY'] || 'main')
+      source = ENV['SOURCE'] || File.join(APP_CONFIG['root_path'], 'mirror.yandex.ru', 'mandriva', release, 'SRPMS', repository.name)
+      owner = Group.find_or_create_by_uname(ENV['OWNER'] || 'import') {|g| g.owner = User.first}
+      branch = "import_#{platform.name}"
+
+      say 'START'
+      Dir[File.join source, '{release,updates}', '*.src.rpm'].each do |srpm_file|
+        say "=== Processing '#{srpm_file}'..."
+        if name = `rpm -q --qf '[%{Name}]' -p #{srpm_file}` and $?.success? and name.present? and
+           version = `rpm -q --qf '[%{Version}]' -p #{srpm_file}` and $?.success? and version.present?
+          project_import = ProjectImport.find_or_initialize_by_name name
+          if version != project_import.version.to_s and File.mtime(srpm_file) > project_import.file_mtime
+            unless project = project_import.project
+              if project = repository.projects.find_by_name(name)
+                say "Found project '#{project.owner.uname}/#{project.name}'"
+              elsif project = Project.where(:name => name, :owner_id => owner.id, :owner_type => owner.class.to_s).first
+                repository.projects << project
+                say "Add project '#{project.owner.uname}/#{project.name}' to '#{platform.name}/#{repository.name}'"
+              else
+                description = ::Iconv.conv('UTF-8//IGNORE', 'UTF-8', `rpm -q --qf '[%{Description}]' -p #{srpm_file}`)
+                project = Project.create!(:name => name, :description => description) {|p| p.owner = owner}
+                repository.projects << project
+                say "Create project #{project.owner.uname}/#{project.name} in #{platform.name}/#{repository.name}"
+              end
+            end
+            project.import_srpm(srpm_file, branch)
+            say "New version (#{version}) for '#{project.owner.uname}/#{project.name}' successfully imported to branch '#{branch}'!"
+
+            project_import.project = project
+            project_import.version = version
+            project_import.file_mtime = File.mtime(srpm_file)
+            project_import.save!
+
+            # TODO notify import.members
+
+            say '=== Success!'
+          else
+            say '=== Not changed!'
+          end
+        else
+          say '=== Fail!'
+        end
+      end
+      say 'DONE'
+    end
+  end
 end

spec/factories/project_imports.rb

@@ -0,0 +1,9 @@
+# Read about factories at http://github.com/thoughtbot/factory_girl
+
+FactoryGirl.define do
+  factory :project_import do
+      project nil
+      name "MyString"
+      version "MyString"
+    end
+end

spec/models/project_import_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe ProjectImport do
+  pending "add some examples to (or delete) #{__FILE__}"
+end