diff --git a/app/models/group.rb b/app/models/group.rb index 9e8e15905..335804b76 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -21,6 +21,7 @@ class Group < ActiveRecord::Base delegate :ssh_key, :to => :owner after_create :add_owner_to_members + after_initialize lambda {|r| r.name ||= r.uname } # default include Modules::Models::PersonalRepository # include Modules::Models::Owner diff --git a/app/models/project.rb b/app/models/project.rb index b8f3d385a..404da8443 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -8,7 +8,7 @@ class Project < ActiveRecord::Base has_many :build_lists, :dependent => :destroy has_many :auto_build_lists, :dependent => :destroy - # has_many :project_imports, :dependent => :destroy + has_many :project_imports, :dependent => :destroy has_many :project_to_repositories, :dependent => :destroy has_many :repositories, :through => :project_to_repositories @@ -20,7 +20,7 @@ class Project < ActiveRecord::Base validates :owner, :presence => true # validate {errors.add(:base, I18n.t('flash.project.save_warning_ssh_key')) if owner.ssh_key.blank?} validates_attachment_size :srpm, :less_than => 500.megabytes - validates_attachment_content_type :srpm, :content_type => ['application/octet-stream'], :message => I18n.t('layout.invalid_content_type') # "application/x-rpm", "application/x-redhat-package-manager" ? + validates_attachment_content_type :srpm, :content_type => ['application/octet-stream', "application/x-rpm", "application/x-redhat-package-manager"], :message => I18n.t('layout.invalid_content_type') #attr_accessible :category_id, :name, :description, :visibility attr_readonly :name @@ -34,7 +34,7 @@ class Project < ActiveRecord::Base after_create :attach_to_personal_repository after_create :create_git_repo after_destroy :destroy_git_repo - after_save {|p| p.delay.import_srpm if p.srpm?} # should be after create_git_repo + after_save {|p| p.delay.import_attached_srpm if p.srpm?} # should be after create_git_repo # after_rollback lambda { destroy_git_repo rescue true if new_record? } has_ancestry @@ -143,11 +143,8 @@ class Project < ActiveRecord::Base @platforms ||= repositories.map(&:platform).uniq end - def import_srpm(branch_name = 'import') - if srpm? - system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm.path} #{path} #{branch_name} >> /dev/null 2>&1") - self.srpm = nil; save # clear srpm - end + def import_srpm(srpm_path = srpm.path, branch_name = 'import') + system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} >> /dev/null 2>&1") end class << self @@ -159,19 +156,26 @@ class Project < ActiveRecord::Base protected - def build_path(dir) - File.join(APP_CONFIG['root_path'], 'git_projects', "#{dir}.git") - end + def build_path(dir) + File.join(APP_CONFIG['root_path'], 'git_projects', "#{dir}.git") + end - def attach_to_personal_repository - repositories << self.owner.personal_repository if !repositories.exists?(:id => self.owner.personal_repository) - end + def attach_to_personal_repository + repositories << self.owner.personal_repository if !repositories.exists?(:id => self.owner.personal_repository) + end - def create_git_repo - is_root? ? Grit::Repo.init_bare(path) : parent.git_repository.repo.delay.fork_bare(path) - end + def create_git_repo + is_root? ? Grit::Repo.init_bare(path) : parent.git_repository.repo.delay.fork_bare(path) + end - def destroy_git_repo - FileUtils.rm_rf path + def destroy_git_repo + FileUtils.rm_rf path + end + + def import_attached_srpm + if srpm? + import_srpm # srpm.path + self.srpm = nil; save # clear srpm end + end end diff --git a/app/models/project_import.rb b/app/models/project_import.rb new file mode 100644 index 000000000..23e99ede7 --- /dev/null +++ b/app/models/project_import.rb @@ -0,0 +1,5 @@ +class ProjectImport < ActiveRecord::Base + belongs_to :project + + after_initialize lambda {|r| r.file_mtime ||= Time.current - 10.years } # default +end diff --git a/config/schedule.rb b/config/schedule.rb index a9b4c3440..bafa716c6 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -11,3 +11,12 @@ every 5.minutes do runner "Download.rotate_nginx_log" runner "Download.parse_and_remove_nginx_log" end + +every 1.day, :at => '4:00 am' do + rake "import:sync:all" # RELEASE=official/2011 PLATFORM=mandriva2011 REPOSITORY=main + rake "import:sync:all REPOSITORY=contrib" # RELEASE=official/2011 PLATFORM=mandriva2011 + rake "import:sync:all REPOSITORY=non-free" # RELEASE=official/2011 PLATFORM=mandriva2011 + rake "import:sync:all RELEASE=devel/cooker PLATFORM=cooker" # REPOSITORY=main + rake "import:sync:all RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=contrib" + rake "import:sync:all RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=non-free" +end diff --git a/db/migrate/20120117210132_create_project_imports.rb b/db/migrate/20120117210132_create_project_imports.rb new file mode 100644 index 000000000..28231a844 --- /dev/null +++ b/db/migrate/20120117210132_create_project_imports.rb @@ -0,0 +1,16 @@ +class CreateProjectImports < ActiveRecord::Migration + def self.up + create_table :project_imports do |t| + t.references :project + t.string :name + t.string :version + t.datetime :file_mtime + + t.timestamps + end + end + + def self.down + drop_table :project_imports + end +end diff --git a/db/schema.rb b/db/schema.rb index 825f237ba..5cfb1d6a2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 20120117110723) do +ActiveRecord::Schema.define(:version => 20120124101727) do create_table "arches", :force => true do |t| t.string "name", :null => false @@ -167,13 +167,6 @@ ActiveRecord::Schema.define(:version => 20120117110723) do add_index "issues", ["project_id", "serial_id"], :name => "index_issues_on_project_id_and_serial_id", :unique => true - create_table "permissions", :force => true do |t| - t.integer "right_id" - t.integer "role_id" - t.datetime "created_at" - t.datetime "updated_at" - end - create_table "platforms", :force => true do |t| t.string "description" t.string "name" @@ -228,6 +221,15 @@ ActiveRecord::Schema.define(:version => 20120117110723) do t.boolean "use_cron", :default => false end + create_table "project_imports", :force => true do |t| + t.integer "project_id" + t.string "name" + t.string "version" + t.datetime "file_mtime" + t.datetime "created_at" + t.datetime "updated_at" + end + create_table "project_to_repositories", :force => true do |t| t.integer "project_id" t.integer "repository_id" @@ -241,11 +243,15 @@ ActiveRecord::Schema.define(:version => 20120117110723) do t.datetime "updated_at" t.integer "owner_id" t.string "owner_type" - t.string "visibility", :default => "open" + t.string "visibility", :default => "open" t.integer "category_id" t.text "description" t.string "ancestry" - t.boolean "has_issues", :default => true + t.boolean "has_issues", :default => true + t.string "srpm_content_type" + t.datetime "srpm_updated_at" + t.integer "srpm_file_size" + t.string "srpm_file_name" end add_index "projects", ["category_id"], :name => "index_projects_on_category_id" @@ -270,14 +276,6 @@ ActiveRecord::Schema.define(:version => 20120117110723) do t.string "owner_type" end - create_table "rights", :force => true do |t| - t.string "name", :null => false - t.string "controller", :null => false - t.string "action", :null => false - t.datetime "created_at" - t.datetime "updated_at" - end - create_table "rpms", :force => true do |t| t.string "name", :null => false t.integer "arch_id", :null => false diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 88f65117d..2ed133ff3 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -25,4 +25,73 @@ namespace :import do end say 'DONE' end + + namespace :sync do + task :all => [:rsync, :parse] + + desc "Rsync with mirror.yandex.ru" + task :rsync => :environment do + release = ENV['RELEASE'] || 'official/2011' + repository = ENV['REPOSITORY'] || 'main' + source = "rsync://mirror.yandex.ru/mandriva/#{release}/SRPMS/#{repository}/" + destination = ENV['DESTINATION'] || File.join(APP_CONFIG['root_path'], 'mirror.yandex.ru', 'mandriva', release, 'SRPMS', repository) + say "START rsync projects (*.src.rpm) from '#{source}' to '#{destination}'" + if system "rsync -rtv --delete #{source} #{destination}" # TODO --include='*.src.rpm' --exclude='*' + say 'Rsync ok!' + else + say 'Rsync failed!' + end + say 'DONE' + end + + desc "Parse repository for changes" + task :parse => :environment do + release = ENV['RELEASE'] || 'official/2011' + platform = Platform.find_by_name(ENV['PLATFORM'] || "mandriva2011") + repository = platform.repositories.find_by_name(ENV['REPOSITORY'] || 'main') + source = ENV['SOURCE'] || File.join(APP_CONFIG['root_path'], 'mirror.yandex.ru', 'mandriva', release, 'SRPMS', repository.name) + owner = Group.find_or_create_by_uname(ENV['OWNER'] || 'import') {|g| g.owner = User.first} + branch = "import_#{platform.name}" + + say 'START' + Dir[File.join source, '{release,updates}', '*.src.rpm'].each do |srpm_file| + say "=== Processing '#{srpm_file}'..." + if name = `rpm -q --qf '[%{Name}]' -p #{srpm_file}` and $?.success? and name.present? and + version = `rpm -q --qf '[%{Version}]' -p #{srpm_file}` and $?.success? and version.present? + project_import = ProjectImport.find_or_initialize_by_name name + if version != project_import.version.to_s and File.mtime(srpm_file) > project_import.file_mtime + unless project = project_import.project + if project = repository.projects.find_by_name(name) + say "Found project '#{project.owner.uname}/#{project.name}'" + elsif project = Project.where(:name => name, :owner_id => owner.id, :owner_type => owner.class.to_s).first + repository.projects << project + say "Add project '#{project.owner.uname}/#{project.name}' to '#{platform.name}/#{repository.name}'" + else + description = ::Iconv.conv('UTF-8//IGNORE', 'UTF-8', `rpm -q --qf '[%{Description}]' -p #{srpm_file}`) + project = Project.create!(:name => name, :description => description) {|p| p.owner = owner} + repository.projects << project + say "Create project #{project.owner.uname}/#{project.name} in #{platform.name}/#{repository.name}" + end + end + project.import_srpm(srpm_file, branch) + say "New version (#{version}) for '#{project.owner.uname}/#{project.name}' successfully imported to branch '#{branch}'!" + + project_import.project = project + project_import.version = version + project_import.file_mtime = File.mtime(srpm_file) + project_import.save! + + # TODO notify import.members + + say '=== Success!' + else + say '=== Not changed!' + end + else + say '=== Fail!' + end + end + say 'DONE' + end + end end diff --git a/spec/factories/project_imports.rb b/spec/factories/project_imports.rb new file mode 100644 index 000000000..e52c8a374 --- /dev/null +++ b/spec/factories/project_imports.rb @@ -0,0 +1,9 @@ +# Read about factories at http://github.com/thoughtbot/factory_girl + +FactoryGirl.define do + factory :project_import do + project nil + name "MyString" + version "MyString" + end +end \ No newline at end of file diff --git a/spec/models/project_import_spec.rb b/spec/models/project_import_spec.rb new file mode 100644 index 000000000..e7448f1d6 --- /dev/null +++ b/spec/models/project_import_spec.rb @@ -0,0 +1,5 @@ +require 'spec_helper' + +describe ProjectImport do + pending "add some examples to (or delete) #{__FILE__}" +end