diff --git a/bin/calc_sha b/bin/calc_sha deleted file mode 100755 index 43b4c3fff..000000000 --- a/bin/calc_sha +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -pwd=`pwd` -cd $1 -git filter-branch --tree-filter "$pwd/bin/file-store.rb $2" --prune-empty --tag-name-filter cat -- --all -cd .. diff --git a/bin/file-store.rb b/bin/file-store.rb index e481d907d..70d457473 100755 --- a/bin/file-store.rb +++ b/bin/file-store.rb @@ -2,7 +2,7 @@ require 'json' require 'rest-client' -abf_yml, new_sources = 'abf.yml', [] +abf_yml, new_sources = '.abf.yml', [] old_sources = if File.exist? abf_yml File.read(abf_yml).split("\n").reject {|line| line =~ /sources/} else @@ -15,7 +15,7 @@ rclient = RestClient::Resource.new(url, :user => ARGF.argv[0]) # user auth token Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each do |file| begin - puts " work with file \"#{file}\"" + #puts " work with file \"#{file}\"" next if File.size(file) < MAX_SIZE sha1 = Digest::SHA1.file(file).hexdigest @@ -24,12 +24,14 @@ Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each # file already exists at file-store new_sources << " \"#{file}\": #{sha1}" FileUtils.rm_rf file + puts " file \"#{file}\" already exists in the file-store" elsif resp == [] # try to put file at file-store resp = JSON(rclient.post :file_store => {:file => File.new(file, 'rb')}) unless resp['sha1_hash'].nil? - new_sources << " \"#{file}\" #{sha1}" + new_sources << " \"#{file}\": #{sha1}" FileUtils.rm_rf file + puts " upload file \"#{file}\" to the file-store" end else raise "Response unknown!\n #{resp}" @@ -39,8 +41,12 @@ Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each # e.response end end - -File.open(abf_yml, 'w') do |abf| - abf.puts 'sources:' - (old_sources | new_sources).sort.each {|line| abf.puts line} +sources = (old_sources | new_sources) +unless sources.empty? + Dir.chdir(ARGF.argv[1]) do # Need if in filter-branch + File.open(abf_yml, 'w') do |abf| + abf.puts 'sources:' + (old_sources | new_sources).sort.each {|line| abf.puts line} + end + end end diff --git a/bin/import_srpm.sh b/bin/import_srpm.sh index b82ce4374..02d4bd449 100755 --- a/bin/import_srpm.sh +++ b/bin/import_srpm.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash # import_srpm.sh: Import SRPM packages to git repo - +pwd=$PWD # Input data srpm_path=$1 git_path=$2 git_branch=$3 +token=$4 name=$(rpm -q --qf '[%{Name}]' -p $srpm_path) version=$(rpm -q --qf '[%{Version}-%{Release}]' -p $srpm_path) tmp_dir=/tmp/$name-$version-$RANDOM @@ -30,6 +31,9 @@ rpm2cpio $srpm_path > srpm.cpio cpio -idv < srpm.cpio rm -f srpm.cpio +# Remove archives +$pwd/file-store.rb $token $PWD + # Commit and push changes git add -A . git commit -m "Automatic import for version $version" diff --git a/bin/remove_archives.rb b/bin/remove_archives.rb new file mode 100755 index 000000000..cf009745f --- /dev/null +++ b/bin/remove_archives.rb @@ -0,0 +1,59 @@ +#!/usr/bin/env ruby +# git_dir_projects[0] clone_path[1] owner[2] project_name[3] + +require 'fileutils' +require 'digest' + +token = '[CENSORED]' + +owners = ARGF.argv[2] || '[a-z0-9_]*' +project_names = ARGF.argv[3] || '[a-zA-Z0-9_\-\+\.]*' + +begin_time = Time.now +pr_count = total_count = 0 + +Dir.chdir ARGF.argv[0] +Dir.glob(owners).each do |owner| + Dir.chdir "#{ARGF.argv[0]}/#{owner}" + Dir.glob(project_names).each do |project| + name_with_owner = "#{owner}/#{project.chomp('.git')}" + project_path = "#{ARGF.argv[0]}/#{name_with_owner}.git" + time, total_count = Time.now, total_count + 1 + Dir.chdir project_path + project_stats = "#{name_with_owner}: #{total_count}" + if system('git log -n 1 --oneline > /dev/null 2>&1') == false + p "Skipping empty project #{project_stats}" + else + p "Start working with #{project_stats}" + path = "#{ARGF.argv[1].chomp('/')}/repos/#{name_with_owner}" + FileUtils.rm_rf path + #-- hack for refs/heads (else git branch return only master) + system "git clone --mirror #{project_path} #{path}/.git" + system "cd #{path} && git config --bool core.bare false && git checkout -f HEAD" + #-- + Dir.chdir(path) + archives_exists = false + %w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z).each do |ext| + archives_exists=true and break unless `git log --all --format='%H' -- *.#{ext}`.empty? + end + + if archives_exists + system "git filter-branch -d /dev/shm/git_task --tree-filter \"/home/rosa/git_task/file-store.rb #{token} #{path}\" --prune-empty --tag-name-filter cat -- --all" + ##### + # This is dangerous !!! + system "rm -rf #{project_path} && git clone --bare #{path} #{project_path}" + ##### + + p "Worked with #{name_with_owner}: #{(Time.now - time).truncate} sec." + pr_count +=1 + else + p "Skipping project with no archives #{project_stats}" + end + `rm -rf #{path}` + end + p '-------------' + end +end +p '=======================' +p "Total count of projects are #{total_count}" +p "Finished work with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}" diff --git a/lib/modules/models/git.rb b/lib/modules/models/git.rb index 2ba147db8..d90c029e9 100644 --- a/lib/modules/models/git.rb +++ b/lib/modules/models/git.rb @@ -92,7 +92,8 @@ module Modules end def import_srpm(srpm_path = srpm.path, branch_name = 'import') - system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} >> /dev/null 2>&1") + token = User.find_by_uname('rosa_system').token_authenticatable + system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} #{token} >> /dev/null 2>&1") end protected diff --git a/lib/tasks/remove_archives_from_git.rake b/lib/tasks/remove_archives_from_git.rake deleted file mode 100644 index 4838e561c..000000000 --- a/lib/tasks/remove_archives_from_git.rake +++ /dev/null @@ -1,73 +0,0 @@ -namespace :project do - desc "Truncate blobs from git repo" - task :remove_archives => :environment do - #raise 'Need set GIT_PROJECTS_DIR' if ENV['GIT_PROJECTS_DIR'].blank? - raise 'Need set CLONE_PATH' if ENV['CLONE_PATH'].blank? - raise 'Need special "rosa_system" user' unless User.where(:uname => 'rosa_system').exists? - token = User.find_by_uname('rosa_system').authentication_token - - abf_existing_log = File.open "#{ENV['CLONE_PATH']}/projects_with_abf_yml.log", 'w' - projects = if uname = ENV['OWNER'] - owner = User.find_by_uname(uname) || Group.find_by_uname(uname) - owner.projects - else - Project.scoped - end - - projects = projects.where :id => eval(ENV['PROJECT_ID']) if ENV['PROJECT_ID'] - total_count = projects.count - #FileUtils.mkdir_p "#{ENV['CLONE_PATH']}/archives" if total_count > 0 - begin_time = Time.now - pr_count = 0 - projects.each_with_index do |project, ind| - project_stats = "#{project.name_with_owner}: #{ind+1}/#{total_count}" - if project.repo.commits.count == 0 - say "Skipping empty project #{project_stats}" - else - say "Start working with #{project_stats}" - time = Time.now - path = "#{ENV['CLONE_PATH'].chomp('/')}/repos/#{project.name_with_owner}" - FileUtils.rm_rf path - project_path = project.path#"#{ENV['GIT_PROJECTS_DIR']}/#{project.name_with_owner}/.git" - archives_exists = false - Dir.chdir(project_path) do - %w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z).each do |ext| - archives_exists=true and break if `git log --all --format='%H' -- *.#{ext}`.present? - end - end - #if `cd #{project_path} && git log --all -- abf.yml`.present? # already have abf.yml in repo? - # message="Skipping project with abf.yml file #{project_stats}" - # say message - # abf_existing_log.puts message - # `rm -rf #{path}` - #elsif archives_exists.present? - if archives_exists.present? - #-- hack for refs/heads (else git branch return only master) - system "git clone --mirror #{project_path} #{path}/.git" - system "cd #{path} && git config --bool core.bare false && git checkout -f HEAD" - system "bin/calc_sha #{path} #{token}" # FIXME change filename - #-- - - ##### - # This is dangerous !!! - #system "rm -rf #{project_path} && git clone --bare #{path} #{project_path} && rm -rf #{path}" - system "rm -rf #{path}" - ##### - - say "Worked with #{project.name_with_owner}: #{(Time.now - time).truncate} sec." - pr_count +=1 - else - message="Skipping project with no archives #{project_stats}" - say message - abf_existing_log.puts message - `rm -rf #{path}` - end - say '-------------' - end - end - say '=======================' - say "Total count of projects are #{total_count}" - say "Finished with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}" - abf_existing_log.close - end -end