Software
/
rosa-build
mirror of https://github.com/OpenMandrivaSoftware/rosa-build.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[file-store-4] remove archives in the srpm import

This commit is contained in:
Alexander Machehin 2012-11-09 20:14:26 +06:00
parent 08b3099b90
commit 726aee49ff
6 changed files with 79 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bin/calc_sha
View File

@ -1,5 +0,0 @@
#!/bin/bash
pwd=`pwd`
cd $1
git filter-branch --tree-filter "$pwd/bin/file-store.rb $2" --prune-empty --tag-name-filter cat -- --all
cd ..

14
bin/file-store.rb
View File

@ -2,7 +2,7 @@
require 'json'
require 'rest-client'
abf_yml, new_sources = 'abf.yml', []
abf_yml, new_sources = '.abf.yml', []
old_sources = if File.exist? abf_yml
File.read(abf_yml).split("\n").reject {|line| line =~ /sources/}
else
@ -15,7 +15,7 @@ rclient = RestClient::Resource.new(url, :user => ARGF.argv[0]) # user auth token
Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each do |file|
begin
puts " work with file \"#{file}\""
#puts " work with file \"#{file}\""
next if File.size(file) < MAX_SIZE
sha1 = Digest::SHA1.file(file).hexdigest
@ -24,12 +24,14 @@ Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each
# file already exists at file-store
new_sources << " \"#{file}\": #{sha1}"
FileUtils.rm_rf file
puts " file \"#{file}\" already exists in the file-store"
elsif resp == []
# try to put file at file-store
resp = JSON(rclient.post :file_store => {:file => File.new(file, 'rb')})
unless resp['sha1_hash'].nil?
new_sources << " \"#{file}\" #{sha1}"
new_sources << " \"#{file}\": #{sha1}"
FileUtils.rm_rf file
puts " upload file \"#{file}\" to the file-store"
end
else
raise "Response unknown!\n #{resp}"
@ -39,8 +41,12 @@ Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each
# e.response
end
end
sources = (old_sources | new_sources)
unless sources.empty?
Dir.chdir(ARGF.argv[1]) do # Need if in filter-branch
File.open(abf_yml, 'w') do |abf|
abf.puts 'sources:'
(old_sources | new_sources).sort.each {|line| abf.puts line}
end
end
end

6
bin/import_srpm.sh
View File

@ -1,10 +1,11 @@
#!/usr/bin/env bash
# import_srpm.sh: Import SRPM packages to git repo
pwd=$PWD
# Input data
srpm_path=$1
git_path=$2
git_branch=$3
token=$4
name=$(rpm -q --qf '[%{Name}]' -p $srpm_path)
version=$(rpm -q --qf '[%{Version}-%{Release}]' -p $srpm_path)
tmp_dir=/tmp/$name-$version-$RANDOM
@ -30,6 +31,9 @@ rpm2cpio $srpm_path > srpm.cpio
cpio -idv < srpm.cpio
rm -f srpm.cpio
# Remove archives
$pwd/file-store.rb $token $PWD
# Commit and push changes
git add -A .
git commit -m "Automatic import for version $version"

59
bin/remove_archives.rb Executable file
View File

@ -0,0 +1,59 @@
#!/usr/bin/env ruby
# git_dir_projects[0] clone_path[1] owner[2] project_name[3]
require 'fileutils'
require 'digest'
token = '[CENSORED]'
owners = ARGF.argv[2] || '[a-z0-9_]*'
project_names = ARGF.argv[3] || '[a-zA-Z0-9_\-\+\.]*'
begin_time = Time.now
pr_count = total_count = 0
Dir.chdir ARGF.argv[0]
Dir.glob(owners).each do |owner|
Dir.chdir "#{ARGF.argv[0]}/#{owner}"
Dir.glob(project_names).each do |project|
name_with_owner = "#{owner}/#{project.chomp('.git')}"
project_path = "#{ARGF.argv[0]}/#{name_with_owner}.git"
time, total_count = Time.now, total_count + 1
Dir.chdir project_path
project_stats = "#{name_with_owner}: #{total_count}"
if system('git log -n 1 --oneline > /dev/null 2>&1') == false
p "Skipping empty project #{project_stats}"
else
p "Start working with #{project_stats}"
path = "#{ARGF.argv[1].chomp('/')}/repos/#{name_with_owner}"
FileUtils.rm_rf path
#-- hack for refs/heads (else git branch return only master)
system "git clone --mirror #{project_path} #{path}/.git"
system "cd #{path} && git config --bool core.bare false && git checkout -f HEAD"
#--
Dir.chdir(path)
archives_exists = false
%w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z).each do |ext|
archives_exists=true and break unless `git log --all --format='%H' -- *.#{ext}`.empty?
end
if archives_exists
system "git filter-branch -d /dev/shm/git_task --tree-filter \"/home/rosa/git_task/file-store.rb #{token} #{path}\" --prune-empty --tag-name-filter cat -- --all"
#####
# This is dangerous !!!
system "rm -rf #{project_path} && git clone --bare #{path} #{project_path}"
#####
p "Worked with #{name_with_owner}: #{(Time.now - time).truncate} sec."
pr_count +=1
else
p "Skipping project with no archives #{project_stats}"
end
`rm -rf #{path}`
end
p '-------------'
end
end
p '======================='
p "Total count of projects are #{total_count}"
p "Finished work with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}"

3
lib/modules/models/git.rb
View File

@ -92,7 +92,8 @@ module Modules
end
def import_srpm(srpm_path = srpm.path, branch_name = 'import')
system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} >> /dev/null 2>&1")
token = User.find_by_uname('rosa_system').token_authenticatable
system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} #{token} >> /dev/null 2>&1")
end
protected

73
lib/tasks/remove_archives_from_git.rake
View File

@ -1,73 +0,0 @@
namespace :project do
desc "Truncate blobs from git repo"
task :remove_archives => :environment do
#raise 'Need set GIT_PROJECTS_DIR' if ENV['GIT_PROJECTS_DIR'].blank?
raise 'Need set CLONE_PATH' if ENV['CLONE_PATH'].blank?
raise 'Need special "rosa_system" user' unless User.where(:uname => 'rosa_system').exists?
token = User.find_by_uname('rosa_system').authentication_token
abf_existing_log = File.open "#{ENV['CLONE_PATH']}/projects_with_abf_yml.log", 'w'
projects = if uname = ENV['OWNER']
owner = User.find_by_uname(uname) || Group.find_by_uname(uname)
owner.projects
else
Project.scoped
end
projects = projects.where :id => eval(ENV['PROJECT_ID']) if ENV['PROJECT_ID']
total_count = projects.count
#FileUtils.mkdir_p "#{ENV['CLONE_PATH']}/archives" if total_count > 0
begin_time = Time.now
pr_count = 0
projects.each_with_index do |project, ind|
project_stats = "#{project.name_with_owner}: #{ind+1}/#{total_count}"
if project.repo.commits.count == 0
say "Skipping empty project #{project_stats}"
else
say "Start working with #{project_stats}"
time = Time.now
path = "#{ENV['CLONE_PATH'].chomp('/')}/repos/#{project.name_with_owner}"
FileUtils.rm_rf path
project_path = project.path#"#{ENV['GIT_PROJECTS_DIR']}/#{project.name_with_owner}/.git"
archives_exists = false
Dir.chdir(project_path) do
%w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z).each do |ext|
archives_exists=true and break if `git log --all --format='%H' -- *.#{ext}`.present?
end
end
#if `cd #{project_path} && git log --all -- abf.yml`.present? # already have abf.yml in repo?
# message="Skipping project with abf.yml file #{project_stats}"
# say message
# abf_existing_log.puts message
# `rm -rf #{path}`
#elsif archives_exists.present?
if archives_exists.present?
#-- hack for refs/heads (else git branch return only master)
system "git clone --mirror #{project_path} #{path}/.git"
system "cd #{path} && git config --bool core.bare false && git checkout -f HEAD"
system "bin/calc_sha #{path} #{token}" # FIXME change filename
#--
#####
# This is dangerous !!!
#system "rm -rf #{project_path} && git clone --bare #{path} #{project_path} && rm -rf #{path}"
system "rm -rf #{path}"
#####
say "Worked with #{project.name_with_owner}: #{(Time.now - time).truncate} sec."
pr_count +=1
else
message="Skipping project with no archives #{project_stats}"
say message
abf_existing_log.puts message
`rm -rf #{path}`
end
say '-------------'
end
end
say '======================='
say "Total count of projects are #{total_count}"
say "Finished with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}"
abf_existing_log.close
end
end