[file-store-4] remove archives in the srpm import

bin/calc_sha

@@ -1,5 +0,0 @@
-#!/bin/bash
-pwd=`pwd`
-cd $1
-git filter-branch --tree-filter "$pwd/bin/file-store.rb $2" --prune-empty --tag-name-filter cat -- --all
-cd ..

bin/file-store.rb

@@ -2,7 +2,7 @@
 require 'json'
 require 'rest-client'
 
-abf_yml, new_sources = 'abf.yml', []
+abf_yml, new_sources = '.abf.yml', []
 old_sources = if File.exist? abf_yml
                 File.read(abf_yml).split("\n").reject {|line| line =~ /sources/}
               else
@@ -15,7 +15,7 @@ rclient = RestClient::Resource.new(url, :user => ARGF.argv[0]) # user auth token
 
 Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each do |file|
   begin
-    puts " work with file \"#{file}\""
+    #puts " work with file \"#{file}\""
     next if File.size(file) < MAX_SIZE
 
     sha1 = Digest::SHA1.file(file).hexdigest
@@ -24,12 +24,14 @@ Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each
       # file already exists at file-store
       new_sources << "  \"#{file}\": #{sha1}"
       FileUtils.rm_rf file
+      puts " file \"#{file}\" already exists in the file-store"
     elsif resp == []
       # try to put file at file-store
       resp = JSON(rclient.post :file_store => {:file => File.new(file, 'rb')})
       unless resp['sha1_hash'].nil?
-        new_sources << "  \"#{file}\" #{sha1}"
+        new_sources << "  \"#{file}\": #{sha1}"
         FileUtils.rm_rf file
+         puts " upload file \"#{file}\" to the file-store"
       end
     else
       raise "Response unknown!\n #{resp}"
@@ -39,8 +41,12 @@ Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each
   #  e.response
   end
 end
-
+sources = (old_sources | new_sources)
-File.open(abf_yml, 'w') do |abf|
+unless sources.empty?
-  abf.puts 'sources:'
+  Dir.chdir(ARGF.argv[1]) do # Need if in filter-branch
-  (old_sources | new_sources).sort.each {|line| abf.puts line}
+    File.open(abf_yml, 'w') do |abf|
+      abf.puts 'sources:'
+      (old_sources | new_sources).sort.each {|line| abf.puts line}
+    end
+  end
 end

bin/import_srpm.sh

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 # import_srpm.sh: Import SRPM packages to git repo
-
+pwd=$PWD
 # Input data
 srpm_path=$1
 git_path=$2
 git_branch=$3
+token=$4
 name=$(rpm -q --qf '[%{Name}]' -p $srpm_path)
 version=$(rpm -q --qf '[%{Version}-%{Release}]' -p $srpm_path)
 tmp_dir=/tmp/$name-$version-$RANDOM
@@ -30,6 +31,9 @@ rpm2cpio $srpm_path > srpm.cpio
 cpio -idv < srpm.cpio
 rm -f srpm.cpio
 
+# Remove archives
+$pwd/file-store.rb $token $PWD
+
 # Commit and push changes
 git add -A .
 git commit -m "Automatic import for version $version"

bin/remove_archives.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+# git_dir_projects[0] clone_path[1] owner[2] project_name[3]
+
+require 'fileutils'
+require 'digest'
+
+token = '[CENSORED]'
+
+owners = ARGF.argv[2] || '[a-z0-9_]*'
+project_names = ARGF.argv[3] || '[a-zA-Z0-9_\-\+\.]*'
+
+begin_time = Time.now
+pr_count = total_count = 0
+
+Dir.chdir ARGF.argv[0]
+Dir.glob(owners).each do |owner|
+  Dir.chdir "#{ARGF.argv[0]}/#{owner}"
+  Dir.glob(project_names).each do |project|
+    name_with_owner = "#{owner}/#{project.chomp('.git')}"
+    project_path = "#{ARGF.argv[0]}/#{name_with_owner}.git"
+    time, total_count = Time.now, total_count + 1
+    Dir.chdir project_path
+    project_stats = "#{name_with_owner}: #{total_count}"
+    if system('git log -n 1 --oneline > /dev/null 2>&1') == false
+      p "Skipping empty project #{project_stats}"
+    else
+      p "Start working with #{project_stats}"
+      path = "#{ARGF.argv[1].chomp('/')}/repos/#{name_with_owner}"
+      FileUtils.rm_rf path
+      #-- hack for refs/heads (else git branch return only master)
+      system "git clone --mirror #{project_path} #{path}/.git"
+      system "cd #{path} && git config --bool core.bare false && git checkout -f HEAD"
+      #--
+      Dir.chdir(path)
+      archives_exists = false
+      %w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z).each do |ext|
+        archives_exists=true and break unless `git log --all --format='%H' -- *.#{ext}`.empty?
+      end
+
+      if archives_exists
+        system "git filter-branch -d /dev/shm/git_task --tree-filter \"/home/rosa/git_task/file-store.rb #{token} #{path}\" --prune-empty --tag-name-filter cat -- --all"
+        #####
+        # This is dangerous !!!
+        system "rm -rf #{project_path} && git clone --bare #{path} #{project_path}"
+        #####
+
+        p "Worked with #{name_with_owner}: #{(Time.now - time).truncate} sec."
+        pr_count +=1
+      else
+        p "Skipping project with no archives #{project_stats}"
+      end
+      `rm -rf #{path}`
+    end
+    p '-------------'
+  end
+end
+p '======================='
+p "Total count of projects are #{total_count}"
+p "Finished work with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}"

lib/modules/models/git.rb

@@ -92,7 +92,8 @@ module Modules
       end
 
       def import_srpm(srpm_path = srpm.path, branch_name = 'import')
-        system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} >> /dev/null 2>&1")
+        token = User.find_by_uname('rosa_system').token_authenticatable
+        system("#{Rails.root.join('bin', 'import_srpm.sh')} #{srpm_path} #{path} #{branch_name} #{token} >> /dev/null 2>&1")
       end
 
       protected

lib/tasks/remove_archives_from_git.rake

@@ -1,73 +0,0 @@
-namespace :project do
-  desc "Truncate blobs from git repo"
-  task :remove_archives => :environment do
-    #raise 'Need set GIT_PROJECTS_DIR' if ENV['GIT_PROJECTS_DIR'].blank?
-    raise 'Need set CLONE_PATH' if ENV['CLONE_PATH'].blank?
-    raise 'Need special "rosa_system" user' unless User.where(:uname => 'rosa_system').exists?
-    token = User.find_by_uname('rosa_system').authentication_token
-
-    abf_existing_log = File.open "#{ENV['CLONE_PATH']}/projects_with_abf_yml.log", 'w'
-    projects = if uname = ENV['OWNER']
-                 owner = User.find_by_uname(uname) || Group.find_by_uname(uname)
-                 owner.projects
-               else
-                 Project.scoped
-               end
-
-    projects = projects.where :id => eval(ENV['PROJECT_ID']) if ENV['PROJECT_ID']
-    total_count = projects.count
-    #FileUtils.mkdir_p "#{ENV['CLONE_PATH']}/archives" if total_count > 0
-    begin_time = Time.now
-    pr_count = 0
-    projects.each_with_index do |project, ind|
-      project_stats = "#{project.name_with_owner}: #{ind+1}/#{total_count}"
-      if project.repo.commits.count == 0
-        say "Skipping empty project #{project_stats}"
-      else
-        say "Start working with #{project_stats}"
-        time = Time.now
-        path = "#{ENV['CLONE_PATH'].chomp('/')}/repos/#{project.name_with_owner}"
-        FileUtils.rm_rf path
-        project_path = project.path#"#{ENV['GIT_PROJECTS_DIR']}/#{project.name_with_owner}/.git"
-        archives_exists = false
-        Dir.chdir(project_path) do
-          %w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z).each do |ext|
-            archives_exists=true and break if `git log --all --format='%H' -- *.#{ext}`.present?
-          end
-        end
-        #if `cd #{project_path} && git log --all -- abf.yml`.present? # already have abf.yml in repo?
-        #  message="Skipping project with abf.yml file #{project_stats}"
-        #  say message
-        #  abf_existing_log.puts message
-        #  `rm -rf #{path}`
-        #elsif archives_exists.present?
-        if archives_exists.present?
-          #-- hack for refs/heads (else git branch return only master)
-          system "git clone --mirror #{project_path} #{path}/.git"
-          system "cd #{path} && git config --bool core.bare false && git checkout -f HEAD"
-          system "bin/calc_sha #{path} #{token}" # FIXME change filename
-          #--
-
-          #####
-          # This is dangerous !!!
-          #system "rm -rf #{project_path} && git clone --bare #{path} #{project_path} && rm -rf #{path}"
-          system "rm -rf #{path}"
-          #####
-
-          say "Worked with #{project.name_with_owner}: #{(Time.now - time).truncate} sec."
-          pr_count +=1
-        else
-          message="Skipping project with no archives #{project_stats}"
-          say message
-          abf_existing_log.puts message
-          `rm -rf #{path}`
-        end
-        say '-------------'
-      end
-    end
-    say '======================='
-    say "Total count of projects are #{total_count}"
-    say "Finished with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}"
-    abf_existing_log.close
-  end
-end