Merge pull request #628 from warpc/file-store-4_git_task

[file-store-4] file-store task

Gemfile

@@ -54,6 +54,7 @@ gem 'ruby-haml-js', '~> 0.0.3'
 gem 'rails-backbone', '~> 0.7.2'
 
 gem 'rack-throttle'
+gem 'rest-client', '~> 1.6.6'
 
 group :assets do
   gem 'sass-rails', '~> 3.2.5'

Gemfile.lock

@@ -265,6 +265,8 @@ GEM
       uuid (~> 2.3)
     resque_mailer (2.1.0)
       actionmailer (~> 3.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     rr (1.0.4)
     rspec (2.11.0)
       rspec-core (~> 2.11.0)
@@ -401,6 +403,7 @@ DEPENDENCIES
   resque (~> 1.21.0)
   resque-status (~> 0.3.3)
   resque_mailer (~> 2.1.0)
+  rest-client (~> 1.6.6)
   rr (~> 1.0.4)
   rspec-rails (~> 2.11.0)
   ruby-haml-js (~> 0.0.3)

bin/calc_sha

@@ -0,0 +1,5 @@
+#!/bin/bash
+pwd=`pwd`
+cd $1
+git filter-branch --tree-filter "$pwd/bin/file-store.rb $2" --prune-empty --tag-name-filter cat -- --all
+cd ..

bin/file-store.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+require 'json'
+require 'rest-client'
+
+abf_yml, new_sources = 'abf.yml', []
+old_sources = if File.exist? abf_yml
+                File.read(abf_yml).split("\n").reject {|line| line =~ /sources/}
+              else
+                []
+              end
+MAX_SIZE = 2 * 1024 * 1024 # 2.megabytes
+url = 'http://file-store.rosalinux.ru/api/v1/file_stores.json'
+#url = 'http://localhost:3001/api/v1/file_stores.json'
+rclient = RestClient::Resource.new(url, :user => ARGF.argv[0]) # user auth token
+
+Dir.glob("*.{tar\.bz2,tar\.gz,bz2,rar,gz,tar,tbz2,tgz,zip,Z,7z}").uniq.sort.each do |file|
+  begin
+    puts "Work with file \"#{file}\""
+    next if File.size(file) < MAX_SIZE
+
+    sha1 = Digest::SHA1.file(file).hexdigest
+    resp = JSON(RestClient.get url, :params => {:hash => sha1})
+    if resp[0].respond_to?('[]') && resp[0]['file_name'] && resp[0]['sha1_hash']
+      # file already exists at file-store
+      new_sources << "  \"#{file}\": #{sha1}"
+      FileUtils.rm_rf file
+    elsif resp == []
+      # try to put file at file-store
+      resp = JSON(rclient.post :file_store => {:file => File.new(file, 'rb')})
+      unless resp['sha1_hash'].nil?
+        new_sources << "  \"#{file}\" #{sha1}"
+        FileUtils.rm_rf file
+      end
+    else
+      raise "Response unknown!\n #{resp}"
+    end
+
+  #rescue => e
+  #  e.response
+  end
+end
+
+File.open(abf_yml, 'w') do |abf|
+  abf.puts 'sources:'
+  (old_sources | new_sources).sort.each {|line| abf.puts line}
+end

db/seeds.rb

@@ -3,3 +3,6 @@ ARCHES = %w(i386 i586 x86_64)
 ARCHES.each do |arch|
   Arch.find_or_create_by_name arch
 end
+
+user = User.new uname: 'rosa_system', email: 'rosa_system@rosalinux.ru', password: SecureRandom.base64
+user.confirmed_at = Time.now.utc; user.save

lib/tasks/shrink_git_repo.rake

@@ -0,0 +1,69 @@
+namespace :project do
+  desc "Truncate blobs from git repo"
+  task :remove_archives => :environment do
+    #raise 'Need set GIT_PROJECTS_DIR' if ENV['GIT_PROJECTS_DIR'].blank?
+    raise 'Need set CLONE_PATH' if ENV['CLONE_PATH'].blank?
+    raise 'Need special "rosa_system" user' unless User.where(:uname => 'rosa_system').exists?
+    token = User.find_by_uname('rosa_system').authentication_token
+
+    abf_existing_log = File.open "#{ENV['CLONE_PATH']}/projects_with_abf_yml.log", 'w'
+    projects = if uname = ENV['OWNER']
+                 owner = User.find_by_uname(uname) || Group.find_by_uname(uname)
+                 owner.projects
+               else
+                 Project.scoped
+               end
+
+    projects = projects.where :id => eval(ENV['PROJECT_ID']) if ENV['PROJECT_ID']
+    total_count = projects.count
+    #FileUtils.mkdir_p "#{ENV['CLONE_PATH']}/archives" if total_count > 0
+    begin_time = Time.now
+    pr_count = 0
+    projects.each_with_index do |project, ind|
+      project_stats = "#{project.name_with_owner}: #{ind+1}/#{total_count}"
+      if project.repo.commits.count == 0
+        say "Skipping empty project #{project_stats}"
+      else
+        say "Start working with #{project_stats}"
+        time = Time.now
+        path = "#{ENV['CLONE_PATH'].chomp('/')}/repos/#{project.name_with_owner}"
+        FileUtils.rm_rf path
+        project_path = project.path#"#{ENV['GIT_PROJECTS_DIR']}/#{project.name_with_owner}/.git"
+        system "git clone --mirror #{project_path} #{path}/.git"
+        archives_exists = false
+        Dir.chdir(path) do
+          system 'git config --bool core.bare false && git checkout -f HEAD' # hack for refs/heads
+          %w(tar.bz2 tar.gz bz2 rar gz tar tbz2 tgz zip Z 7z).each do |ext|
+            archives_exists=true and break if `git log --all --format='%H' -- *.#{ext}`.present?
+          end
+        end
+        if `cd #{path} && git log --all -- abf.yml`.present? # already have abf.yml in repo?
+          message="Skipping project with abf.yml file #{project_stats}"
+          say message
+          abf_existing_log.puts message
+          `rm -rf #{path}`
+        elsif archives_exists.present?
+          system "bin/calc_sha #{path} #{token}" # FIXME change filename
+
+          #####
+          # This is dangerous !!!
+          system "rm -rf #{project_path} && git clone --bare #{path} #{project_path} && rm -rf #{path}"
+          #####
+
+          say "Worked with #{project.name_with_owner}: #{(Time.now - time).truncate} sec."
+          pr_count +=1
+        else
+          message="Skipping project with no archives #{project_stats}"
+          say message
+          abf_existing_log.puts message
+          `rm -rf #{path}`
+        end
+        say '-------------'
+      end
+    end
+    say '======================='
+    say "Total count of projects are #{total_count}"
+    say "Finished with #{pr_count} project(s) in #{Time.at((Time.now - begin_time).truncate).gmtime.strftime('%R:%S')}"
+    abf_existing_log.close
+  end
+end