Merge pull request #383 from abf/rosa-build:build-scripts

Use FileStore for build scripts

app/admin/build_scripts.rb

@@ -0,0 +1,71 @@
+ActiveAdmin.register BuildScript do
+
+  menu priority: 4
+
+  filter :project_name, as: :string
+
+  controller do
+    def scoped_collection
+      BuildScript.includes(:project)
+    end
+  end
+
+  index do
+    column(:project) do |bs|
+      link_to(bs.project.name_with_owner, project_path(bs.project))
+    end
+    column :treeish
+    column :commit
+    column :sha1
+
+    column(:status, sortable: :status) do |bs|
+      status_tag(bs.status, build_script_status_color(bs))
+    end
+    column :updated_at
+
+    default_actions
+  end
+
+  show do
+    attributes_table do
+      row :id
+      row(:project) do |bs|
+        link_to(bs.project.name_with_owner, project_path(bs.project))
+      end
+      row :treeish
+      row :commit
+      row :sha1
+      row(:status, sortable: :status) do |bs|
+        status_tag(bs.status, build_script_status_color(bs))
+      end
+      row :created_at
+      row :updated_at
+    end
+  end
+
+  form do |f|
+    f.inputs do
+      f.input :project_name
+      f.input :treeish
+      f.input :commit
+      f.input :sha1
+      f.input :status,  as: :select, include_blank: false, collection: BuildScript::STATUSES
+    end
+    f.actions
+  end
+
+  sidebar 'Actions', only: :show do
+    %w(enable disable update_archive).each do |state|
+      div do
+        link_to state.humanize, force_admin_build_script_path(resource, state: state), method: :patch
+      end if resource.send("can_#{state}?")
+    end
+  end
+
+  member_action :force, method: :patch do
+    resource.send(params[:state])
+    flash[:notice] = 'Updated successfully'
+    redirect_to admin_build_script_path(resource)
+  end
+
+end

app/admin/shared/build_script.rb

@@ -0,0 +1,3 @@
+def build_script_status_color(bs)
+  bs.active? ? :green : :red
+end

app/controllers/projects/git/trees_controller.rb

@@ -17,14 +17,19 @@ class Projects::Git::TreesController < Projects::Git::BaseController
 
   def archive
     format, @treeish = params[:format], params[:treeish]
-    if (@treeish =~ /^#{@project.name}-/) && !(@treeish =~ /[\s]+/) && (format =~ /^(zip|tar\.gz)$/)
-      @treeish = @treeish.gsub(/^#{@project.name}-/, '')
+    raise Grit::NoSuchPathError unless  @treeish =~ /^#{@project.name}-/ && 
+                                        @treeish !~ /[\s]+/ &&
+                                        format =~ /^(zip|tar\.gz)$/
+    @treeish.gsub!(/^#{@project.name}-/, '')
+    sha1 = @project.build_scripts.by_active.by_treeish(@treeish).first.try(:sha1)
+    unless sha1
       @commit = @project.repo.commits(@treeish, 1).first
+      raise Grit::NoSuchPathError unless @commit
+      tag     = @project.repo.tags.find{ |t| t.name == @treeish }
+      sha1    = @project.get_project_tag_sha1(tag, format) if tag
     end
-    raise Grit::NoSuchPathError unless @commit
-    tag   = @project.repo.tags.find{ |t| t.name == @treeish }
-    sha1  = @project.get_project_tag_sha1(tag, format) if tag
-    if sha1
+
+    if sha1.present?
       redirect_to "#{APP_CONFIG['file_store_url']}/api/v1/file_stores/#{sha1}"
     else
       archive = @project.archive_by_treeish_and_format @treeish, format

app/models/build_script.rb

@@ -0,0 +1,63 @@
+class BuildScript < ActiveRecord::Base
+  include FileStoreClean
+
+  STATUSES = [
+    ACTIVE  = 'active',
+    BLOCKED = 'blocked'
+  ]
+  FORMAT = 'tar.gz'
+
+  belongs_to :project
+
+  validates :treeish,     presence: true
+  validates :project_id,  presence: true, uniqueness: { scope: :treeish }
+
+  scope :by_active,   -> { where(status: ACTIVE) }
+  scope :by_treeish,  -> treeish { where(treeish: treeish) }
+
+  before_validation :attach_project
+  attr_writer       :project_name
+  attr_accessible   :project_name, :treeish, :commit, :sha1, :status
+
+  state_machine :status, initial: :active do
+    event(:disable) { transition active: :blocked }
+    event(:enable)  { transition blocked: :active }
+  end
+
+  def sha1_of_file_store_files
+    [sha1].select(&:present?)
+  end
+
+  def project_name
+    @project_name.presence || project.try(:name_with_owner)
+  end
+
+  def can_update_archive?
+    last_commit != commit
+  end
+
+  def update_archive
+    old_sha1, new_commit = sha1, last_commit
+
+    archive   = project.archive_by_treeish_and_format(treeish, FORMAT)
+    new_sha1  = FileStoreClean.save_file_to_file_store(archive)
+    if new_sha1.present?
+      update_attributes(sha1: new_sha1, commit: new_commit)
+      destroy_files_from_file_store(old_sha1) if old_sha1.present?
+    end
+  end
+  later :update_archive, queue: :middle
+
+  protected
+
+  def last_commit
+    project.repo.commits(treeish, 1).first.try(:id)
+  end
+
+  def attach_project
+    if @project_name.present?
+      self.project = Project.find_by_owner_and_name(@project_name)
+    end
+  end
+
+end

app/models/concerns/file_store_clean.rb

@@ -14,7 +14,7 @@ module FileStoreClean
 
     def destroy_files_from_file_store(args = sha1_of_file_store_files)
       files = *args
-      token = User.find_by(uname: 'file_store').authentication_token
+      token = FileStoreClean.file_store_authentication_token
       uri   = URI APP_CONFIG['file_store_url']
       Net::HTTP.start(uri.host, uri.port) do |http|
         files.each do |sha1|
@@ -34,6 +34,41 @@ module FileStoreClean
     later :later_destroy_files_from_file_store, queue: :middle
   end
 
+  def self.file_store_authentication_token
+    User.find_by(uname: 'file_store').authentication_token
+  end
+
+  # @param [Hash] data:
+  # - [String] path     - path to file
+  # - [String] fullname - file name
+  def self.save_file_to_file_store(data)
+    sha1_hash = Digest::SHA1.hexdigest(File.read(data[:path]))
+    return sha1_hash if file_exist_on_file_store?(sha1_hash)
+
+    begin
+      resource  = RestClient::Resource.new(
+        "#{APP_CONFIG['file_store_url']}/api/v1/upload",
+        user: file_store_authentication_token
+      )
+
+      file = File.new(data[:path])
+      # Hook for RestClient
+      # See: [RestClient::Payload#create_file_field](https://github.com/rest-client/rest-client/blob/master/lib/restclient/payload.rb#L202-L215)
+      file.define_singleton_method(:original_filename) { data[:fullname] }
+      resp = resource.post(file_store: { file: file })
+      resp = JSON(resp)
+    rescue RestClient::UnprocessableEntity => e # 422, file already exist
+      return sha1_hash
+    rescue # Dont care about it
+      return nil
+    end
+    if resp.respond_to?(:[]) && resp['sha1_hash'].present?
+      resp['sha1_hash']
+    else
+      nil
+    end
+  end
+
   def self.file_exist_on_file_store?(sha1)
     begin
       resp = JSON(RestClient.get "#{APP_CONFIG['file_store_url']}/api/v1/file_stores.json", params: {hash: sha1})

app/models/project.rb

@@ -16,9 +16,10 @@ class Project < ActiveRecord::Base
   belongs_to :owner, polymorphic: true, counter_cache: :own_projects_count
   belongs_to :maintainer, class_name: 'User'
 
-  has_many :issues, dependent: :destroy
-  has_many :pull_requests, dependent: :destroy, foreign_key: 'to_project_id'
-  has_many :labels, dependent: :destroy
+  has_many :issues,         dependent: :destroy
+  has_many :pull_requests,  dependent: :destroy, foreign_key: 'to_project_id'
+  has_many :labels,         dependent: :destroy
+  has_many :build_scripts,  dependent: :destroy
 
   has_many :project_imports, dependent: :destroy
   has_many :project_to_repositories, dependent: :destroy
@@ -243,16 +244,9 @@ class Project < ActiveRecord::Base
     return project_tag.sha1 if project_tag && project_tag.commit_id == tag.commit.id && FileStoreClean.file_exist_on_file_store?(project_tag.sha1)
 
     archive = archive_by_treeish_and_format tag.name, format
-    sha1    = Digest::SHA1.file(archive[:path]).hexdigest
-    unless FileStoreClean.file_exist_on_file_store? sha1
-      token = User.find_by(uname: 'rosa_system').authentication_token
-      begin
-        resp = JSON `curl --user #{token}: -POST -F 'file_store[file]=@#{archive[:path]};filename=#{name}-#{tag.name}.#{tag_file_format(format)}' #{APP_CONFIG['file_store_url']}/api/v1/upload`
-      rescue # Dont care about it
-        resp = {}
-      end
-      return nil if resp['sha1_hash'].nil?
-    end
+    sha1    = FileStoreClean.save_file_to_file_store(archive)
+    return nil if sha1.blank?
+
     if project_tag
       project_tag.destroy_files_from_file_store(project_tag.sha1)
       project_tag.update_attributes(sha1: sha1)

db/migrate/20140502160112_create_build_scripts.rb

@@ -0,0 +1,15 @@
+class CreateBuildScripts < ActiveRecord::Migration
+  def change
+    create_table :build_scripts do |t|
+      t.integer :project_id,  null: false
+      t.string  :treeish,     null: false
+      t.string  :commit
+      t.string  :sha1
+      t.string  :status
+
+      t.timestamps
+    end
+
+    add_index :build_scripts, [:project_id, :treeish], unique: true
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20140414195426) do
+ActiveRecord::Schema.define(version: 20140502160112) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -148,6 +148,44 @@ ActiveRecord::Schema.define(version: 20140414195426) do
     t.index ["project_id"], :name => "index_build_lists_on_project_id"
   end
 
+  create_table "projects", force: true do |t|
+    t.string   "name"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.integer  "owner_id"
+    t.string   "owner_type"
+    t.string   "visibility",               default: "open"
+    t.text     "description"
+    t.string   "ancestry"
+    t.boolean  "has_issues",               default: true
+    t.string   "srpm_file_name"
+    t.integer  "srpm_file_size"
+    t.datetime "srpm_updated_at"
+    t.string   "srpm_content_type"
+    t.boolean  "has_wiki",                 default: false
+    t.string   "default_branch",           default: "master"
+    t.boolean  "is_package",               default: true,     null: false
+    t.integer  "maintainer_id"
+    t.boolean  "publish_i686_into_x86_64", default: false
+    t.string   "owner_uname",                                 null: false
+    t.boolean  "architecture_dependent",   default: false,    null: false
+    t.integer  "autostart_status"
+    t.index ["name", "owner_id", "owner_type"], :name => "index_projects_on_name_and_owner_id_and_owner_type", :unique => true, :case_sensitive => false
+  end
+
+  create_table "build_scripts", force: true do |t|
+    t.integer  "project_id", null: false
+    t.string   "treeish",    null: false
+    t.string   "commit"
+    t.string   "sha1"
+    t.string   "status"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.index ["project_id", "treeish"], :name => "index_build_scripts_on_project_id_and_treeish", :unique => true
+    t.index ["project_id"], :name => "fk__build_scripts_project_id"
+    t.foreign_key ["project_id"], "projects", ["id"], :on_update => :no_action, :on_delete => :no_action, :name => "fk_build_scripts_project_id"
+  end
+
   create_table "comments", force: true do |t|
     t.string   "commentable_type"
     t.integer  "user_id"
@@ -440,31 +478,6 @@ ActiveRecord::Schema.define(version: 20140414195426) do
     t.index ["repository_id", "project_id"], :name => "index_project_to_repositories_on_repository_id_and_project_id", :unique => true
   end
 
-  create_table "projects", force: true do |t|
-    t.string   "name"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.integer  "owner_id"
-    t.string   "owner_type"
-    t.string   "visibility",               default: "open"
-    t.text     "description"
-    t.string   "ancestry"
-    t.boolean  "has_issues",               default: true
-    t.string   "srpm_file_name"
-    t.integer  "srpm_file_size"
-    t.datetime "srpm_updated_at"
-    t.string   "srpm_content_type"
-    t.boolean  "has_wiki",                 default: false
-    t.string   "default_branch",           default: "master"
-    t.boolean  "is_package",               default: true,     null: false
-    t.integer  "maintainer_id"
-    t.boolean  "publish_i686_into_x86_64", default: false
-    t.string   "owner_uname",                                 null: false
-    t.boolean  "architecture_dependent",   default: false,    null: false
-    t.integer  "autostart_status"
-    t.index ["name", "owner_id", "owner_type"], :name => "index_projects_on_name_and_owner_id_and_owner_type", :unique => true, :case_sensitive => false
-  end
-
   create_table "pull_requests", force: true do |t|
     t.integer "issue_id",                 null: false
     t.integer "to_project_id",            null: false

spec/controllers/admin/build_scripts_controller_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe Admin::BuildScriptsController do
+  it_should_behave_like 'an admin controller'
+end

spec/factories/build_scripts.rb

@@ -0,0 +1,6 @@
+FactoryGirl.define do
+  factory :build_script do
+    association :project, factory: :project
+    treeish { FactoryGirl.generate(:string) }
+  end
+end

spec/models/build_script_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe BuildScript do
+  before { stub_symlink_methods }
+
+  let(:build_script) { FactoryGirl.build(:build_script) }
+
+  it 'is valid given valid attributes' do
+    build_script.should be_valid
+  end
+
+  context 'ensures that validations and associations exist' do
+    it { should belong_to(:project) }
+
+    it { should validate_presence_of(:project_id) }
+    it { should validate_presence_of(:treeish) }
+
+    context 'uniqueness' do
+      before { build_script.save }
+      it { should validate_uniqueness_of(:project_id).scoped_to(:treeish) }
+    end
+  end
+
+end