From d5180f73d74ffcc59963b79e72b2496ecc3ecbcc Mon Sep 17 00:00:00 2001 From: Wedge Date: Tue, 22 Mar 2016 20:02:49 +0300 Subject: [PATCH] Code clean up, removed access to api via standart login --- app/controllers/api/v1/base_controller.rb | 10 ++ app/controllers/application_controller.rb | 30 ---- app/controllers/projects/base_controller.rb | 12 +- app/jobs/remove_outdated_items_job.rb | 22 +++ config/resque_schedule.yml | 8 +- config/schedule.rb | 16 -- lib/tasks/import.rake | 166 -------------------- 7 files changed, 41 insertions(+), 223 deletions(-) create mode 100644 app/jobs/remove_outdated_items_job.rb delete mode 100644 lib/tasks/import.rake diff --git a/app/controllers/api/v1/base_controller.rb b/app/controllers/api/v1/base_controller.rb index a40bedfe0..9246b85de 100644 --- a/app/controllers/api/v1/base_controller.rb +++ b/app/controllers/api/v1/base_controller.rb @@ -1,6 +1,8 @@ class Api::V1::BaseController < ApplicationController include PaginateHelper respond_to :json + skip_before_action :verify_authenticity_token + before_action :check_auth helper_method :member_path @@ -13,6 +15,14 @@ class Api::V1::BaseController < ApplicationController protected + def check_auth + authenticate_or_request_with_http_basic do |token,pw| + if user = User.find_by_authentication_token(token) + sign_in user, false + end + end + end + def set_csv_file_headers(file_name) headers['Content-Type'] = 'text/csv' headers['Content-disposition'] = "attachment; filename=\"#{file_name}.csv\"" diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 0daa9e78e..56cce0c5a 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -53,36 +53,6 @@ class ApplicationController < ActionController::Base end end - # For this example, we are simply using token authentication - # via parameters. However, anyone could use Rails's token - # authentication features to get the token from a header. - def authenticate_user! - if user = find_user_by_token - # Notice we are passing store false, so the user is not - # actually stored in the session and a token is needed - # for every request. If you want the token to work as a - # sign in token, you can simply remove store: false. - sign_in user, store: false - else - super - end - end - - def authenticate_user - if user = find_user_by_token - sign_in user, store: false - end - end - - def find_user_by_token - user_token = params[:authentication_token].presence - if user_token.blank? && request.authorization.present? - token, pass = *ActionController::HttpAuthentication::Basic::user_name_and_password(request) - user_token = token if pass.blank? - end - user = user_token && User.find_by_authentication_token(user_token.to_s) - end - def render_500(e) #check for exceptions Airbrake ignores by default and exclude them from manual Airbrake notification if Rails.env.production? && !AIRBRAKE_IGNORE.include?(e.class) diff --git a/app/controllers/projects/base_controller.rb b/app/controllers/projects/base_controller.rb index 6f7453521..321fdfd91 100644 --- a/app/controllers/projects/base_controller.rb +++ b/app/controllers/projects/base_controller.rb @@ -1,17 +1,9 @@ class Projects::BaseController < ApplicationController - prepend_before_action :authenticate_user_and_find_project + prepend_before_action :find_project protected - def find_collaborators - search = "%#{params[:search_user]}%" - @users = @project.collaborators.where("users.uname ILIKE ?", search) - @users |= @project.owner.members.where("users.uname ILIKE ?", search) if @project.owner.is_a?(Group) - @users = @users.sort_by(&:uname).first(10) - end - - def authenticate_user_and_find_project - authenticate_user + def find_project return if params[:name_with_owner].blank? authorize @project = Project.find_by_owner_and_name!(params[:name_with_owner]), :show? end diff --git a/app/jobs/remove_outdated_items_job.rb b/app/jobs/remove_outdated_items_job.rb new file mode 100644 index 000000000..f39eec58d --- /dev/null +++ b/app/jobs/remove_outdated_items_job.rb @@ -0,0 +1,22 @@ +class RemoveOutdatedItemsJob + @queue = :low + + def self.perform + log_file = Rails.root.join("log", "remove_outdated.log").to_s + counter_bl = 0 + BuildList.outdated.find_each(batch_size: 100) do |bl| + bl.destroy && (counter_bl += 1) if bl.id != bl.last_published.first.try(:id) + end + counter_mb = 0 + MassBuild.outdated.find_each do |mb| + mb.destroy && (counter_mb += 1) if mb.build_lists.count == 0 + end + counter_pbl = ProductBuildList.outdated.count + ProductBuildList.outdated.destroy_all + File.open(log_file, "w") do |f| + f.puts "Build Lists deleted: #{counter_bl}" + f.puts "Mass Builds deleted: #{counter_mb}" + f.puts "Product Build Lists deleted: #{counter_pbl}" + end + end +end \ No newline at end of file diff --git a/config/resque_schedule.yml b/config/resque_schedule.yml index 4f300494e..c1d60225c 100644 --- a/config/resque_schedule.yml +++ b/config/resque_schedule.yml @@ -38,4 +38,10 @@ clear_builder_on_stale_buildlists: - '2m' class: 'BuildLists::ClearBuilderOnStaleBuildListsJob' queue: low - description: 'Clear builder on build lists which are still pending' \ No newline at end of file + description: 'Clear builder on build lists which are still pending' + +remove_outdated_items: + cron: '0 3 * * *' + class: 'RemoveOutdatedItemsJob' + queue: low + description: 'Removes outdated mass builds, product buid lists and build lists' \ No newline at end of file diff --git a/config/schedule.rb b/config/schedule.rb index 6ac012db6..395541d7c 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -6,22 +6,6 @@ # runner 'Download.parse_and_remove_nginx_log' #end -every :day, at: '4:10 am' do - rake 'product_build_list:clear:outdated', output: 'log/product_build_list_clear.log' -end - -every :day, at: '4:00 am' do - rake 'import:sync:all', output: 'log/sync.log' -end - -every :day, at: '3:50 am' do - rake 'buildlist:clear:outdated', output: 'log/build_list_clear.log' -end - -every :day, at: '3:30 am' do - rake 'pull_requests:clear', output: 'log/pull_requests_clear.log' -end - every :day, at: '3:00 am' do rake 'activity_feeds:clear', output: 'log/activity_feeds.log' end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake deleted file mode 100644 index adb2e8623..000000000 --- a/lib/tasks/import.rake +++ /dev/null @@ -1,166 +0,0 @@ -require 'highline/import' -require 'open-uri' - -namespace :import do - desc "Load projects" - task projects: :environment do - source = ENV['SOURCE'] || 'http://dl.dropbox.com/u/984976/package_list.txt' - owner = Group.find_by uname: "npp_team" - platform = Platform.find_by name: "RosaNPP" # RosaNPP - repo = platform.repositories.first rescue nil - say "START import projects from '#{source}' for '#{owner.uname}'.#{repo ? " To repo '#{platform.name}/#{repo.name}'." : ''}" - ask 'Press enter to continue' - open(source).readlines.each do |name| - name.chomp!; name.strip! #; name.downcase! - # name = name.match(/^([a-z\d_\-\+\.]+?)-(\d[a-z\d\-\.]+)\.src\.rpm$/)[1] # parse - print "Import '#{name}'..." - p = Project.find_or_create_by_name_and_owner_type_and_owner_id(name, owner.class.to_s, owner.id) - print p.persisted? ? "Ok!" : "Fail!" - if repo - print " Add to repo '#{platform.name}/#{repo.name}'." - repo.projects << p rescue print ' Fail!' - end - puts - end - say 'DONE' - end - - # bundle exec rake import:srpm RAILS_ENV=production BASE=/share/platforms/naulinux5x_personal/tmp/SRPMS LIST=https://dl.dropbox.com/u/984976/nauschool5x.srpms.txt OWNER=naulinux PLATFORM=naulinux REPO=main CLEAR=true HIDDEN=true > log/srpm_naulinux.log & - desc 'Import SRPMs as projects' - task srpm: :environment do - base = ENV['BASE'] || '/share/alt_repos/rsync' - list = ENV['LIST'] #|| 'https://dl.dropbox.com/u/984976/alt_import.txt' - mask = ENV['MASK'] || '*.src.rpm' - hidden = ENV['HIDDEN'] == 'true' ? true : false - owner = User.find_by(uname: ENV['OWNER']) || Group.find_by!(uname: ENV['OWNER'] || 'altlinux') - platform = Platform.find_by!(name: ENV['PLATFORM'] || 'altlinux5') - repo = platform.repositories.find_by!(name: ENV['REPO'] || 'main') - clear = ENV['CLEAR'] == 'true' ? true : false - - say "START import projects from '#{base}' using '#{list || mask}' for '#{owner.uname}' to repo '#{platform.name}/#{repo.name}'." - repo.project_to_repositories.clear if clear - (list ? open(list).readlines.map{|n| File.join base, n.chomp.strip} : Dir[File.join base, mask]).each do |srpm_file| - print "Processing '#{srpm_file}'... " - if name = `rpm -q --qf '[%{Name}]' -p #{srpm_file}` and $?.success? and name.present? - if clear # simply add - project = Project.find_or_create_by_name_and_owner_type_and_owner_id(name, owner.class.to_s, owner.id) - repo.projects << project rescue nil - else # check if project already added - if project = repo.projects.find_by(name: name) || repo.projects.by_name(name).first # fallback to speedup - print "Found project '#{project.name_with_owner}' in '#{platform.name}/#{repo.name}'." - elsif scoped = Project.where(owner_id: owner.id, owner_type: owner.class) and - project = scoped.find_by(name: name) || scoped.by_name(name).first - begin - repo.projects << project rescue nil - rescue Exception => e - print "Add project '#{project.name_with_owner}' to '#{platform.name}/#{repo.name}' FAILED: #{e.message}." - else - print "Add project '#{project.name_with_owner}' to '#{platform.name}/#{repo.name}' OK." - end - else - description = `rpm -q --qf '[%{Description}]' -p #{srpm_file}`.scrub('') - project = Project.create!(name: name, description: description) {|p| p.owner = owner} - repo.projects << project rescue nil - print "Create project #{project.name_with_owner} in #{platform.name}/#{repo.name} OK." - end - end - project.update_attributes(visibility: 'hidden') if hidden - project.import_srpm(srpm_file, platform.name) - print " Code import complete!" - else - print 'RPM Error!' - end - puts - end - say 'DONE' - end - - namespace :sync do - desc "Sync all repos" - task all: :environment do - # system("bundle exec rake import:sync:run RELEASE=official/2011 PLATFORM=mandriva2011 REPOSITORY=main") - # system("bundle exec rake import:sync:run RELEASE=official/2011 PLATFORM=mandriva2011 REPOSITORY=contrib") - # system("bundle exec rake import:sync:run RELEASE=official/2011 PLATFORM=mandriva2011 REPOSITORY=non-free") - #system("bundle exec rake import:sync:run RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=main") - #system("bundle exec rake import:sync:run RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=contrib") - #system("bundle exec rake import:sync:run RELEASE=devel/cooker PLATFORM=cooker REPOSITORY=non-free") - system("bundle exec rake import:sync:run SOURCE=rsync://mirror.yandex.ru/fedora-epel/6/SRPMS/ DESTINATION=#{File.join(APP_CONFIG['root_path'], 'mirror.yandex.ru', 'fedora-epel', '6', 'SRPMS')} PLATFORM=server_personal REPOSITORY=main OWNER=server BRANCH=import") - system("bundle exec rake import:sync:run SOURCE=rsync://rh-mirror.redhat.com/redhat/linux/enterprise/6Server/en/os/SRPMS/ DESTINATION=#{File.join(APP_CONFIG['root_path'], 'rh-mirror.redhat.com', 'redhat', 'linux', 'enterprise', '6Server', 'en', 'os', 'SRPMS')} PLATFORM=server_personal REPOSITORY=main OWNER=server BRANCH=import") - end - - task run: [:rsync, :parse] - - desc "Rsync with mirror.yandex.ru" - task rsync: :environment do - release = ENV['RELEASE'] || 'official/2011' - repository = ENV['REPOSITORY'] || 'main' - source = ENV['SOURCE'] || "rsync://mirror.yandex.ru/mandriva/#{release}/SRPMS/#{repository}/" - destination = ENV['DESTINATION'] || File.join(APP_CONFIG['root_path'], 'mirror.yandex.ru', 'mandriva', release, 'SRPMS', repository) - say "START rsync projects (*.src.rpm) from '#{source}' to '#{destination}' (#{Time.now.utc})" - if system "rsync -rtv --delete --exclude='backports/*' --exclude='testing/*' #{source} #{destination}" # --include='*.src.rpm' - say 'Rsync ok!' - else - say 'Rsync failed!' - end - say "DONE (#{Time.now.utc})" - end - - desc "Parse repository for changes" - task parse: :environment do - release = ENV['RELEASE'] || 'official/2011' - platform = Platform.find_by(name: ENV['PLATFORM'] || "mandriva2011") - repository = platform.repositories.find_by(name: ENV['REPOSITORY'] || 'main') - source = ENV['DESTINATION'] || File.join(APP_CONFIG['root_path'], 'mirror.yandex.ru', 'mandriva', release, 'SRPMS', repository.name, '{release,updates}') - owner = Group.find_or_create_by(uname: ENV['OWNER'] || 'import') {|g| g.name = g.uname; g.owner = User.first} - branch = ENV['BRANCH'] || "import_#{platform.name}" - - say "START (#{Time.now.utc})" - Dir[File.join source, '*.src.rpm'].each do |srpm_file| - print "Processing '#{srpm_file}'... " - if name = `rpm -q --qf '[%{Name}]' -p #{srpm_file}` and $?.success? and name.present? and - version = `rpm -q --qf '[%{Version}-%{Release}]' -p #{srpm_file}` and $?.success? and version.present? - project_import = ProjectImport.find_by_name_and_platform_id(name, platform.id) || ProjectImport.by_name(name).where(platform_id: platform.id).first || ProjectImport.new(name: name, platform_id: platform.id) - if version != project_import.version.to_s and File.mtime(srpm_file) > project_import.file_mtime - unless project = project_import.project - if platform.personal? # search project through owner # used for testhat - project = Project.find_or_create_by_name_and_owner_type_and_owner_id(name, owner.class.to_s, owner.id) - print "Use project #{project.name_with_owner}. " - else # search project through repository - if project = repository.projects.find_by(name: name) || repository.projects.by_name(name).first # fallback to speedup - print "Found project #{project.name_with_owner} in #{platform.name}/#{repository.name}. " - elsif scoped = Project.where(owner_id: owner.id, owner_type: owner.class) and - project = scoped.find_by(name: name) || scoped.by_name(name).first - repository.projects << project - print "Add project #{project.name_with_owner} to #{platform.name}/#{repository.name}. " - else - description = `rpm -q --qf '[%{Description}]' -p #{srpm_file}`.scrub('') - project = Project.create!(name: name, description: description) {|p| p.owner = owner} - repository.projects << project - print "Create project #{project.name_with_owner} at #{platform.name}/#{repository.name}. " - end - end - end - project.import_srpm(srpm_file, branch) - print "New version (#{version}) for #{project.name_with_owner} successfully imported to branch #{branch}! " - - project_import.project = project - # project_import.platform = platform - project_import.version = version - project_import.file_mtime = File.mtime(srpm_file) - project_import.save! - - # TODO notify import.members - - print 'Ok!' - else - print 'Not updated!' - end - else - print 'RPM Error!' - end - puts - end - say "DONE (#{Time.now.utc})" - end - end -end