#370: generate sitemap

Gemfile

@@ -83,6 +83,7 @@ gem 'compass-rails', '~> 1.1.6'
 gem 'uglifier', '~> 2.4'
 gem 'therubyracer', '~> 0.12.1', platforms: [:mri, :rbx]
 gem 'therubyrhino', '~> 1.73.1', platforms: :jruby
+gem 'sitemap_generator'
 
 
 group :production do

Gemfile.lock

@@ -395,6 +395,8 @@ GEM
       rack (~> 1.4)
       rack-protection (~> 1.4)
       tilt (~> 1.3, >= 1.3.4)
+    sitemap_generator (5.0.1)
+      builder
     skinny (0.2.3)
       eventmachine (~> 1.0.0)
       thin (~> 1.5.0)
@@ -536,6 +538,7 @@ DEPENDENCIES
   shotgun
   shoulda
   shoulda-matchers
+  sitemap_generator
   skype
   soundmanager-rails
   state_machine (~> 1.2)

app/controllers/sitemap_controller.rb

@@ -0,0 +1,11 @@
+class SitemapController < ApplicationController
+
+  def show
+    redirect_to "/sitemaps/#{request.host_with_port.gsub(/www./, '')}/sitemap.xml.gz"
+  end
+
+  def robots
+    render file: 'sitemap/robots', layout: false, content_type: Mime::TEXT
+  end
+
+end

app/views/sitemap/robots.erb

@@ -0,0 +1,22 @@
+# robots.txt for <%= root_url %>
+
+User-agent: *
+Disallow: /*/*/tree/*
+Disallow: /*/*/blob/*
+Disallow: /*/*/wiki/*/*
+Disallow: /*/*/issues?*
+Disallow: /*/*/commits/*
+Disallow: /*/blame/
+Disallow: /*/raw/
+Disallow: /*.git
+Disallow: /*.git/
+Disallow: /search
+Disallow: /users/sign_up
+Disallow: /users/sign_in
+Disallow: /users/password/new
+Disallow: /users/register_requests/new
+Disallow: /invite.html
+Disallow: /thanks.html
+Allow: /
+Host: <%= root_url.gsub(/\/$/, '') %>
+Sitemap: <%= sitemap_url %>

config/routes.rb

@@ -1,4 +1,9 @@
 Rosa::Application.routes.draw do
+
+  # Redirect sitemap1.xml.gz file on AWS S3
+  match '/sitemap.xml.gz' => 'sitemap#show', via: [:get, :post, :head], as: :sitemap
+  match '/robots.txt' => 'sitemap#robots', via: [:get, :post, :head], as: :robots
+
   resource :contact, only: [:new, :create, :sended] do
     get '/' => 'contacts#new'
     get :sended

config/schedule.rb

@@ -26,6 +26,10 @@ every :day, at: '3:00 am' do
   rake 'activity_feeds:clear', output: 'log/activity_feeds.log'
 end
 
+every :day, at: '2:30 am' do
+  rake 'sitemap:refresh', output: 'log/sitemap.log'
+end
+
 every 1.hour do
   rake 'buildlist:clear:outdated_canceling', output: 'log/canceling_build_list_clear.log'
 end

config/sitemap.rb

@@ -0,0 +1,58 @@
+%w(abf.rosalinux.ru abf.io).each do |domain|
+
+  SitemapGenerator::Sitemap.create(
+    default_host:   "https://#{domain}",
+    sitemaps_path:  "sitemaps/#{domain}"
+  ) do
+
+    # root
+    add(root_path)
+
+    # Projects
+    Project.opened.find_each do |project|
+      add(project_path(project), lastmod: project.updated_at)
+    end
+
+    # BuildLists
+    BuildList.for_status(BuildList::BUILD_PUBLISHED).
+      where(platforms: { visibility: 'open' }, projects: { visibility: 'open' }).
+      joins(:save_to_platform, :project).
+      find_each do |bl|
+        add(build_list_path(bl), lastmod: bl.updated_at)
+    end
+
+    # Platforms
+    Platform.opened.find_each do |platform|
+      add(platform_path(platform), lastmod: platform.updated_at)
+
+      # Repositories
+      add(platform_repositories_path(platform), lastmod: platform.updated_at)
+      platform.repositories.find_each do |repository|
+        add(platform_repository_path(platform, repository), lastmod: repository.updated_at)
+      end
+
+
+      # Products
+      add(platform_products_path(platform), lastmod: platform.updated_at)
+      platform.products.find_each do |product|
+        add(platform_product_path(platform, product), lastmod: product.updated_at)
+
+        # ProductBuildList (ISO)
+        product.product_build_lists.for_status(ProductBuildList::BUILD_COMPLETED).find_each do |pbl|
+          add(platform_product_product_build_list_path(platform, product, pbl), lastmod: pbl.updated_at)
+        end
+      end
+    end
+
+    # Users
+    User.find_each do |user|
+      add(user_path(user), lastmod: user.updated_at)
+    end
+
+    # Groups
+    Group.find_each do |group|
+      add(group_path(group), lastmod: group.updated_at)
+    end
+
+  end
+end

public/robots.txt

@@ -1,202 +0,0 @@
-User-agent: Googlebot
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: baiduspider
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: msnbot
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: naverbot
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: seznambot
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: Slurp
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: teoma
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: Yandex
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: CCBot
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: Telefonica
-Allow: /*/*/tree/master
-Allow: /*/*/blob/master
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-
-User-agent: *
-Disallow: /