From ad77e882af8a29ddfd262b77a360a3474c4b42ca Mon Sep 17 00:00:00 2001
From: Alexander Khryukin <alexander@mezon.ru>
Date: Thu, 17 Mar 2016 15:09:10 +0300
Subject: [PATCH] disable crawler bots

---
 app/views/sitemap/robots.erb | 20 ++------------------
 1 file changed, 2 insertions(+), 18 deletions(-)

diff --git a/app/views/sitemap/robots.erb b/app/views/sitemap/robots.erb
index 4462e67de..136243691 100644
--- a/app/views/sitemap/robots.erb
+++ b/app/views/sitemap/robots.erb
@@ -1,22 +1,6 @@
 # robots.txt for <%= root_url %>
 
 User-agent: *
-Disallow: /*/*/tree/*
-Disallow: /*/*/blob/*
-Disallow: /*/*/wiki/*/*
-Disallow: /*/*/issues?*
-Disallow: /*/*/commits/*
-Disallow: /*/blame/
-Disallow: /*/raw/
-Disallow: /*.git
-Disallow: /*.git/
-Disallow: /search
-Disallow: /users/sign_up
-Disallow: /users/sign_in
-Disallow: /users/password/new
-Disallow: /users/register_requests/new
-Disallow: /invite.html
-Disallow: /thanks.html
-Allow: /
+Disallow: /
 Host: <%= root_url.gsub(/\/$/, '') %>
-Sitemap: <%= sitemap_url %>
\ No newline at end of file
+Sitemap: <%= sitemap_url %>