Merge pull request #5501 from rtfd/remove-search-robots-txt

ericholscher · web-flow · commit 199480e241ef · 2019-03-20T09:28:41.000-03:00
Remove search &amp; API from robots.txt
diff --git a/media/robots.txt b/media/robots.txt
@@ -1,3 +1,9 @@
 User-agent: *
+Disallow: /search/
+Disallow: /api/
 Disallow: /builds/
 Disallow: /sustainability/click/
+
+# This was hitting our site and causing a lot of issues
+User-agent: AhrefsBot 
+Disallow: /
