Add a specific ban on AhrefsBot

ericholscher · ericholscher · commit 7d77435aadac · 2019-03-19T14:17:39.000-03:00
diff --git a/media/robots.txt b/media/robots.txt
@@ -3,3 +3,7 @@ Disallow: /search/
 Disallow: /api/
 Disallow: /builds/
 Disallow: /sustainability/click/
+
+# This was hitting our site and causing a lot of issues
+User-agent: AhrefsBot 
+Disallow: /
