From a7019340bdff9d3eee47c16ad7317834bfc7846c Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 19 Mar 2019 14:14:15 -0300 Subject: [PATCH 1/3] Remove search from robots.txt --- media/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/media/robots.txt b/media/robots.txt index a42cb659037..1645e5876b0 100644 --- a/media/robots.txt +++ b/media/robots.txt @@ -1,3 +1,4 @@ User-agent: * +Disallow: /search/ Disallow: /builds/ Disallow: /sustainability/click/ From abd95e4869be2ba98ca6430fb833d671086af5c7 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 19 Mar 2019 14:14:28 -0300 Subject: [PATCH 2/3] Remove API as well. --- media/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/media/robots.txt b/media/robots.txt index 1645e5876b0..0f5644de3a7 100644 --- a/media/robots.txt +++ b/media/robots.txt @@ -1,4 +1,5 @@ User-agent: * Disallow: /search/ +Disallow: /api/ Disallow: /builds/ Disallow: /sustainability/click/ From 7d77435aadacfb0bc7ba8054b92412a0d0b58f9c Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 19 Mar 2019 14:17:39 -0300 Subject: [PATCH 3/3] Add a specific ban on AhrefsBot --- media/robots.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/media/robots.txt b/media/robots.txt index 0f5644de3a7..c20e174afb9 100644 --- a/media/robots.txt +++ b/media/robots.txt @@ -3,3 +3,7 @@ Disallow: /search/ Disallow: /api/ Disallow: /builds/ Disallow: /sustainability/click/ + +# This was hitting our site and causing a lot of issues +User-agent: AhrefsBot +Disallow: /