Support delisting of projects

readthedocs/projects/migrations/0096_add_project_delisted.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.18 on 2023-02-24 16:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("projects", "0095_default_branch_helptext"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="historicalproject",
+            name="delisted",
+            field=models.BooleanField(
+                default=False,
+                help_text="Delisting a project removes it from Read the Docs search indexing and asks external search engines to remove it via robots.txt",
+                verbose_name="Delisted",
+            ),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="delisted",
+            field=models.BooleanField(
+                default=False,
+                help_text="Delisting a project removes it from Read the Docs search indexing and asks external search engines to remove it via robots.txt",
+                verbose_name="Delisted",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="historicalproject",
+            name="skip",
+            field=models.BooleanField(
+                default=False, verbose_name="Skip (disable) building this project"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="project",
+            name="skip",
+            field=models.BooleanField(
+                default=False, verbose_name="Skip (disable) building this project"
+            ),
+        ),
+    ]

readthedocs/projects/models.py

@@ -359,7 +359,18 @@ class Project(models.Model):
     )
 
     featured = models.BooleanField(_('Featured'), default=False)
-    skip = models.BooleanField(_('Skip'), default=False)
+
+    skip = models.BooleanField(_("Skip (disable) building this project"), default=False)
+
+    delisted = models.BooleanField(
+        default=False,
+        verbose_name=_("Delisted"),
+        help_text=_(
+            "Delisting a project removes it from Read the Docs search indexing and asks external "
+            "search engines to remove it via robots.txt"
+        ),
+    )
+
     install_project = models.BooleanField(
         _('Install Project'),
         help_text=_(

readthedocs/proxito/views/serve.py

@@ -285,7 +285,7 @@ def get(self, request, proxito_path, template_name='404.html'):
         """
         Handler for 404 pages on subdomains.
 
-        This does a couple things:
+        This does a couple of things:
 
         * Handles directory indexing for URLs that don't end in a slash
         * Handles directory indexing for README.html (for now)
@@ -473,12 +473,22 @@ def get(self, request):
         """
         Serve custom user's defined ``/robots.txt``.
 
+        If the project is delisted or is a spam project, we force a special robots.txt.
+
         If the user added a ``robots.txt`` in the "default version" of the
         project, we serve it directly.
         """
         project = request.unresolved_domain.project
+
+        if project.delisted:
+            return render(
+                request,
+                "robots.delisted.txt",
+                content_type="text/plain",
+            )
+
         # Verify if the project is marked as spam and return a custom robots.txt
-        if 'readthedocsext.spamfighting' in settings.INSTALLED_APPS:
+        elif "readthedocsext.spamfighting" in settings.INSTALLED_APPS:
             from readthedocsext.spamfighting.utils import is_robotstxt_denied  # noqa
             if is_robotstxt_denied(project):
                 return render(

readthedocs/search/documents.py

@@ -48,6 +48,17 @@ class ProjectDocument(RTDDocTypeMixin, Document):
 
     modified_model_field = 'modified_date'
 
+    def get_queryset(self):
+        """
+        Additional filtering of default queryset.
+
+        Don't include delisted projects.
+        This will also break in-doc search for these projects,
+        but it's not a priority to find a solution for this as long as "delisted" projects are
+        understood to be projects with a negative reason for being delisted.
+        """
+        return super().get_queryset().exclude(delisted=True).exclude(is_spam=True)
+
     class Django:
         model = Project
         fields = []
@@ -175,11 +186,13 @@ def prepare_domains(self, html_file):
         return all_domains
 
     def get_queryset(self):
-        """Don't include ignored files."""
+        """Don't include ignored files and delisted projects."""
         queryset = super().get_queryset()
         queryset = (
             queryset
             .exclude(ignore=True)
+            .exclude(project__delisted=True)
+            .exclude(project__is_spam=True)
             .select_related('version', 'project')
         )
         return queryset

readthedocs/templates/core/project_bar_base.html

@@ -21,6 +21,9 @@ <h1>
           {% endblock %}
         </span>
         <a href="{{ project.get_absolute_url }}">{{ project }}</a>
+        {% if project.delisted %}
+          <a href="https://docs.readthedocs.io/en/stable/unofficial-projects.html" class="quiet" target="_blank">{% trans "delisted" %}</a>
+        {% endif %}
       </h1>
     </div>
 

readthedocs/templates/robots.delisted.txt

@@ -0,0 +1,4 @@
+# Delisted project, removing from search indexing
+# See: https://docs.readthedocs.io/en/stable/unofficial-projects.html
+User-agent: *
+Disallow: /