Support delisting of projects

readthedocs/projects/migrations/0096_add_project_delisted.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.2.18 on 2023-02-22 12:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("projects", "0095_default_branch_helptext"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="historicalproject",
+            name="delisted",
+            field=models.BooleanField(
+                default=False,
+                help_text="Delisting a project removes it from search results and asks search engines to remove it via robots.txt",
+                verbose_name="Delisted",
+            ),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="delisted",
+            field=models.BooleanField(
+                default=False,
+                help_text="Delisting a project removes it from search results and asks search engines to remove it via robots.txt",
+                verbose_name="Delisted",
+            ),
+        ),
+    ]

readthedocs/projects/models.py

@@ -359,7 +359,18 @@ class Project(models.Model):
     )
 
     featured = models.BooleanField(_('Featured'), default=False)
+
+    # TODO: What is this for?
     skip = models.BooleanField(_('Skip'), default=False)
+
+    delisted = models.BooleanField(
+        default=False,
+        verbose_name=_("Delisted"),
+        help_text=_(
+            "Delisting a project removes it from search results and asks search engines to remove it via robots.txt"
+        ),
+    )
+
     install_project = models.BooleanField(
         _('Install Project'),
         help_text=_(

readthedocs/proxito/views/serve.py

@@ -477,8 +477,15 @@ def get(self, request):
         project, we serve it directly.
         """
         project = request.unresolved_domain.project
+        # Check if project is delisted
+        if project.delisted:
+            return render(
+                request,
+                "robots.delisted.txt",
+                content_type="text/plain",
+            )
         # Verify if the project is marked as spam and return a custom robots.txt
-        if 'readthedocsext.spamfighting' in settings.INSTALLED_APPS:
+        elif "readthedocsext.spamfighting" in settings.INSTALLED_APPS:
             from readthedocsext.spamfighting.utils import is_robotstxt_denied  # noqa
             if is_robotstxt_denied(project):
                 return render(

readthedocs/search/documents.py

@@ -48,6 +48,15 @@ class ProjectDocument(RTDDocTypeMixin, Document):
 
     modified_model_field = 'modified_date'
 
+    def get_queryset(self):
+        """
+        Don't include delisted projects.
+        This will also break in-doc search for these projects,
+        but it's not a priority to find a solution for this as long as "delisted" projects are
+        understood to be projects with a negative reason for being delisted.
+        """
+        return super().get_queryset().exclude(delisted=True)
+
     class Django:
         model = Project
         fields = []
@@ -175,11 +184,12 @@ def prepare_domains(self, html_file):
         return all_domains
 
     def get_queryset(self):
-        """Don't include ignored files."""
+        """Don't include ignored files and delisted projects."""
         queryset = super().get_queryset()
         queryset = (
             queryset
             .exclude(ignore=True)
+            .exclude(project__delisted=True)
             .select_related('version', 'project')
         )
         return queryset

readthedocs/templates/core/project_bar_base.html

@@ -21,6 +21,9 @@ <h1>
           {% endblock %}
         </span>
         <a href="{{ project.get_absolute_url }}">{{ project }}</a>
+        {% if project.delisted %}
+          <span class="quiet">{% trans "delisted" %}</span>
+        {% endif %}
       </h1>
     </div>
 

readthedocs/templates/robots.delisted.txt

@@ -0,0 +1,3 @@
+# Delisted project, removing from search indexing
+User-agent: *
+Disallow: /