Support delisting of projects

readthedocs/projects/migrations/0096_add_project_delisted.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.2.18 on 2023-02-22 12:29
+# Generated by Django 3.2.18 on 2023-02-24 16:02
 
 from django.db import migrations, models
 
@@ -15,7 +15,7 @@ class Migration(migrations.Migration):
             name="delisted",
             field=models.BooleanField(
                 default=False,
-                help_text="Delisting a project removes it from search results and asks search engines to remove it via robots.txt",
+                help_text="Delisting a project removes it from Read the Docs search indexing and asks external search engines to remove it via robots.txt",
                 verbose_name="Delisted",
             ),
         ),
@@ -24,8 +24,22 @@ class Migration(migrations.Migration):
             name="delisted",
             field=models.BooleanField(
                 default=False,
-                help_text="Delisting a project removes it from search results and asks search engines to remove it via robots.txt",
+                help_text="Delisting a project removes it from Read the Docs search indexing and asks external search engines to remove it via robots.txt",
                 verbose_name="Delisted",
             ),
         ),
+        migrations.AlterField(
+            model_name="historicalproject",
+            name="skip",
+            field=models.BooleanField(
+                default=False, verbose_name="Skip (disable) building this project"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="project",
+            name="skip",
+            field=models.BooleanField(
+                default=False, verbose_name="Skip (disable) building this project"
+            ),
+        ),
     ]

readthedocs/projects/models.py

@@ -360,14 +360,14 @@ class Project(models.Model):
 
     featured = models.BooleanField(_('Featured'), default=False)
 
-    # TODO: What is this for?
-    skip = models.BooleanField(_('Skip'), default=False)
+    skip = models.BooleanField(_("Skip (disable) building this project"), default=False)
 
     delisted = models.BooleanField(
         default=False,
         verbose_name=_("Delisted"),
         help_text=_(
-            "Delisting a project removes it from search results and asks search engines to remove it via robots.txt"
+            "Delisting a project removes it from Read the Docs search indexing and asks external "
+            "search engines to remove it via robots.txt"
         ),
     )
 

readthedocs/proxito/views/serve.py

@@ -285,7 +285,7 @@ def get(self, request, proxito_path, template_name='404.html'):
         """
         Handler for 404 pages on subdomains.
 
-        This does a couple things:
+        This does a couple of things:
 
         * Handles directory indexing for URLs that don't end in a slash
         * Handles directory indexing for README.html (for now)
@@ -473,17 +473,20 @@ def get(self, request):
         """
         Serve custom user's defined ``/robots.txt``.
 
+        If the project is delisted or is a spam project, we force a special robots.txt.
+
         If the user added a ``robots.txt`` in the "default version" of the
         project, we serve it directly.
         """
         project = request.unresolved_domain.project
-        # Check if project is delisted
+
         if project.delisted:
             return render(
                 request,
                 "robots.delisted.txt",
                 content_type="text/plain",
             )
+
         # Verify if the project is marked as spam and return a custom robots.txt
         elif "readthedocsext.spamfighting" in settings.INSTALLED_APPS:
             from readthedocsext.spamfighting.utils import is_robotstxt_denied  # noqa

readthedocs/search/documents.py

@@ -50,12 +50,14 @@ class ProjectDocument(RTDDocTypeMixin, Document):
 
     def get_queryset(self):
         """
+        Additional filtering of default queryset.
+
         Don't include delisted projects.
         This will also break in-doc search for these projects,
         but it's not a priority to find a solution for this as long as "delisted" projects are
         understood to be projects with a negative reason for being delisted.
         """
-        return super().get_queryset().exclude(delisted=True)
+        return super().get_queryset().exclude(delisted=True).exclude(is_spam=True)
 
     class Django:
         model = Project
@@ -190,6 +192,7 @@ def get_queryset(self):
             queryset
             .exclude(ignore=True)
             .exclude(project__delisted=True)
+            .exclude(project__is_spam=True)
             .select_related('version', 'project')
         )
         return queryset

readthedocs/templates/core/project_bar_base.html

@@ -22,7 +22,7 @@ <h1>
         </span>
         <a href="{{ project.get_absolute_url }}">{{ project }}</a>
         {% if project.delisted %}
-          <span class="quiet">{% trans "delisted" %}</span>
+          <a href="https://docs.readthedocs.io/en/stable/unofficial-projects.html" class="quiet" target="_blank">{% trans "delisted" %}</a>
         {% endif %}
       </h1>
     </div>

readthedocs/templates/robots.delisted.txt

@@ -1,3 +1,4 @@
 # Delisted project, removing from search indexing
+# See: https://docs.readthedocs.io/en/stable/unofficial-projects.html
 User-agent: *
 Disallow: /