Merge pull request #8632 from readthedocs/humitos/spamfighting

Author: humitos

media/images/header-logo.png

@@ -1,5 +1,7 @@
 """Django administration interface for `projects.models`."""
 
+from django.db.models import Sum
+from django.conf import settings
 from django.contrib import admin, messages
 from django.contrib.admin.actions import delete_selected
 from django.forms import BaseInlineFormSet
@@ -143,42 +145,90 @@ def queryset(self, request, queryset):
         return queryset
 
 
+class ProjectSpamThreshold(admin.SimpleListFilter):
+
+    """Filter for projects that are potentially SPAM."""
+
+    title = 'Spam Threshold'
+    parameter_name = 'spam_threshold'
+
+    DONT_SHOW_ADS = 'dont_show_ads'
+    DENY_ON_ROBOTS = 'deny_on_robots'
+    DONT_SERVE_DOCS = 'dont_serve_docs'
+    DONT_SHOW_DASHBOARD = 'dont_show_dashboard'
+    DELETE_PROJECT = 'delete_project'
+
+    def lookups(self, request, model_admin):
+        return (
+            (self.DONT_SHOW_ADS, _("Don't show Ads")),
+            (self.DENY_ON_ROBOTS, _('Deny on robots')),
+            (self.DONT_SERVE_DOCS, _("Don't serve docs")),
+            (self.DONT_SHOW_DASHBOARD, _("Don't show dashboard")),
+            (self.DELETE_PROJECT, _('Delete project')),
+        )
+
+    def queryset(self, request, queryset):
+        queryset = queryset.annotate(spam_score=Sum('spam_rules__value'))
+        if self.value() == self.DONT_SHOW_ADS:
+            return queryset.filter(spam_score__gte=settings.RTD_SPAM_THRESHOLD_DONT_SHOW_ADS)
+        if self.value() == self.DENY_ON_ROBOTS:
+            return queryset.filter(spam_score__gte=settings.RTD_SPAM_THRESHOLD_DENY_ON_ROBOTS)
+        if self.value() == self.DONT_SERVE_DOCS:
+            return queryset.filter(spam_score__gte=settings.RTD_SPAM_THRESHOLD_DONT_SERVE_DOCS)
+        if self.value() == self.DONT_SHOW_DASHBOARD:
+            return queryset.filter(spam_score__gte=settings.RTD_SPAM_THRESHOLD_DONT_SHOW_DASHBOARD)
+        if self.value() == self.DELETE_PROJECT:
+            return queryset.filter(spam_score__gte=settings.RTD_SPAM_THRESHOLD_DELETE_PROJECT)
+        return queryset
+
+
 class ProjectAdmin(ExtraSimpleHistoryAdmin):
 
     """Project model admin view."""
 
     prepopulated_fields = {'slug': ('name',)}
-    list_display = ('name', 'slug', 'repo', 'repo_type', 'featured')
-    list_filter = (
+    list_display = ('name', 'slug', 'repo')
+
+    list_filter = tuple()
+    if 'readthedocsext.spamfighting' in settings.INSTALLED_APPS:
+        list_filter = list_filter + (ProjectSpamThreshold,)
+
+    list_filter = list_filter + (
+        ProjectOwnerBannedFilter,
+        'feature__feature_id',
         'repo_type',
-        'featured',
         'privacy_level',
-        'documentation_type',
         'programming_language',
-        'feature__feature_id',
-        ProjectOwnerBannedFilter,
+        'documentation_type',
     )
-    list_editable = ('featured',)
+
     search_fields = ('slug', 'repo')
     inlines = [
         ProjectRelationshipInline,
         RedirectInline,
         VersionInline,
         DomainInline,
     ]
-    readonly_fields = ('pub_date', 'feature_flags',)
+    readonly_fields = ('pub_date', 'feature_flags', 'matching_spam_rules')
     raw_id_fields = ('users', 'main_language_project', 'remote_repository')
     actions = [
         'send_owner_email',
         'ban_owner',
+        'run_spam_rule_checks',
         'build_default_version',
         'reindex_active_versions',
         'wipe_all_versions',
         'import_tags_from_vcs',
     ]
 
+    def matching_spam_rules(self, obj):
+        result = []
+        for spam_rule in obj.spam_rules.filter(enabled=True):
+            result.append(f'{spam_rule.spam_rule_type} ({spam_rule.value})')
+        return '\n'.join(result) or 'No matching spam rules'
+
     def feature_flags(self, obj):
-        return ', '.join([str(f.get_feature_display()) for f in obj.features])
+        return '\n'.join([str(f.get_feature_display()) for f in obj.features])
 
     def send_owner_email(self, request, queryset):
         view = ProjectSendNotificationView.as_view(
@@ -188,6 +238,26 @@ def send_owner_email(self, request, queryset):
 
     send_owner_email.short_description = 'Notify project owners'
 
+    def run_spam_rule_checks(self, request, queryset):
+        """Run all the spam checks on this project."""
+        if 'readthedocsext.spamfighting' not in settings.INSTALLED_APPS:
+            messages.add_message(
+                request,
+                messages.ERROR,
+                'Spam fighting Django application not installed',
+            )
+            return
+
+        from readthedocsext.spamfighting.tasks import spam_rules_check  # noqa
+        project_slugs = queryset.values_list('slug', flat=True)
+        # NOTE: convert queryset to a simple list so Celery can serialize it
+        spam_rules_check.delay(project_slugs=list(project_slugs))
+        messages.add_message(
+            request,
+            messages.INFO,
+            'Spam check task triggered for {} projects'.format(queryset.count()),
+        )
+
     def ban_owner(self, request, queryset):
         """
         Ban project owner.

readthedocs/projects/admin.py

@@ -1252,6 +1252,10 @@ def show_advertising(self):
         if self.ad_free or self.gold_owners.exists():
             return False
 
+        if 'readthedocsext.spamfighting' in settings.INSTALLED_APPS:
+            from readthedocsext.spamfighting.utils import is_show_ads_denied  # noqa
+            return not is_show_ads_denied(self)
+
         return True
 
     def environment_variables(self, *, public_only=True):

readthedocs/projects/models.py

@@ -176,6 +176,12 @@ def get_version_from_host(self, request, version_slug):
             version_slug = request.host_version_slug
         return version_slug
 
+    def _spam_response(self, request, project):
+        if 'readthedocsext.spamfighting' in settings.INSTALLED_APPS:
+            from readthedocsext.spamfighting.utils import is_serve_docs_denied  # noqa
+            if is_serve_docs_denied(project):
+                return render(request, template_name='spam.html', status=401)
+
 
 class ServeRedirectMixin:
 

readthedocs/proxito/views/mixins.py

@@ -5,6 +5,7 @@
 from urllib.parse import urlparse
 
 from readthedocs.core.resolver import resolve_path
+from django.conf import settings
 from django.http import Http404, HttpResponse, HttpResponseRedirect
 from django.shortcuts import render
 from django.urls import resolve as url_resolve
@@ -82,6 +83,11 @@ def get(self,
             final_project.slug, subproject_slug, lang_slug, version_slug, filename
         )
 
+        # Verify if the project is marked as spam and return a 401 in that case
+        spam_response = self._spam_response(request, final_project)
+        if spam_response:
+            return spam_response
+
         # Handle requests that need canonicalizing (eg. HTTP -> HTTPS, redirect to canonical domain)
         if hasattr(request, 'canonicalize'):
             try:
@@ -348,6 +354,16 @@ def get(self, request, project):
         project, we serve it directly.
         """
 
+        # Verify if the project is marked as spam and return a custom robots.txt
+        if 'readthedocsext.spamfighting' in settings.INSTALLED_APPS:
+            from readthedocsext.spamfighting.utils import is_robotstxt_denied  # noqa
+            if is_robotstxt_denied(project):
+                return render(
+                    request,
+                    'robots.spam.txt',
+                    content_type='text/plain',
+                )
+
         # Use the ``robots.txt`` file from the default version configured
         version_slug = project.get_default_version()
         version = project.versions.get(slug=version_slug)

readthedocs/proxito/views/serve.py

@@ -848,3 +848,9 @@ def DOCKER_LIMITS(self):
     RTD_EMBED_API_DEFAULT_REQUEST_TIMEOUT = 1
     RTD_EMBED_API_DOMAIN_RATE_LIMIT = 50
     RTD_EMBED_API_DOMAIN_RATE_LIMIT_TIMEOUT = 60
+
+    RTD_SPAM_THRESHOLD_DONT_SHOW_ADS = 100
+    RTD_SPAM_THRESHOLD_DENY_ON_ROBOTS = 200
+    RTD_SPAM_THRESHOLD_DONT_SHOW_DASHBOARD = 300
+    RTD_SPAM_THRESHOLD_DONT_SERVE_DOCS = 500
+    RTD_SPAM_THRESHOLD_DELETE_PROJECT = 1000

readthedocs/settings/base.py

@@ -0,0 +1,3 @@
+# Spam project detected, removing from search indexing
+User-agent: *
+Disallow: /

readthedocs/templates/robots.spam.txt

@@ -0,0 +1,32 @@
+{% load static %}
+<!DOCTYPE html>
+<html lang=en>
+    <head>
+        <meta charset=utf-8>
+        <meta name=viewport content="width=device-width, initial-scale=1, shrink-to-fit=no">
+        <link rel="icon" type="image/png" href="{% static 'images/favicon.png' %}">
+        <link rel="stylesheet" href="{% static 'css/core.css' %}">
+        <title>Spam Project</title>
+        <style>
+         main {
+             max-width: 600px;
+             margin: 4rem auto;
+             text-align: center;
+         }
+         img {
+             width: 100%;
+             max-width: 500px;
+         }
+        </style>
+    </head>
+    <body>
+        <main>
+            <img alt="Read the Docs Logo" src={% static 'images/header-logo.png' %}>
+            <h1>Project marked as spam</h1>
+            <p>Read the Docs has marked this content as spam and is not serving it anymore.</p>
+            {# We can't use url templatetag here because the URL is not defined in El Proxito #}
+            <p>Please <a href="https://{{ PUBLIC_DOMAIN }}/support/">contact us</a> if you think this is a mistake.</p>
+            <p style="color: #666;">401 - Unauthorized</p>
+        </main>
+    </body>
+</html>