Security logs: delete old user security logs (#8620)

Delete all logs that aren't attached to an organization, this is since those logs should be deleted according to their plan on .com.

Logs that aren't attached to an organization are logs from authentication to the dashboard, aka user security logs.

Author: stsewd

readthedocs/audit/tasks.py

@@ -0,0 +1,30 @@
+"""Celery tasks related to audit logs."""
+
+import structlog
+from django.conf import settings
+from django.utils import timezone
+
+from readthedocs.audit.models import AuditLog
+from readthedocs.worker import app
+
+log = structlog.get_logger(__name__)
+
+
+@app.task(queue="web")
+def delete_old_personal_audit_logs(days=None):
+    """
+    Delete personal security logs older than `days`.
+
+    If `days` isn't given, default to ``RTD_AUDITLOGS_DEFAULT_RETENTION_DAYS``.
+
+    We delete logs that aren't related to an organization,
+    there are tasks in .com to delete those according to their plan.
+    """
+    days = days or settings.RTD_AUDITLOGS_DEFAULT_RETENTION_DAYS
+    days_ago = timezone.now() - timezone.timedelta(days=days)
+    audit_logs = AuditLog.objects.filter(
+        log_organization_id__isnull=True,
+        created__lt=days_ago,
+    )
+    log.info("Deleting old audit logs.", days=days, count=audit_logs.count())
+    audit_logs.delete()

readthedocs/audit/tests/test_tasks.py

@@ -0,0 +1,115 @@
+from unittest import mock
+
+from django.contrib.auth.models import User
+from django.test import TestCase
+from django.utils import timezone
+from django_dynamic_fixture import get
+
+from readthedocs.audit.models import AuditLog
+from readthedocs.audit.tasks import delete_old_personal_audit_logs
+from readthedocs.organizations.models import Organization
+from readthedocs.projects.models import Project
+
+
+class AuditTasksTest(TestCase):
+    def setUp(self):
+        self.user = get(User)
+        self.project = get(
+            Project,
+            slug="project",
+        )
+        self.organization = get(
+            Organization,
+            owners=[self.user],
+            name="testorg",
+        )
+        self.organization.projects.add(self.project)
+
+        self.another_user = get(User)
+        self.another_project = get(
+            Project,
+            slug="another-project",
+            users=[self.user],
+        )
+
+    @mock.patch("django.utils.timezone.now")
+    def test_delete_old_personal_audit_logs(self, now_mock):
+        now_mock.return_value = timezone.datetime(
+            year=2021,
+            month=5,
+            day=5,
+        )
+        newer_date = timezone.datetime(
+            year=2021,
+            month=4,
+            day=30,
+        )
+        middle_date = timezone.datetime(
+            year=2021,
+            month=4,
+            day=5,
+        )
+        old_date = timezone.datetime(
+            year=2021,
+            month=3,
+            day=20,
+        )
+        for date in [newer_date, middle_date, old_date]:
+            for user in [self.user, self.another_user]:
+                # Log attached to a project and organization.
+                get(
+                    AuditLog,
+                    user=user,
+                    project=self.project,
+                    created=date,
+                    action=AuditLog.PAGEVIEW,
+                )
+                # Log attached to a project only.
+                get(
+                    AuditLog,
+                    user=user,
+                    project=self.another_project,
+                    created=date,
+                    action=AuditLog.PAGEVIEW,
+                )
+
+                # Log attached to the user only.
+                get(
+                    AuditLog,
+                    user=user,
+                    created=date,
+                    action=AuditLog.AUTHN,
+                )
+
+            # Log without a user.
+            get(
+                AuditLog,
+                created=date,
+                action=AuditLog.AUTHN_FAILURE,
+            )
+
+            # Log with a organization, and without a user.
+            get(
+                AuditLog,
+                project=self.project,
+                created=date,
+                action=AuditLog.AUTHN_FAILURE,
+            )
+
+        self.assertEqual(AuditLog.objects.all().count(), 24)
+
+        # We don't have logs older than 90 days.
+        delete_old_personal_audit_logs(days=90)
+        self.assertEqual(AuditLog.objects.all().count(), 24)
+
+        # Only 5 logs can be deteled.
+        delete_old_personal_audit_logs(days=30)
+        self.assertEqual(AuditLog.objects.all().count(), 19)
+
+        # Only 5 logs can be deteled.
+        delete_old_personal_audit_logs(days=10)
+        self.assertEqual(AuditLog.objects.all().count(), 14)
+
+        # Only 5 logs can be deteled.
+        delete_old_personal_audit_logs(days=1)
+        self.assertEqual(AuditLog.objects.all().count(), 9)

readthedocs/settings/base.py

@@ -435,6 +435,11 @@ def TEMPLATES(self):
             'schedule': crontab(minute=0, hour=2),
             'options': {'queue': 'web'},
         },
+        'weekly-delete-old-personal-audit-logs': {
+            'task': 'readthedocs.audit.tasks.delete_old_personal_audit_logs',
+            'schedule': crontab(day_of_week='wednesday', minute=0, hour=7),
+            'options': {'queue': 'web'},
+        },
         'every-day-resync-sso-organization-users': {
             'task': 'readthedocs.oauth.tasks.sync_remote_repositories_organizations',
             'schedule': crontab(minute=0, hour=4),