Merge pull request #8418 from readthedocs/humitos/assets-cleanup

Author: humitos

readthedocs/organizations/migrations/0006_add_assets_cleaned.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.24 on 2021-08-17 14:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('organizations', '0005_historicalorganization_historicalteam'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='historicalorganization',
+            name='artifacts_cleaned',
+            field=models.BooleanField(default=False, help_text='Artifacts are cleaned out from storage', verbose_name='Artifacts Cleaned'),
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='artifacts_cleaned',
+            field=models.BooleanField(default=False, help_text='Artifacts are cleaned out from storage', verbose_name='Artifacts Cleaned'),
+        ),
+    ]

readthedocs/organizations/models.py

@@ -76,6 +76,11 @@ class Organization(models.Model):
         help_text='Docs and builds are disabled for this organization',
         default=False,
     )
+    artifacts_cleaned = models.BooleanField(
+        _('Artifacts Cleaned'),
+        help_text='Artifacts are cleaned out from storage',
+        default=False,
+    )
     max_concurrent_builds = models.IntegerField(
         _('Maximum concurrent builds allowed for this organization'),
         null=True,

readthedocs/organizations/signals.py

@@ -7,7 +7,9 @@
 from django.db.models.signals import pre_delete
 from django.dispatch import receiver
 
-from readthedocs.builds.models import Version
+from readthedocs.builds.constants import BUILD_STATE_FINISHED
+from readthedocs.builds.models import Build, Version
+from readthedocs.builds.signals import build_complete
 from readthedocs.organizations.models import (
     Organization,
     Team,
@@ -16,6 +18,8 @@
 )
 from readthedocs.projects.models import Project
 
+from .tasks import mark_organization_assets_not_cleaned as mark_organization_assets_not_cleaned_task
+
 log = logging.getLogger(__name__)
 
 
@@ -74,3 +78,17 @@ def remove_organization_completely(sender, instance, using, **kwargs):
         version.delete()
 
     projects.delete()
+
+
+@receiver(build_complete, sender=Build)
+def mark_organization_assets_not_cleaned(sender, build, **kwargs):
+    """
+    Mark the organization assets as not cleaned if there is a new build.
+
+    This signal triggers a Celery task because the `build_complete` signal is
+    fired by the builder and it does not have access to the database. So, we
+    trigger a Celery task that will be executed in the web and mark the
+    organization assets as not cleaned.
+    """
+    if build['state'] == BUILD_STATE_FINISHED:
+        mark_organization_assets_not_cleaned_task.delay(build['id'])

readthedocs/organizations/tasks.py

@@ -0,0 +1,25 @@
+"""Organization's tasks related."""
+
+import logging
+
+from readthedocs.builds.models import Build
+from readthedocs.worker import app
+
+
+log = logging.getLogger(__name__)
+
+
+@app.task(queue='web')
+def mark_organization_assets_not_cleaned(build_pk):
+    """Mark an organization as `artifacts_cleaned=False`."""
+    try:
+        build = Build.objects.get(pk=build_pk)
+    except Build.DoesNotExist:
+        log.info("Build does not exist. build=%s", build_pk)
+        return
+
+    organization = build.project.organizations.first()
+    if organization:
+        log.info("Marking organization as not cleaned. organization=%s", organization.slug)
+        organization.artifacts_cleaned = False
+        organization.save()