Define useful celery beat task for development (#3762)

* Task to remove orphan symlinks

* Use a better db query to accomplish this task

* Setup task to be ran by celery beat

* Register celery beat task useful for development

Author: humitos

readthedocs/projects/tasks.py

@@ -810,6 +810,24 @@ def symlink_domain(project_pk, domain_pk, delete=False):
             sym.symlink_cnames(domain)
 
 
+@app.task(queue='web')
+def remove_orphan_symlinks():
+    """
+    Remove orphan symlinks.
+
+    List CNAME_ROOT for Public and Private symlinks, check that all the listed
+    cname exist in the database and if doesn't exist, they are un-linked.
+    """
+    for symlink in [PublicSymlink, PrivateSymlink]:
+        for domain_path in [symlink.PROJECT_CNAME_ROOT, symlink.CNAME_ROOT]:
+            valid_cnames = set(Domain.objects.all().values_list('domain', flat=True))
+            orphan_cnames = set(os.listdir(domain_path)) - valid_cnames
+            for cname in orphan_cnames:
+                orphan_domain_path = os.path.join(domain_path, cname)
+                log.info('Unlinking orphan CNAME: %s', orphan_domain_path)
+                os.unlink(orphan_domain_path)
+
+
 @app.task(queue='web')
 def symlink_subproject(project_pk):
     project = Project.objects.get(pk=project_pk)

readthedocs/rtd_tests/tests/test_project_symlinks.py

@@ -14,7 +14,7 @@
 
 from readthedocs.builds.models import Version
 from readthedocs.projects.models import Project, Domain
-from readthedocs.projects.tasks import symlink_project
+from readthedocs.projects.tasks import symlink_project, remove_orphan_symlinks
 from readthedocs.core.symlink import PublicSymlink, PrivateSymlink
 
 
@@ -166,6 +166,79 @@ def test_symlink_cname(self):
             filesystem['private_web_root'] = public_root
         self.assertFilesystem(filesystem)
 
+    def test_symlink_remove_orphan_symlinks(self):
+        self.domain = get(Domain, project=self.project, domain='woot.com',
+                          url='http://woot.com', cname=True)
+        self.symlink.symlink_cnames()
+
+        # Editing the Domain and calling save will symlink the new domain and
+        # leave the old one as orphan.
+        self.domain.domain = 'foobar.com'
+        self.domain.save()
+
+        filesystem = {
+            'private_cname_project': {
+                'foobar.com': {'type': 'link', 'target': 'user_builds/kong'},
+                'woot.com': {'type': 'link', 'target': 'user_builds/kong'},
+            },
+            'private_cname_root': {
+                'foobar.com': {'type': 'link', 'target': 'private_web_root/kong'},
+                'woot.com': {'type': 'link', 'target': 'private_web_root/kong'},
+            },
+            'private_web_root': {'kong': {'en': {}}},
+            'public_cname_project': {
+                'foobar.com': {'type': 'link', 'target': 'user_builds/kong'},
+                'woot.com': {'type': 'link', 'target': 'user_builds/kong'},
+            },
+            'public_cname_root': {
+                'foobar.com': {'type': 'link', 'target': 'public_web_root/kong'},
+                'woot.com': {'type': 'link', 'target': 'public_web_root/kong'},
+            },
+            'public_web_root': {
+                'kong': {'en': {'latest': {
+                    'type': 'link',
+                    'target': 'user_builds/kong/rtd-builds/latest',
+                }}}
+            }
+        }
+        if self.privacy == 'private':
+            public_root = filesystem['public_web_root'].copy()
+            private_root = filesystem['private_web_root'].copy()
+            filesystem['public_web_root'] = private_root
+            filesystem['private_web_root'] = public_root
+        self.assertFilesystem(filesystem)
+
+        remove_orphan_symlinks()
+        filesystem = {
+            'private_cname_project': {
+                'foobar.com': {'type': 'link', 'target': 'user_builds/kong'},
+            },
+            'private_cname_root': {
+                'foobar.com': {'type': 'link', 'target': 'private_web_root/kong'},
+            },
+            'private_web_root': {'kong': {'en': {}}},
+            'public_cname_project': {
+                'foobar.com': {'type': 'link', 'target': 'user_builds/kong'},
+            },
+            'public_cname_root': {
+                'foobar.com': {'type': 'link', 'target': 'public_web_root/kong'},
+            },
+            'public_web_root': {
+                'kong': {'en': {'latest': {
+                    'type': 'link',
+                    'target': 'user_builds/kong/rtd-builds/latest',
+                }}},
+            },
+        }
+        if self.privacy == 'private':
+            public_root = filesystem['public_web_root'].copy()
+            private_root = filesystem['private_web_root'].copy()
+            filesystem['public_web_root'] = private_root
+            filesystem['private_web_root'] = public_root
+
+        self.assertFilesystem(filesystem)
+
+
     def test_symlink_cname_dont_link_missing_domains(self):
         """Domains should be relinked after deletion"""
         self.domain = get(Domain, project=self.project, domain='woot.com',

readthedocs/settings/base.py

@@ -8,6 +8,9 @@
 
 from readthedocs.core.settings import Settings
 
+from celery.schedules import crontab
+
+
 try:
     import readthedocsext  # noqa
     ext = True
@@ -241,6 +244,19 @@ def USE_PROMOS(self):  # noqa
     CELERY_CREATE_MISSING_QUEUES = True
 
     CELERY_DEFAULT_QUEUE = 'celery'
+    CELERYBEAT_SCHEDULE = {
+        # Ran every hour on minute 30
+        'hourly-remove-orphan-symlinks': {
+            'task': 'readthedocs.projects.tasks.remove_orphan_symlinks',
+            'schedule': crontab(minute=30),
+            'options': {'queue': 'web'},
+        },
+        'quarter-finish-inactive-builds': {
+            'task': 'readthedocs.projects.tasks.finish_inactive_builds',
+            'schedule': crontab(minute='*/15'),
+            'options': {'queue': 'web'},
+        },
+    }
 
     # Docker
     DOCKER_ENABLE = False