From 9cb4ef6f23e63c270d313a24b8d3415e58faf7ce Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 23 Jan 2018 14:33:49 -0500 Subject: [PATCH 1/4] Task to remove orphan symlinks --- readthedocs/projects/tasks.py | 19 +++++ .../rtd_tests/tests/test_project_symlinks.py | 74 +++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index a670b2a3b3c..94c2cec8aef 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -778,6 +778,25 @@ def symlink_domain(project_pk, domain_pk, delete=False): sym.symlink_cnames(domain) +@app.task(queue='web') +def remove_orphan_symlinks(): + """ + Remove orphan symlinks. + + List CNAME_ROOT for Public and Private symlinks, check that all the listed + cname exist in the database and if doesn't exist, they are un-linked. + """ + for symlink in [PublicSymlink, PrivateSymlink]: + for domain_path in [symlink.PROJECT_CNAME_ROOT, symlink.CNAME_ROOT]: + for domain in os.listdir(domain_path): + try: + Domain.objects.get(domain=domain) + except Domain.DoesNotExist: + orphan_domain_path = os.path.join(domain_path, domain) + log.info('Unlinking orphan CNAME: %s', orphan_domain_path) + os.unlink(orphan_domain_path) + + @app.task(queue='web') def symlink_subproject(project_pk): project = Project.objects.get(pk=project_pk) diff --git a/readthedocs/rtd_tests/tests/test_project_symlinks.py b/readthedocs/rtd_tests/tests/test_project_symlinks.py index c55e6ea8f23..c1506c9dbd3 100644 --- a/readthedocs/rtd_tests/tests/test_project_symlinks.py +++ b/readthedocs/rtd_tests/tests/test_project_symlinks.py @@ -15,6 +15,7 @@ from readthedocs.builds.models import Version from readthedocs.projects.models import Project, Domain +from readthedocs.projects.tasks import remove_orphan_symlinks from readthedocs.core.symlink import PublicSymlink, PrivateSymlink @@ -166,6 +167,79 @@ def test_symlink_cname(self): filesystem['private_web_root'] = public_root self.assertFilesystem(filesystem) + def test_symlink_remove_orphan_symlinks(self): + self.domain = get(Domain, project=self.project, domain='woot.com', + url='http://woot.com', cname=True) + self.symlink.symlink_cnames() + + # Editing the Domain and calling save will symlink the new domain and + # leave the old one as orphan. + self.domain.domain = 'foobar.com' + self.domain.save() + + filesystem = { + 'private_cname_project': { + 'foobar.com': {'type': 'link', 'target': 'user_builds/kong'}, + 'woot.com': {'type': 'link', 'target': 'user_builds/kong'}, + }, + 'private_cname_root': { + 'foobar.com': {'type': 'link', 'target': 'private_web_root/kong'}, + 'woot.com': {'type': 'link', 'target': 'private_web_root/kong'}, + }, + 'private_web_root': {'kong': {'en': {}}}, + 'public_cname_project': { + 'foobar.com': {'type': 'link', 'target': 'user_builds/kong'}, + 'woot.com': {'type': 'link', 'target': 'user_builds/kong'}, + }, + 'public_cname_root': { + 'foobar.com': {'type': 'link', 'target': 'public_web_root/kong'}, + 'woot.com': {'type': 'link', 'target': 'public_web_root/kong'}, + }, + 'public_web_root': { + 'kong': {'en': {'latest': { + 'type': 'link', + 'target': 'user_builds/kong/rtd-builds/latest', + }}} + } + } + if self.privacy == 'private': + public_root = filesystem['public_web_root'].copy() + private_root = filesystem['private_web_root'].copy() + filesystem['public_web_root'] = private_root + filesystem['private_web_root'] = public_root + self.assertFilesystem(filesystem) + + remove_orphan_symlinks() + filesystem = { + 'private_cname_project': { + 'foobar.com': {'type': 'link', 'target': 'user_builds/kong'}, + }, + 'private_cname_root': { + 'foobar.com': {'type': 'link', 'target': 'private_web_root/kong'}, + }, + 'private_web_root': {'kong': {'en': {}}}, + 'public_cname_project': { + 'foobar.com': {'type': 'link', 'target': 'user_builds/kong'}, + }, + 'public_cname_root': { + 'foobar.com': {'type': 'link', 'target': 'public_web_root/kong'}, + }, + 'public_web_root': { + 'kong': {'en': {'latest': { + 'type': 'link', + 'target': 'user_builds/kong/rtd-builds/latest', + }}}, + }, + } + if self.privacy == 'private': + public_root = filesystem['public_web_root'].copy() + private_root = filesystem['private_web_root'].copy() + filesystem['public_web_root'] = private_root + filesystem['private_web_root'] = public_root + + self.assertFilesystem(filesystem) + + def test_symlink_cname_dont_link_missing_domains(self): """Domains should be relinked after deletion""" self.domain = get(Domain, project=self.project, domain='woot.com', From e035eacdfb186a45f5be7295495de12bdf4366ba Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Fri, 9 Mar 2018 09:12:55 -0500 Subject: [PATCH 2/4] Use a better db query to accomplish this task --- readthedocs/projects/tasks.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 94c2cec8aef..70c7bd936b8 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -788,13 +788,12 @@ def remove_orphan_symlinks(): """ for symlink in [PublicSymlink, PrivateSymlink]: for domain_path in [symlink.PROJECT_CNAME_ROOT, symlink.CNAME_ROOT]: - for domain in os.listdir(domain_path): - try: - Domain.objects.get(domain=domain) - except Domain.DoesNotExist: - orphan_domain_path = os.path.join(domain_path, domain) - log.info('Unlinking orphan CNAME: %s', orphan_domain_path) - os.unlink(orphan_domain_path) + valid_cnames = set(Domain.objects.all().values_list('domain', flat=True)) + orphan_cnames = set(os.listdir(domain_path)) - valid_cnames + for cname in orphan_cnames: + orphan_domain_path = os.path.join(domain_path, cname) + log.info('Unlinking orphan CNAME: %s', orphan_domain_path) + os.unlink(orphan_domain_path) @app.task(queue='web') From 2555fe0e5e30797a58dea1e0b9fe19a4ae4c053f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Fri, 9 Mar 2018 20:03:50 -0500 Subject: [PATCH 3/4] Setup task to be ran by celery beat --- readthedocs/settings/base.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 9e59347bda2..e0370e36797 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -8,6 +8,9 @@ from readthedocs.core.settings import Settings +from celery.schedules import crontab + + try: import readthedocsext # noqa ext = True @@ -240,6 +243,14 @@ def USE_PROMOS(self): # noqa CELERY_CREATE_MISSING_QUEUES = True CELERY_DEFAULT_QUEUE = 'celery' + CELERYBEAT_SCHEDULE = { + # Ran every hour on minute 30 + 'hourly-remove-orphan-symlinks': { + 'task': 'readthedocs.projects.tasks.remove_orphan_symlinks', + 'schedule': crontab(minute=30), + 'options': {'queue': 'web'}, + }, + } # Docker DOCKER_ENABLE = False From b68ea63921f66ab38df7ef7e1812f7be1f323384 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Fri, 9 Mar 2018 20:26:06 -0500 Subject: [PATCH 4/4] Register celery beat task useful for development --- readthedocs/settings/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index e0370e36797..ccca7805d73 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -250,6 +250,11 @@ def USE_PROMOS(self): # noqa 'schedule': crontab(minute=30), 'options': {'queue': 'web'}, }, + 'quarter-finish-inactive-builds': { + 'task': 'readthedocs.projects.tasks.finish_inactive_builds', + 'schedule': crontab(minute='*/15'), + 'options': {'queue': 'web'}, + }, } # Docker