From e42942e0d17002532f6483faff89204aed7b043a Mon Sep 17 00:00:00 2001 From: ali_cramstack Date: Sun, 11 Nov 2018 15:08:45 +0600 Subject: [PATCH 1/2] Moved project tasks --- readthedocs/builds/tasks.py | 132 +++++++++++ readthedocs/core/tasks.py | 1 - readthedocs/notifications/tasks.py | 111 +++++++++ readthedocs/projects/admin.py | 2 +- readthedocs/projects/tasks.py | 218 +----------------- .../tests/test_build_notifications.py | 6 +- readthedocs/rtd_tests/tests/test_celery.py | 7 +- .../rtd_tests/tests/test_imported_file.py | 2 +- readthedocs/rtd_tests/tests/test_project.py | 2 +- 9 files changed, 257 insertions(+), 224 deletions(-) create mode 100644 readthedocs/builds/tasks.py create mode 100644 readthedocs/notifications/tasks.py diff --git a/readthedocs/builds/tasks.py b/readthedocs/builds/tasks.py new file mode 100644 index 00000000000..fc73bfc1368 --- /dev/null +++ b/readthedocs/builds/tasks.py @@ -0,0 +1,132 @@ +import datetime +import logging +import shutil + +from django.db.models import Q +from django.utils import timezone +from django.utils.translation import ugettext_lazy as _ + +from readthedocs.builds.constants import BUILD_STATE_FINISHED +from readthedocs.builds.models import Version, Build +from readthedocs.doc_builder.constants import DOCKER_LIMITS +from readthedocs.projects.constants import LOG_TEMPLATE + +from readthedocs.worker import app + + +log = logging.getLogger(__name__) + + +@app.task(queue='web') +def fileify(version_pk, commit): + """ + Create ImportedFile objects for all of a version's files. + + This is so we have an idea of what files we have in the database. + """ + version = Version.objects.get(pk=version_pk) + project = version.project + + if not commit: + log.info( + LOG_TEMPLATE.format( + project=project.slug, + version=version.slug, + msg=( + 'Imported File not being built because no commit ' + 'information' + ), + ) + ) + return + + path = project.rtd_build_path(version.slug) + if path: + log.info( + LOG_TEMPLATE.format( + project=version.project.slug, + version=version.slug, + msg='Creating ImportedFiles', + ) + ) + from readthedocs.projects.tasks import _manage_imported_files + _manage_imported_files(version, path, commit) + else: + log.info( + LOG_TEMPLATE.format( + project=project.slug, + version=version.slug, + msg='No ImportedFile files', + ) + ) + + +# Random Tasks +@app.task() +def remove_dir(path): + """ + Remove a directory on the build/celery server. + + This is mainly a wrapper around shutil.rmtree so that app servers can kill + things on the build server. + """ + log.info('Removing %s', path) + shutil.rmtree(path, ignore_errors=True) + + +@app.task() +def clear_artifacts(paths): + """ + Remove artifacts from the web servers. + + :param paths: list containing PATHs where production media is on disk + (usually ``Version.get_artifact_paths``) + """ + for path in paths: + remove_dir(path) + + +@app.task() +def finish_inactive_builds(): + """ + Finish inactive builds. + + A build is consider inactive if it's not in ``FINISHED`` state and it has been + "running" for more time that the allowed one (``Project.container_time_limit`` + or ``DOCKER_LIMITS['time']`` plus a 20% of it). + + These inactive builds will be marked as ``success`` and ``FINISHED`` with an + ``error`` to be communicated to the user. + """ + time_limit = int(DOCKER_LIMITS['time'] * 1.2) + delta = datetime.timedelta(seconds=time_limit) + query = (~Q(state=BUILD_STATE_FINISHED) & + Q(date__lte=timezone.now() - delta)) + + builds_finished = 0 + builds = Build.objects.filter(query)[:50] + for build in builds: + + if build.project.container_time_limit: + custom_delta = datetime.timedelta( + seconds=int(build.project.container_time_limit), + ) + if build.date + custom_delta > timezone.now(): + # Do not mark as FINISHED builds with a custom time limit that wasn't + # expired yet (they are still building the project version) + continue + + build.success = False + build.state = BUILD_STATE_FINISHED + build.error = _( + 'This build was terminated due to inactivity. If you ' + 'continue to encounter this error, file a support ' + 'request with and reference this build id ({0}).'.format(build.pk), + ) + build.save() + builds_finished += 1 + + log.info( + 'Builds marked as "Terminated due inactivity": %s', + builds_finished, + ) diff --git a/readthedocs/core/tasks.py b/readthedocs/core/tasks.py index 8ed81f1bb1e..e636b05a969 100644 --- a/readthedocs/core/tasks.py +++ b/readthedocs/core/tasks.py @@ -12,7 +12,6 @@ from readthedocs.worker import app - log = logging.getLogger(__name__) EMAIL_TIME_LIMIT = 30 diff --git a/readthedocs/notifications/tasks.py b/readthedocs/notifications/tasks.py new file mode 100644 index 00000000000..5a02ad8e64f --- /dev/null +++ b/readthedocs/notifications/tasks.py @@ -0,0 +1,111 @@ +import json +import logging + +import requests +from django.conf import settings +from django.core.urlresolvers import reverse +from django.utils.translation import ugettext_lazy as _ + +from readthedocs.builds.models import Version, Build +from readthedocs.core.utils import send_email +from readthedocs.projects.constants import LOG_TEMPLATE +from readthedocs.worker import app + + +log = logging.getLogger(__name__) + + +@app.task(queue='web') +def send_notifications(version_pk, build_pk): + version = Version.objects.get(pk=version_pk) + build = Build.objects.get(pk=build_pk) + + for hook in version.project.webhook_notifications.all(): + webhook_notification(version, build, hook.url) + for email in version.project.emailhook_notifications.all().values_list('email', flat=True): + email_notification(version, build, email) + + +def email_notification(version, build, email): + """ + Send email notifications for build failure. + + :param version: :py:class:`Version` instance that failed + :param build: :py:class:`Build` instance that failed + :param email: Email recipient address + """ + log.debug( + LOG_TEMPLATE.format( + project=version.project.slug, + version=version.slug, + msg='sending email to: %s' % email, + ) + ) + + # We send only what we need from the Django model objects here to avoid + # serialization problems in the ``readthedocs.core.tasks.send_email_task`` + context = { + 'version': { + 'verbose_name': version.verbose_name, + }, + 'project': { + 'name': version.project.name, + }, + 'build': { + 'pk': build.pk, + 'error': build.error, + }, + 'build_url': 'https://{0}{1}'.format( + getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), + build.get_absolute_url(), + ), + 'unsub_url': 'https://{0}{1}'.format( + getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), + reverse('projects_notifications', args=[version.project.slug]), + ), + } + + if build.commit: + title = _('Failed: {project[name]} ({commit})').format(commit=build.commit[:8], **context) + else: + title = _('Failed: {project[name]} ({version[verbose_name]})').format(**context) + + send_email( + email, + title, + template='projects/email/build_failed.txt', + template_html='projects/email/build_failed.html', + context=context, + ) + + +def webhook_notification(version, build, hook_url): + """ + Send webhook notification for project webhook. + + :param version: Version instance to send hook for + :param build: Build instance that failed + :param hook_url: Hook URL to send to + """ + project = version.project + + data = json.dumps({ + 'name': project.name, + 'slug': project.slug, + 'build': { + 'id': build.id, + 'success': build.success, + 'date': build.date.strftime('%Y-%m-%d %H:%M:%S'), + }, + }) + log.debug( + LOG_TEMPLATE.format( + project=project.slug, + version='', + msg='sending notification to: %s' % hook_url, + ) + ) + try: + requests.post(hook_url, data=data) + except Exception: + log.exception('Failed to POST on webhook url: url=%s', hook_url) diff --git a/readthedocs/projects/admin.py b/readthedocs/projects/admin.py index f564b65297e..95ec224c269 100644 --- a/readthedocs/projects/admin.py +++ b/readthedocs/projects/admin.py @@ -14,6 +14,7 @@ from guardian.admin import GuardedModelAdmin from readthedocs.builds.models import Version +from readthedocs.builds.tasks import remove_dir from readthedocs.core.models import UserProfile from readthedocs.core.utils import broadcast from readthedocs.notifications.views import SendNotificationView @@ -31,7 +32,6 @@ WebHook, ) from .notifications import ResourceUsageNotification -from .tasks import remove_dir class ProjectSendNotificationView(SendNotificationView): diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index b36df403baf..5dcba2fee8b 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -13,16 +13,13 @@ unicode_literals, ) -import datetime import hashlib import json import logging import os -import shutil import socket from collections import Counter, defaultdict -import requests from builtins import str from celery.exceptions import SoftTimeLimitExceeded from django.conf import settings @@ -44,12 +41,14 @@ from readthedocs.builds.models import APIVersion, Build, Version from readthedocs.builds.signals import build_complete from readthedocs.builds.syncers import Syncer +from readthedocs.builds.tasks import fileify, remove_dir from readthedocs.config import ConfigError from readthedocs.core.resolver import resolve_path from readthedocs.core.symlink import PrivateSymlink, PublicSymlink from readthedocs.core.utils import broadcast, safe_unlink, send_email +from readthedocs.core.symlink import PublicSymlink, PrivateSymlink +from readthedocs.core.utils import broadcast, safe_unlink from readthedocs.doc_builder.config import load_yaml_config -from readthedocs.doc_builder.constants import DOCKER_LIMITS from readthedocs.doc_builder.environments import ( DockerBuildEnvironment, LocalBuildEnvironment, @@ -63,6 +62,7 @@ ) from readthedocs.doc_builder.loader import get_builder_class from readthedocs.doc_builder.python_environments import Conda, Virtualenv +from readthedocs.notifications.tasks import send_notifications from readthedocs.projects.models import APIProject from readthedocs.restapi.client import api as api_v2 from readthedocs.restapi.utils import index_search_request @@ -1085,49 +1085,6 @@ def symlink_subproject(project_pk): sym.symlink_subprojects() -@app.task(queue='web') -def fileify(version_pk, commit): - """ - Create ImportedFile objects for all of a version's files. - - This is so we have an idea of what files we have in the database. - """ - version = Version.objects.get(pk=version_pk) - project = version.project - - if not commit: - log.info( - LOG_TEMPLATE.format( - project=project.slug, - version=version.slug, - msg=( - 'Imported File not being built because no commit ' - 'information' - ), - ) - ) - return - - path = project.rtd_build_path(version.slug) - if path: - log.info( - LOG_TEMPLATE.format( - project=version.project.slug, - version=version.slug, - msg='Creating ImportedFiles', - ) - ) - _manage_imported_files(version, path, commit) - else: - log.info( - LOG_TEMPLATE.format( - project=project.slug, - version=version.slug, - msg='No ImportedFile files', - ) - ) - - def _manage_imported_files(version, path, commit): """ Update imported files for version. @@ -1172,102 +1129,6 @@ def _manage_imported_files(version, path, commit): files=changed_files) -@app.task(queue='web') -def send_notifications(version_pk, build_pk): - version = Version.objects.get(pk=version_pk) - build = Build.objects.get(pk=build_pk) - - for hook in version.project.webhook_notifications.all(): - webhook_notification(version, build, hook.url) - for email in version.project.emailhook_notifications.all().values_list('email', flat=True): - email_notification(version, build, email) - - -def email_notification(version, build, email): - """ - Send email notifications for build failure. - - :param version: :py:class:`Version` instance that failed - :param build: :py:class:`Build` instance that failed - :param email: Email recipient address - """ - log.debug( - LOG_TEMPLATE.format( - project=version.project.slug, - version=version.slug, - msg='sending email to: %s' % email, - ) - ) - - # We send only what we need from the Django model objects here to avoid - # serialization problems in the ``readthedocs.core.tasks.send_email_task`` - context = { - 'version': { - 'verbose_name': version.verbose_name, - }, - 'project': { - 'name': version.project.name, - }, - 'build': { - 'pk': build.pk, - 'error': build.error, - }, - 'build_url': 'https://{0}{1}'.format( - getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), - build.get_absolute_url(), - ), - 'unsub_url': 'https://{0}{1}'.format( - getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), - reverse('projects_notifications', args=[version.project.slug]), - ), - } - - if build.commit: - title = _('Failed: {project[name]} ({commit})').format(commit=build.commit[:8], **context) - else: - title = _('Failed: {project[name]} ({version[verbose_name]})').format(**context) - - send_email( - email, - title, - template='projects/email/build_failed.txt', - template_html='projects/email/build_failed.html', - context=context, - ) - - -def webhook_notification(version, build, hook_url): - """ - Send webhook notification for project webhook. - - :param version: Version instance to send hook for - :param build: Build instance that failed - :param hook_url: Hook URL to send to - """ - project = version.project - - data = json.dumps({ - 'name': project.name, - 'slug': project.slug, - 'build': { - 'id': build.id, - 'success': build.success, - 'date': build.date.strftime('%Y-%m-%d %H:%M:%S'), - }, - }) - log.debug( - LOG_TEMPLATE.format( - project=project.slug, - version='', - msg='sending notification to: %s' % hook_url, - ) - ) - try: - requests.post(hook_url, data=data) - except Exception: - log.exception('Failed to POST on webhook url: url=%s', hook_url) - - @app.task(queue='web') def update_static_metadata(project_pk, path=None): """ @@ -1320,31 +1181,6 @@ def update_static_metadata(project_pk, path=None): ) -# Random Tasks -@app.task() -def remove_dir(path): - """ - Remove a directory on the build/celery server. - - This is mainly a wrapper around shutil.rmtree so that app servers can kill - things on the build server. - """ - log.info('Removing %s', path) - shutil.rmtree(path, ignore_errors=True) - - -@app.task() -def clear_artifacts(paths): - """ - Remove artifacts from the web servers. - - :param paths: list containing PATHs where production media is on disk - (usually ``Version.get_artifact_paths``) - """ - for path in paths: - remove_dir(path) - - @app.task(queue='web') def sync_callback(_, version_pk, commit, *args, **kwargs): """ @@ -1354,49 +1190,3 @@ def sync_callback(_, version_pk, commit, *args, **kwargs): """ fileify(version_pk, commit=commit) update_search(version_pk, commit=commit) - - -@app.task() -def finish_inactive_builds(): - """ - Finish inactive builds. - - A build is consider inactive if it's not in ``FINISHED`` state and it has been - "running" for more time that the allowed one (``Project.container_time_limit`` - or ``DOCKER_LIMITS['time']`` plus a 20% of it). - - These inactive builds will be marked as ``success`` and ``FINISHED`` with an - ``error`` to be communicated to the user. - """ - time_limit = int(DOCKER_LIMITS['time'] * 1.2) - delta = datetime.timedelta(seconds=time_limit) - query = (~Q(state=BUILD_STATE_FINISHED) & - Q(date__lte=timezone.now() - delta)) - - builds_finished = 0 - builds = Build.objects.filter(query)[:50] - for build in builds: - - if build.project.container_time_limit: - custom_delta = datetime.timedelta( - seconds=int(build.project.container_time_limit), - ) - if build.date + custom_delta > timezone.now(): - # Do not mark as FINISHED builds with a custom time limit that wasn't - # expired yet (they are still building the project version) - continue - - build.success = False - build.state = BUILD_STATE_FINISHED - build.error = _( - 'This build was terminated due to inactivity. If you ' - 'continue to encounter this error, file a support ' - 'request with and reference this build id ({0}).'.format(build.pk), - ) - build.save() - builds_finished += 1 - - log.info( - 'Builds marked as "Terminated due inactivity": %s', - builds_finished, - ) diff --git a/readthedocs/rtd_tests/tests/test_build_notifications.py b/readthedocs/rtd_tests/tests/test_build_notifications.py index 9460589463a..a7d99c6cbcb 100644 --- a/readthedocs/rtd_tests/tests/test_build_notifications.py +++ b/readthedocs/rtd_tests/tests/test_build_notifications.py @@ -11,7 +11,7 @@ from readthedocs.builds.models import Build, Version from readthedocs.projects.models import Project, EmailHook, WebHook -from readthedocs.projects.tasks import send_notifications +from readthedocs.notifications.tasks import send_notifications from readthedocs.projects.forms import WebHookForm @@ -27,7 +27,7 @@ def test_send_notification_none(self): def test_send_webhook_notification(self): fixture.get(WebHook, project=self.project) - with patch('readthedocs.projects.tasks.requests.post') as mock: + with patch('readthedocs.notifications.tasks.requests.post') as mock: mock.return_value = None send_notifications(self.version.pk, self.build.pk) mock.assert_called_once() @@ -42,7 +42,7 @@ def test_send_email_notification(self): def test_send_email_and_webhook__notification(self): fixture.get(EmailHook, project=self.project) fixture.get(WebHook, project=self.project) - with patch('readthedocs.projects.tasks.requests.post') as mock: + with patch('readthedocs.notifications.tasks.requests.post') as mock: mock.return_value = None send_notifications(self.version.pk, self.build.pk) mock.assert_called_once() diff --git a/readthedocs/rtd_tests/tests/test_celery.py b/readthedocs/rtd_tests/tests/test_celery.py index 4e5ed8856e6..669e20ad53d 100644 --- a/readthedocs/rtd_tests/tests/test_celery.py +++ b/readthedocs/rtd_tests/tests/test_celery.py @@ -15,6 +15,7 @@ from readthedocs.builds.models import Build from readthedocs.projects.models import Project from readthedocs.projects import tasks +from readthedocs.builds.tasks import clear_artifacts, remove_dir from readthedocs.rtd_tests.utils import ( create_git_branch, create_git_tag, delete_git_branch) @@ -49,7 +50,7 @@ def tearDown(self): def test_remove_dir(self): directory = mkdtemp() self.assertTrue(exists(directory)) - result = tasks.remove_dir.delay(directory) + result = remove_dir.delay(directory) self.assertTrue(result.successful()) self.assertFalse(exists(directory)) @@ -58,14 +59,14 @@ def test_clear_artifacts(self): directory = self.project.get_production_media_path(type_='pdf', version_slug=version.slug) os.makedirs(directory) self.assertTrue(exists(directory)) - result = tasks.clear_artifacts.delay(paths=version.get_artifact_paths()) + result = clear_artifacts.delay(paths=version.get_artifact_paths()) self.assertTrue(result.successful()) self.assertFalse(exists(directory)) directory = version.project.rtd_build_path(version=version.slug) os.makedirs(directory) self.assertTrue(exists(directory)) - result = tasks.clear_artifacts.delay(paths=version.get_artifact_paths()) + result = clear_artifacts.delay(paths=version.get_artifact_paths()) self.assertTrue(result.successful()) self.assertFalse(exists(directory)) diff --git a/readthedocs/rtd_tests/tests/test_imported_file.py b/readthedocs/rtd_tests/tests/test_imported_file.py index 81f5e2a684b..9d8f168ce8b 100644 --- a/readthedocs/rtd_tests/tests/test_imported_file.py +++ b/readthedocs/rtd_tests/tests/test_imported_file.py @@ -2,8 +2,8 @@ import os from django.test import TestCase -from readthedocs.projects.tasks import _manage_imported_files from readthedocs.projects.models import Project, ImportedFile +from readthedocs.projects.tasks import _manage_imported_files base_dir = os.path.dirname(os.path.dirname(__file__)) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 5029eb58630..44509ca968b 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -16,9 +16,9 @@ from readthedocs.builds.constants import ( BUILD_STATE_CLONING, BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST) from readthedocs.builds.models import Build +from readthedocs.builds.tasks import finish_inactive_builds from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project -from readthedocs.projects.tasks import finish_inactive_builds from readthedocs.rtd_tests.mocks.paths import fake_paths_by_regex From ceb0579773a294c321dfa5e4acf4289d966e4cec Mon Sep 17 00:00:00 2001 From: --replace Date: Thu, 22 Nov 2018 17:49:03 +0600 Subject: [PATCH 2/2] Made requested changes --- readthedocs/builds/tasks.py | 50 +++++++++++++++++- readthedocs/projects/tasks.py | 52 +------------------ .../rtd_tests/tests/test_imported_file.py | 2 +- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/readthedocs/builds/tasks.py b/readthedocs/builds/tasks.py index fc73bfc1368..45ce9d06742 100644 --- a/readthedocs/builds/tasks.py +++ b/readthedocs/builds/tasks.py @@ -1,5 +1,7 @@ import datetime +import hashlib import logging +import os import shutil from django.db.models import Q @@ -8,8 +10,11 @@ from readthedocs.builds.constants import BUILD_STATE_FINISHED from readthedocs.builds.models import Version, Build +from readthedocs.core.resolver import resolve_path from readthedocs.doc_builder.constants import DOCKER_LIMITS from readthedocs.projects.constants import LOG_TEMPLATE +from readthedocs.projects.models import ImportedFile, Project +from readthedocs.projects.signals import files_changed from readthedocs.worker import app @@ -17,6 +22,50 @@ log = logging.getLogger(__name__) +def _manage_imported_files(version, path, commit): + """ + Update imported files for version. + + :param version: Version instance + :param path: Path to search + :param commit: Commit that updated path + """ + changed_files = set() + for root, __, filenames in os.walk(path): + for filename in filenames: + dirpath = os.path.join(root.replace(path, '').lstrip('/'), + filename.lstrip('/')) + full_path = os.path.join(root, filename) + md5 = hashlib.md5(open(full_path, 'rb').read()).hexdigest() + try: + obj, __ = ImportedFile.objects.get_or_create( + project=version.project, + version=version, + path=dirpath, + name=filename, + ) + except ImportedFile.MultipleObjectsReturned: + log.warning('Error creating ImportedFile') + continue + if obj.md5 != md5: + obj.md5 = md5 + changed_files.add(dirpath) + if obj.commit != commit: + obj.commit = commit + obj.save() + # Delete ImportedFiles from previous versions + ImportedFile.objects.filter(project=version.project, + version=version + ).exclude(commit=commit).delete() + changed_files = [ + resolve_path( + version.project, filename=file, version_slug=version.slug, + ) for file in changed_files + ] + files_changed.send(sender=Project, project=version.project, + files=changed_files) + + @app.task(queue='web') def fileify(version_pk, commit): """ @@ -61,7 +110,6 @@ def fileify(version_pk, commit): ) -# Random Tasks @app.task() def remove_dir(path): """ diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 5dcba2fee8b..558448e8fd1 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -13,7 +13,6 @@ unicode_literals, ) -import hashlib import json import logging import os @@ -43,9 +42,6 @@ from readthedocs.builds.syncers import Syncer from readthedocs.builds.tasks import fileify, remove_dir from readthedocs.config import ConfigError -from readthedocs.core.resolver import resolve_path -from readthedocs.core.symlink import PrivateSymlink, PublicSymlink -from readthedocs.core.utils import broadcast, safe_unlink, send_email from readthedocs.core.symlink import PublicSymlink, PrivateSymlink from readthedocs.core.utils import broadcast, safe_unlink from readthedocs.doc_builder.config import load_yaml_config @@ -72,15 +68,15 @@ from .constants import LOG_TEMPLATE from .exceptions import RepositoryError -from .models import Domain, ImportedFile, Project +from .models import Domain, Project from .signals import ( after_build, after_vcs, before_build, before_vcs, - files_changed, ) + log = logging.getLogger(__name__) HTML_ONLY = getattr(settings, 'HTML_ONLY_PROJECTS', ()) @@ -1085,50 +1081,6 @@ def symlink_subproject(project_pk): sym.symlink_subprojects() -def _manage_imported_files(version, path, commit): - """ - Update imported files for version. - - :param version: Version instance - :param path: Path to search - :param commit: Commit that updated path - """ - changed_files = set() - for root, __, filenames in os.walk(path): - for filename in filenames: - dirpath = os.path.join(root.replace(path, '').lstrip('/'), - filename.lstrip('/')) - full_path = os.path.join(root, filename) - md5 = hashlib.md5(open(full_path, 'rb').read()).hexdigest() - try: - obj, __ = ImportedFile.objects.get_or_create( - project=version.project, - version=version, - path=dirpath, - name=filename, - ) - except ImportedFile.MultipleObjectsReturned: - log.warning('Error creating ImportedFile') - continue - if obj.md5 != md5: - obj.md5 = md5 - changed_files.add(dirpath) - if obj.commit != commit: - obj.commit = commit - obj.save() - # Delete ImportedFiles from previous versions - ImportedFile.objects.filter(project=version.project, - version=version - ).exclude(commit=commit).delete() - changed_files = [ - resolve_path( - version.project, filename=file, version_slug=version.slug, - ) for file in changed_files - ] - files_changed.send(sender=Project, project=version.project, - files=changed_files) - - @app.task(queue='web') def update_static_metadata(project_pk, path=None): """ diff --git a/readthedocs/rtd_tests/tests/test_imported_file.py b/readthedocs/rtd_tests/tests/test_imported_file.py index 9d8f168ce8b..5b0c1ad3d87 100644 --- a/readthedocs/rtd_tests/tests/test_imported_file.py +++ b/readthedocs/rtd_tests/tests/test_imported_file.py @@ -3,7 +3,7 @@ from django.test import TestCase from readthedocs.projects.models import Project, ImportedFile -from readthedocs.projects.tasks import _manage_imported_files +from readthedocs.builds.tasks import _manage_imported_files base_dir = os.path.dirname(os.path.dirname(__file__))