Skip to content

Commit 6c7cf45

Browse files
Alig1493Alig1493
committed
Moved project tasks
1 parent 1648006 commit 6c7cf45

File tree

9 files changed

+256
-228
lines changed

9 files changed

+256
-228
lines changed

‎readthedocs/builds/tasks.py

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import datetime
2+
import logging
3+
import shutil
4+
5+
from django.db.models import Q
6+
from django.utils import timezone
7+
from django.utils.translation import ugettext_lazy as _
8+
9+
from readthedocs.builds.constants import BUILD_STATE_FINISHED
10+
from readthedocs.builds.models import Version, Build
11+
from readthedocs.doc_builder.constants import DOCKER_LIMITS
12+
from readthedocs.projects.constants import LOG_TEMPLATE
13+
14+
from readthedocs.worker import app
15+
16+
17+
log = logging.getLogger(__name__)
18+
19+
20+
@app.task(queue='web')
21+
def fileify(version_pk, commit):
22+
"""
23+
Create ImportedFile objects for all of a version's files.
24+
25+
This is so we have an idea of what files we have in the database.
26+
"""
27+
version = Version.objects.get(pk=version_pk)
28+
project = version.project
29+
30+
if not commit:
31+
log.info(
32+
LOG_TEMPLATE.format(
33+
project=project.slug,
34+
version=version.slug,
35+
msg=(
36+
'Imported File not being built because no commit '
37+
'information'
38+
),
39+
)
40+
)
41+
return
42+
43+
path = project.rtd_build_path(version.slug)
44+
if path:
45+
log.info(
46+
LOG_TEMPLATE.format(
47+
project=version.project.slug,
48+
version=version.slug,
49+
msg='Creating ImportedFiles',
50+
)
51+
)
52+
from readthedocs.projects.tasks import _manage_imported_files
53+
_manage_imported_files(version, path, commit)
54+
else:
55+
log.info(
56+
LOG_TEMPLATE.format(
57+
project=project.slug,
58+
version=version.slug,
59+
msg='No ImportedFile files',
60+
)
61+
)
62+
63+
64+
# Random Tasks
65+
@app.task()
66+
def remove_dir(path):
67+
"""
68+
Remove a directory on the build/celery server.
69+
70+
This is mainly a wrapper around shutil.rmtree so that app servers can kill
71+
things on the build server.
72+
"""
73+
log.info('Removing %s', path)
74+
shutil.rmtree(path, ignore_errors=True)
75+
76+
77+
@app.task()
78+
def clear_artifacts(paths):
79+
"""
80+
Remove artifacts from the web servers.
81+
82+
:param paths: list containing PATHs where production media is on disk
83+
(usually ``Version.get_artifact_paths``)
84+
"""
85+
for path in paths:
86+
remove_dir(path)
87+
88+
89+
@app.task()
90+
def finish_inactive_builds():
91+
"""
92+
Finish inactive builds.
93+
94+
A build is consider inactive if it's not in ``FINISHED`` state and it has been
95+
"running" for more time that the allowed one (``Project.container_time_limit``
96+
or ``DOCKER_LIMITS['time']`` plus a 20% of it).
97+
98+
These inactive builds will be marked as ``success`` and ``FINISHED`` with an
99+
``error`` to be communicated to the user.
100+
"""
101+
time_limit = int(DOCKER_LIMITS['time'] * 1.2)
102+
delta = datetime.timedelta(seconds=time_limit)
103+
query = (~Q(state=BUILD_STATE_FINISHED) &
104+
Q(date__lte=timezone.now() - delta))
105+
106+
builds_finished = 0
107+
builds = Build.objects.filter(query)[:50]
108+
for build in builds:
109+
110+
if build.project.container_time_limit:
111+
custom_delta = datetime.timedelta(
112+
seconds=int(build.project.container_time_limit),
113+
)
114+
if build.date + custom_delta > timezone.now():
115+
# Do not mark as FINISHED builds with a custom time limit that wasn't
116+
# expired yet (they are still building the project version)
117+
continue
118+
119+
build.success = False
120+
build.state = BUILD_STATE_FINISHED
121+
build.error = _(
122+
'This build was terminated due to inactivity. If you '
123+
'continue to encounter this error, file a support '
124+
'request with and reference this build id ({0}).'.format(build.pk),
125+
)
126+
build.save()
127+
builds_finished += 1
128+
129+
log.info(
130+
'Builds marked as "Terminated due inactivity": %s',
131+
builds_finished,
132+
)

‎readthedocs/core/tasks.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212

1313
from readthedocs.worker import app
1414

15-
1615
log = logging.getLogger(__name__)
1716

1817
EMAIL_TIME_LIMIT = 30

‎readthedocs/notifications/tasks.py

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
import json
2+
import logging
3+
4+
import requests
5+
from django.conf import settings
6+
from django.core.urlresolvers import reverse
7+
from django.utils.translation import ugettext_lazy as _
8+
9+
from readthedocs.builds.models import Version, Build
10+
from readthedocs.core.utils import send_email
11+
from readthedocs.projects.constants import LOG_TEMPLATE
12+
from readthedocs.worker import app
13+
14+
15+
log = logging.getLogger(__name__)
16+
17+
18+
@app.task(queue='web')
19+
def send_notifications(version_pk, build_pk):
20+
version = Version.objects.get(pk=version_pk)
21+
build = Build.objects.get(pk=build_pk)
22+
23+
for hook in version.project.webhook_notifications.all():
24+
webhook_notification(version, build, hook.url)
25+
for email in version.project.emailhook_notifications.all().values_list('email', flat=True):
26+
email_notification(version, build, email)
27+
28+
29+
def email_notification(version, build, email):
30+
"""
31+
Send email notifications for build failure.
32+
33+
:param version: :py:class:`Version` instance that failed
34+
:param build: :py:class:`Build` instance that failed
35+
:param email: Email recipient address
36+
"""
37+
log.debug(
38+
LOG_TEMPLATE.format(
39+
project=version.project.slug,
40+
version=version.slug,
41+
msg='sending email to: %s' % email,
42+
)
43+
)
44+
45+
# We send only what we need from the Django model objects here to avoid
46+
# serialization problems in the ``readthedocs.core.tasks.send_email_task``
47+
context = {
48+
'version': {
49+
'verbose_name': version.verbose_name,
50+
},
51+
'project': {
52+
'name': version.project.name,
53+
},
54+
'build': {
55+
'pk': build.pk,
56+
'error': build.error,
57+
},
58+
'build_url': 'https://{0}{1}'.format(
59+
getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'),
60+
build.get_absolute_url(),
61+
),
62+
'unsub_url': 'https://{0}{1}'.format(
63+
getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'),
64+
reverse('projects_notifications', args=[version.project.slug]),
65+
),
66+
}
67+
68+
if build.commit:
69+
title = _('Failed: {project[name]} ({commit})').format(commit=build.commit[:8], **context)
70+
else:
71+
title = _('Failed: {project[name]} ({version[verbose_name]})').format(**context)
72+
73+
send_email(
74+
email,
75+
title,
76+
template='projects/email/build_failed.txt',
77+
template_html='projects/email/build_failed.html',
78+
context=context,
79+
)
80+
81+
82+
def webhook_notification(version, build, hook_url):
83+
"""
84+
Send webhook notification for project webhook.
85+
86+
:param version: Version instance to send hook for
87+
:param build: Build instance that failed
88+
:param hook_url: Hook URL to send to
89+
"""
90+
project = version.project
91+
92+
data = json.dumps({
93+
'name': project.name,
94+
'slug': project.slug,
95+
'build': {
96+
'id': build.id,
97+
'success': build.success,
98+
'date': build.date.strftime('%Y-%m-%d %H:%M:%S'),
99+
},
100+
})
101+
log.debug(
102+
LOG_TEMPLATE.format(
103+
project=project.slug,
104+
version='',
105+
msg='sending notification to: %s' % hook_url,
106+
)
107+
)
108+
try:
109+
requests.post(hook_url, data=data)
110+
except Exception:
111+
log.exception('Failed to POST on webhook url: url=%s', hook_url)

‎readthedocs/projects/admin.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from django.utils.translation import ugettext_lazy as _
88
from guardian.admin import GuardedModelAdmin
99

10+
from readthedocs.builds.tasks import remove_dir
1011
from readthedocs.core.models import UserProfile
1112
from readthedocs.core.utils import broadcast
1213
from readthedocs.builds.models import Version
@@ -17,7 +18,6 @@
1718
from .models import (Project, ImportedFile, Feature,
1819
ProjectRelationship, EmailHook, WebHook, Domain)
1920
from .notifications import ResourceUsageNotification
20-
from .tasks import remove_dir
2121

2222

2323
class ProjectSendNotificationView(SendNotificationView):

0 commit comments

Comments
 (0)