Skip to content

Commit b1810c3

Browse files
humitoshumitos
committed
Deprecation: notification and feature flag for build.image config
Define a weekly task to communicate our users about the deprecation of `build.image` using the deprecation plan we used for the configuration file v2 as well. - 3 brownout days - final removal date on October 2nd - weekly onsite/email notification on Wednesday at 11:15 CEST (around ~22k projects affected) - allow to opt-out from these emails - feature flag for brownout days - build detail's page notification Related: * readthedocs/meta#48 * #10354 * #10587
1 parent a9955b6 commit b1810c3

File tree

12 files changed

+294
-8
lines changed

12 files changed

+294
-8
lines changed

‎readthedocs/builds/models.py

+15
Original file line numberDiff line numberDiff line change
@@ -1104,6 +1104,21 @@ def deprecated_config_used(self):
11041104

11051105
return int(self.config.get("version", "1")) != LATEST_CONFIGURATION_VERSION
11061106

1107+
def deprecated_build_image_used(self):
1108+
"""
1109+
Check whether this particular build is using the deprecated "build" config.
1110+
1111+
Note we are using this to communicate deprecation of "build.image".
1112+
See https://github.com/readthedocs/meta/discussions/48
1113+
"""
1114+
if not self.config:
1115+
# Don't notify users without a config file.
1116+
# We hope they will migrate to `build.os` in the process of adding a `.readthedocs.yaml`
1117+
return False
1118+
1119+
build_config_key = self.config.get("build", {})
1120+
return "image" in build_config_key or "os" not in build_config_key
1121+
11071122
def reset(self):
11081123
"""
11091124
Reset the build so it can be re-used when re-trying.

‎readthedocs/core/forms.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,10 @@ class Meta:
2323
model = UserProfile
2424
# Don't allow users edit someone else's user page
2525
profile_fields = ["first_name", "last_name", "homepage"]
26-
optout_email_fields = ["optout_email_config_file_deprecation"]
26+
optout_email_fields = [
27+
"optout_email_config_file_deprecation",
28+
"optout_email_build_image_deprecation",
29+
]
2730
fields = (
2831
*profile_fields,
2932
*optout_email_fields,

‎readthedocs/core/migrations/0014_optout_email_build_image_deprecation.py

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Generated by Django 3.2.20 on 2023-08-01 13:21
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
dependencies = [
8+
("core", "0013_add_optout_email_config_file_deprecation"),
9+
]
10+
11+
operations = [
12+
migrations.AddField(
13+
model_name="historicaluserprofile",
14+
name="optout_email_build_image_deprecation",
15+
field=models.BooleanField(
16+
default=False,
17+
null=True,
18+
verbose_name="Opt-out from email about '\"build.image\" config key deprecation'",
19+
),
20+
),
21+
migrations.AddField(
22+
model_name="userprofile",
23+
name="optout_email_build_image_deprecation",
24+
field=models.BooleanField(
25+
default=False,
26+
null=True,
27+
verbose_name="Opt-out from email about '\"build.image\" config key deprecation'",
28+
),
29+
),
30+
]

‎readthedocs/core/models.py

+7
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,13 @@ class UserProfile(TimeStampedModel):
5151
default=False,
5252
null=True,
5353
)
54+
# NOTE: this is a temporary field that we can remove after October 16, 2023
55+
# See https://blog.readthedocs.com/build-image-config-deprecated/
56+
optout_email_build_image_deprecation = models.BooleanField(
57+
_("Opt-out from email about '\"build.image\" config key deprecation'"),
58+
default=False,
59+
null=True,
60+
)
5461

5562
# Model history
5663
history = ExtraHistoricalRecords()

‎readthedocs/doc_builder/director.py

+10
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,16 @@ def checkout(self):
255255
) and self.data.config.version not in ("2", 2):
256256
raise BuildUserError(BuildUserError.NO_CONFIG_FILE_DEPRECATED)
257257

258+
# Raise a build error if the project is using "build.image" on their config file
259+
if self.data.project.has_feature(Feature.BUILD_IMAGE_CONFIG_KEY_DEPRECATED):
260+
build_config_key = self.data.config.source_config.get("build", {})
261+
if "image" in build_config_key:
262+
raise BuildUserError(BuildUserError.BUILD_IMAGE_CONFIG_KEY_DEPRECATED)
263+
264+
# TODO: move this validation to the Config object once we are settled here
265+
if "image" not in build_config_key and "os" not in build_config_key:
266+
raise BuildUserError(BuildUserError.BUILD_OS_REQUIRED)
267+
258268
if self.vcs_repository.supports_submodules:
259269
self.vcs_repository.update_submodules(self.data.config)
260270

‎readthedocs/doc_builder/exceptions.py

+9
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,15 @@ class BuildUserError(BuildBaseException):
6666
"Add a configuration file to your project to make it build successfully. "
6767
"Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html"
6868
)
69+
BUILD_IMAGE_CONFIG_KEY_DEPRECATED = gettext_noop(
70+
'The configuration key "build.image" is deprecated. '
71+
'Please, use "build.os" instead to make it build successfully. '
72+
"Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html"
73+
)
74+
BUILD_OS_REQUIRED = gettext_noop(
75+
'The configuration key "build.os" is required to build your documentation. '
76+
"Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html"
77+
)
6978

7079

7180
class BuildUserSkip(BuildUserError):

‎readthedocs/projects/models.py

+7
Original file line numberDiff line numberDiff line change
@@ -1950,6 +1950,7 @@ def add_features(sender, **kwargs):
19501950
GIT_CLONE_FETCH_CHECKOUT_PATTERN = "git_clone_fetch_checkout_pattern"
19511951
HOSTING_INTEGRATIONS = "hosting_integrations"
19521952
NO_CONFIG_FILE_DEPRECATED = "no_config_file"
1953+
BUILD_IMAGE_CONFIG_KEY_DEPRECATED = "build_image_config_key_deprecated"
19531954
SCALE_IN_PROTECTION = "scale_in_prtection"
19541955

19551956
FEATURES = (
@@ -2093,6 +2094,12 @@ def add_features(sender, **kwargs):
20932094
NO_CONFIG_FILE_DEPRECATED,
20942095
_("Build: Building without a configuration file is deprecated."),
20952096
),
2097+
(
2098+
BUILD_IMAGE_CONFIG_KEY_DEPRECATED,
2099+
_(
2100+
'Build: Building using "build.image" in the configuration file is deprecated.'
2101+
),
2102+
),
20962103
(
20972104
SCALE_IN_PROTECTION,
20982105
_("Build: Set scale-in protection before/after building."),

‎readthedocs/projects/tasks/utils.py

+156
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,162 @@ def deprecated_config_file_used_notification():
323323
)
324324

325325

326+
class DeprecatedBuildImageSiteNotification(SiteNotification):
327+
failure_message = _(
328+
'Your project(s) "{{ project_slugs }}" are using the deprecated "build.image" '
329+
'config on their ".readthedocs.yaml" file. '
330+
'This config is deprecated in favor of "build.os" and <strong>will be removed on October 16, 2023</strong>. ' # noqa
331+
'<a href="https://blog.readthedocs.com/build-image-config-deprecated/">Read our blog post to migrate to "build.os"</a> ' # noqa
332+
"and ensure your project continues building successfully."
333+
)
334+
failure_level = WARNING_PERSISTENT
335+
336+
337+
class DeprecatedBuildImageEmailNotification(Notification):
338+
app_templates = "projects"
339+
name = "deprecated_build_image_used"
340+
subject = '[Action required] Update your ".readthedocs.yaml" file to use "build.os"'
341+
level = REQUIREMENT
342+
343+
def send(self):
344+
"""Method overwritten to remove on-site backend."""
345+
backend = EmailBackend(self.request)
346+
backend.send(self)
347+
348+
349+
@app.task(queue="web")
350+
def deprecated_build_image_notification():
351+
"""
352+
Send an email notification about using "build.image" to all maintainers of the project.
353+
354+
This is a scheduled task to be executed on the webs.
355+
Note the code uses `.iterator` and `.only` to avoid killing the db with this query.
356+
Besdies, it excludes projects with enough spam score to be skipped.
357+
"""
358+
# Skip projects with a spam score bigger than this value.
359+
# Currently, this gives us ~250k in total (from ~550k we have in our database)
360+
spam_score = 300
361+
362+
projects = set()
363+
start_datetime = datetime.datetime.now()
364+
queryset = Project.objects.exclude(users__profile__banned=True)
365+
if settings.ALLOW_PRIVATE_REPOS:
366+
# Only send emails to active customers
367+
queryset = queryset.filter(
368+
organizations__stripe_subscription__status=SubscriptionStatus.active
369+
)
370+
else:
371+
# Take into account spam score on community
372+
queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter(
373+
Q(spam_score__lt=spam_score) | Q(is_spam=False)
374+
)
375+
queryset = queryset.only("slug", "default_version").order_by("id")
376+
n_projects = queryset.count()
377+
378+
for i, project in enumerate(queryset.iterator()):
379+
if i % 500 == 0:
380+
log.info(
381+
'Finding projects using "build.image" config key.',
382+
progress=f"{i}/{n_projects}",
383+
current_project_pk=project.pk,
384+
current_project_slug=project.slug,
385+
projects_found=len(projects),
386+
time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
387+
)
388+
389+
# Only check for the default version because if the project is using tags
390+
# they won't be able to update those and we will send them emails forever.
391+
# We can update this query if we consider later.
392+
version = (
393+
project.versions.filter(slug=project.default_version).only("id").first()
394+
)
395+
if version:
396+
# Use a fixed date here to avoid changing the date on each run
397+
years_ago = timezone.datetime(2022, 8, 1)
398+
build = (
399+
version.builds.filter(success=True, date__gt=years_ago)
400+
.only("_config")
401+
.order_by("-date")
402+
.first()
403+
)
404+
# TODO: uncomment this line before merging
405+
# if build and build.deprecated_build_image_used():
406+
if build and "image" in build.config.get("build", {}):
407+
projects.add(project.slug)
408+
409+
# Store all the users we want to contact
410+
users = set()
411+
412+
n_projects = len(projects)
413+
queryset = Project.objects.filter(slug__in=projects).order_by("id")
414+
for i, project in enumerate(queryset.iterator()):
415+
if i % 500 == 0:
416+
log.info(
417+
'Querying all the users we want to contact about "build.image" deprecation.',
418+
progress=f"{i}/{n_projects}",
419+
current_project_pk=project.pk,
420+
current_project_slug=project.slug,
421+
users_found=len(users),
422+
time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
423+
)
424+
425+
users.update(AdminPermission.owners(project).values_list("username", flat=True))
426+
427+
# Only send 1 email per user,
428+
# even if that user has multiple projects using "build.image".
429+
# The notification will mention all the projects.
430+
queryset = User.objects.filter(
431+
username__in=users,
432+
profile__banned=False,
433+
profile__optout_email_build_image_deprecation=False,
434+
).order_by("id")
435+
436+
n_users = queryset.count()
437+
for i, user in enumerate(queryset.iterator()):
438+
if i % 500 == 0:
439+
log.info(
440+
'Sending deprecated "build.image" config key notification to users.',
441+
progress=f"{i}/{n_users}",
442+
current_user_pk=user.pk,
443+
current_user_username=user.username,
444+
time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
445+
)
446+
447+
# All the projects for this user that are using "build.image".
448+
# Use set() intersection in Python that's pretty quick since we only need the slugs.
449+
# Otherwise we have to pass 82k slugs to the DB query, which makes it pretty slow.
450+
user_projects = AdminPermission.projects(user, admin=True).values_list(
451+
"slug", flat=True
452+
)
453+
user_projects_slugs = list(set(user_projects) & projects)
454+
user_projects = Project.objects.filter(slug__in=user_projects_slugs)
455+
456+
# Create slug string for onsite notification
457+
user_project_slugs = ", ".join(user_projects_slugs[:5])
458+
if len(user_projects) > 5:
459+
user_project_slugs += " and others..."
460+
461+
n_site = DeprecatedBuildImageSiteNotification(
462+
user=user,
463+
context_object=user,
464+
extra_context={"project_slugs": user_project_slugs},
465+
success=False,
466+
)
467+
n_site.send()
468+
469+
n_email = DeprecatedBuildImageEmailNotification(
470+
user=user,
471+
context_object=user,
472+
extra_context={"projects": user_projects},
473+
)
474+
n_email.send()
475+
476+
log.info(
477+
'Finish sending deprecated "build.image" config key notifications.',
478+
time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
479+
)
480+
481+
326482
@app.task(queue="web")
327483
def set_builder_scale_in_protection(instance, protected_from_scale_in):
328484
"""

‎readthedocs/settings/base.py

+5
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,11 @@ def TEMPLATES(self):
536536
'schedule': crontab(day_of_week='wednesday', hour=11, minute=15),
537537
'options': {'queue': 'web'},
538538
},
539+
'weekly-build-image-notification': {
540+
'task': 'readthedocs.projects.tasks.utils.deprecated_build_image_notification',
541+
'schedule': crontab(day_of_week='wednesday', hour=9, minute=15),
542+
'options': {'queue': 'web'},
543+
},
539544
}
540545

541546
# Sentry

‎readthedocs/templates/builds/build_detail.html

+19-7
Original file line numberDiff line numberDiff line change
@@ -163,17 +163,29 @@
163163
{% endif %}
164164

165165
{# This message is not dynamic and only appears when loading the page after the build has finished #}
166-
{% if build.finished and build.deprecated_config_used %}
167-
<div class="build-ideas">
166+
{% if build.finished and build.deprecated_build_image_used %}
167+
<div class="build-ideas">
168168
<p>
169-
{% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %}
169+
{% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/build-image-config-deprecated/" %}
170170
<strong>Your builds will stop working soon!</strong><br/>
171-
Configuration files will <strong>soon be required</strong> by projects, and will no longer be optional.
172-
<a href="{{ config_file_link }}">Read our blog post to create one</a>
171+
"build.image" config key is deprecated and it will be removed soon.
172+
<a href="{{ config_file_link }}">Read our blog post to know how to migrate to new key "build.os"</a>
173173
and ensure your project continues building successfully.
174-
{% endblocktrans %}
174+
{% endblocktrans %}
175175
</p>
176-
</div>
176+
</div>
177+
{% endif %}
178+
{% if build.finished and build.deprecated_config_used %}
179+
<div class="build-ideas">
180+
<p>
181+
{% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %}
182+
<strong>Your builds will stop working soon!</strong><br/>
183+
Configuration files will <strong>soon be required</strong> by projects, and will no longer be optional.
184+
<a href="{{ config_file_link }}">Read our blog post to create one</a>
185+
and ensure your project continues building successfully.
186+
{% endblocktrans %}
187+
</p>
188+
</div>
177189
{% endif %}
178190

179191
{% endif %}

‎readthedocs/templates/projects/notifications/deprecated_build_image_used_email.html

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{# TODO: copy the text from the TXT version once we agree on its content #}

‎readthedocs/templates/projects/notifications/deprecated_build_image_used_email.txt

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{% extends "core/email/common.txt" %}
2+
{% block content %}
3+
The Read the Docs build system is deprecating "build.image" config key on ".readthedocs.yaml" starting on October 16, 2023.
4+
We are sending weekly notifications about this issue to all impacted users,
5+
as well as temporary build failures (brownouts) as the date approaches for those who haven't migrated their projects.
6+
7+
The timeline for this deprecation is as follows:
8+
9+
* Monday, August 28, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours: 00:01 PST to 11:59 PST (noon)
10+
* Monday, September 18, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours: 00:01 PST to 23:59 PST (midnight)
11+
* Monday, October 2, 2023: Do a third and final brownout (temporarily enforce this deprecation) for 48 hours: 00:01 PST to October 3, 2023 23:59 PST (midnight)
12+
* Monday, October 16, 2023: Fully remove support for building documentation using "build.image" on the configuration file
13+
14+
We have identified that the following projects which you maintain, and were built in the last year, are impacted by this deprecation:
15+
16+
{% for project in projects|slice:":15" %}
17+
* {{ project.slug }} ({{ production_uri }}{{ project.get_absolute_url }})
18+
{% endfor %}
19+
{% if projects.count > 15 %}
20+
* ... and {{ projects.count|add:"-15" }} more projects.
21+
{% endif %}
22+
23+
Please use "build.os" on your configuration file to ensure that they continue building successfully and to stop receiving these notifications.
24+
If you want to opt-out from these emails, you can edit your preferences in your account settings, at https://readthedocs.org/accounts/edit/.
25+
26+
For more information on how to use "build.os",
27+
read our blog post at https://blog.readthedocs.com/build-image-config-deprecated/
28+
29+
Get in touch with us via our support ({{ production_uri }}{% url 'support' %})
30+
and let us know if you are unable to use a configuration file for any reason.
31+
{% endblock %}

0 commit comments

Comments
 (0)