Addons: allow users to opt-in into the beta addons (#10733)

* Addons: allow users to opt-in into the beta addons

I created the URL, models, view and form to prepare ourselves for the beta
addons. Currently, if the `Project.addons` fields is not null we add a HTTP
header to tell Cloudflare to enforce the new beta addons on this project.

In the future, we can use each of the `ProjectAddonsConfig` field to let the
user decide which of the addons they want to enable/disable and change their
configurations.

* Related: readthedocs/ext-theme#212

* Changes from feedback

Use a `AddonsConfig.enabled` field to decide whether or not all the addons are
enabled.

* Docstring to document the HTTP headers to communicate with CF

* Addons: update modelling and queryset

* NGINX: pass the `X-RTD-Force-Addons` HTTP header

* Declare the URL only when ext-theme is enabled

Author: humitos

dockerfiles/nginx/proxito.conf.template

@@ -93,6 +93,8 @@ server {
         # Now, I'm injecting it in all the NGINX responses because `sub_filter` is not allowed inside an `if` statement.
         set $rtd_hosting_integrations $upstream_http_x_rtd_hosting_integrations;
         add_header X-RTD-Hosting-Integrations $rtd_hosting_integrations always;
+        set $rtd_force_addons $upstream_http_x_rtd_force_addons;
+        add_header X-RTD-Force-Addons $rtd_force_addons always;
 
         # Inject our own script dynamically
         # TODO: find a way to make this work _without_ running `npm run dev` from the `addons` repository

readthedocs/projects/forms.py

@@ -22,6 +22,7 @@
 from readthedocs.invitations.models import Invitation
 from readthedocs.oauth.models import RemoteRepository
 from readthedocs.projects.models import (
+    AddonsConfig,
     Domain,
     EmailHook,
     EnvironmentVariable,
@@ -493,6 +494,30 @@ def clean_alias(self):
         return alias
 
 
+class AddonsConfigForm(forms.ModelForm):
+
+    """Form to opt-in into new beta addons."""
+
+    project = forms.CharField(widget=forms.HiddenInput(), required=False)
+
+    class Meta:
+        model = AddonsConfig
+        fields = ("enabled", "project")
+
+    def __init__(self, *args, **kwargs):
+        self.project = kwargs.pop("project", None)
+        kwargs["instance"] = getattr(self.project, "addons", None)
+        super().__init__(*args, **kwargs)
+
+        try:
+            self.fields["enabled"].initial = self.project.addons.enabled
+        except AddonsConfig.DoesNotExist:
+            self.fields["enabled"].initial = False
+
+    def clean_project(self):
+        return self.project
+
+
 class UserForm(forms.Form):
 
     """Project owners form."""

readthedocs/projects/migrations/0106_add_addons_config.py

@@ -0,0 +1,121 @@
+# Generated by Django 4.2.5 on 2023-09-18 11:57
+
+import django.db.models.deletion
+import django_extensions.db.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("projects", "0105_remove_project_urlconf"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="AddonsConfig",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "enabled",
+                    models.BooleanField(
+                        default=True,
+                        help_text="Enable/Disable all the addons on this project",
+                    ),
+                ),
+                ("analytics_enabled", models.BooleanField(default=True)),
+                ("doc_diff_enabled", models.BooleanField(default=True)),
+                ("doc_diff_show_additions", models.BooleanField(default=True)),
+                ("doc_diff_show_deletions", models.BooleanField(default=True)),
+                (
+                    "doc_diff_root_selector",
+                    models.CharField(blank=True, max_length=128, null=True),
+                ),
+                ("external_version_warning_enabled", models.BooleanField(default=True)),
+                ("ethicalads_enabled", models.BooleanField(default=True)),
+                ("flyout_enabled", models.BooleanField(default=True)),
+                ("hotkeys_enabled", models.BooleanField(default=True)),
+                ("search_enabled", models.BooleanField(default=True)),
+                (
+                    "search_default_filter",
+                    models.CharField(blank=True, max_length=128, null=True),
+                ),
+                (
+                    "stable_latest_version_warning_enabled",
+                    models.BooleanField(default=True),
+                ),
+                (
+                    "project",
+                    models.OneToOneField(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="addons",
+                        to="projects.project",
+                    ),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="AddonSearchFilter",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("name", models.CharField(max_length=128)),
+                ("syntaxt", models.CharField(max_length=128)),
+                (
+                    "addons",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="projects.addonsconfig",
+                    ),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+    ]

readthedocs/projects/models.py

@@ -123,6 +123,72 @@ def subproject_prefix(self):
         return unsafe_join_url_path(prefix, self.alias, "/")
 
 
+class AddonsConfig(TimeStampedModel):
+
+    """
+    Addons project configuration.
+
+    Store all the configuration for each of the addons.
+    Everything is enabled by default.
+    """
+
+    DOC_DIFF_DEFAULT_ROOT_SELECTOR = "[role=main]"
+
+    project = models.OneToOneField(
+        "Project",
+        related_name="addons",
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE,
+    )
+
+    enabled = models.BooleanField(
+        default=True,
+        help_text="Enable/Disable all the addons on this project",
+    )
+
+    # Analytics
+    analytics_enabled = models.BooleanField(default=True)
+
+    # Docdiff
+    doc_diff_enabled = models.BooleanField(default=True)
+    doc_diff_show_additions = models.BooleanField(default=True)
+    doc_diff_show_deletions = models.BooleanField(default=True)
+    doc_diff_root_selector = models.CharField(null=True, blank=True, max_length=128)
+
+    # External version warning
+    external_version_warning_enabled = models.BooleanField(default=True)
+
+    # EthicalAds
+    ethicalads_enabled = models.BooleanField(default=True)
+
+    # Flyout
+    flyout_enabled = models.BooleanField(default=True)
+
+    # Hotkeys
+    hotkeys_enabled = models.BooleanField(default=True)
+
+    # Search
+    search_enabled = models.BooleanField(default=True)
+    search_default_filter = models.CharField(null=True, blank=True, max_length=128)
+
+    # Stable/Latest version warning
+    stable_latest_version_warning_enabled = models.BooleanField(default=True)
+
+
+class AddonSearchFilter(TimeStampedModel):
+
+    """
+    Addon search user defined filter.
+
+    Specific filter defined by the user to show on the search modal.
+    """
+
+    addons = models.ForeignKey("AddonsConfig", on_delete=models.CASCADE)
+    name = models.CharField(max_length=128)
+    syntaxt = models.CharField(max_length=128)
+
+
 class Project(models.Model):
 
     """Project model."""

readthedocs/projects/urls/private.py

@@ -9,6 +9,7 @@
 from readthedocs.projects.backends.views import ImportWizardView
 from readthedocs.projects.views import private
 from readthedocs.projects.views.private import (
+    AddonsConfigUpdate,
     AutomationRuleDelete,
     AutomationRuleList,
     AutomationRuleMove,
@@ -204,6 +205,18 @@
 
 urlpatterns += domain_urls
 
+# We are allowing users to enable the new beta addons only from the new dashboard
+if settings.RTD_EXT_THEME_ENABLED:
+    addons_urls = [
+        re_path(
+            r"^(?P<project_slug>[-\w]+)/addons/edit/$$",
+            AddonsConfigUpdate.as_view(),
+            name="projects_addons",
+        ),
+    ]
+
+    urlpatterns += addons_urls
+
 integration_urls = [
     re_path(
         r"^(?P<project_slug>{project_slug})/integrations/$".format(**pattern_opts),

readthedocs/projects/views/private.py

@@ -46,6 +46,7 @@
 from readthedocs.oauth.utils import update_webhook
 from readthedocs.projects.filters import ProjectListFilterSet
 from readthedocs.projects.forms import (
+    AddonsConfigForm,
     DomainForm,
     EmailHookForm,
     EnvironmentVariableForm,
@@ -168,6 +169,15 @@ def get_success_url(self):
         return reverse('projects_detail', args=[self.object.slug])
 
 
+class AddonsConfigUpdate(ProjectAdminMixin, PrivateViewMixin, CreateView, UpdateView):
+    form_class = AddonsConfigForm
+    success_message = _("Project addons updated")
+    template_name = "projects/addons_form.html"
+
+    def get_success_url(self):
+        return reverse("projects_addons", args=[self.object.project.slug])
+
+
 class ProjectDelete(UpdateChangeReasonPostView, ProjectMixin, DeleteView):
 
     success_message = _('Project deleted')

readthedocs/proxito/middleware.py

@@ -25,6 +25,7 @@
     unresolver,
 )
 from readthedocs.core.utils import get_cache_tag
+from readthedocs.projects.models import Project
 from readthedocs.proxito.cache import add_cache_tags, cache_response, private_response
 from readthedocs.proxito.redirects import redirect_to_https
 
@@ -265,16 +266,42 @@ def process_request(self, request):  # noqa
         return None
 
     def add_hosting_integrations_headers(self, request, response):
+        """
+        Add HTTP headers to communicate to Cloudflare Workers.
+
+        We have configured Cloudflare Workers to inject the addons and remove
+        the old flyout integration based on HTTP headers.
+        This method uses two different headers for these purposes:
+
+        - ``X-RTD-Hosting-Integrations``: inject ``readthedocs-addons.js`` to enable addons.
+          Enabled by default on projects using ``build.commands``.
+        - ``X-RTD-Force-Addons``: inject ``readthedocs-addons.js``
+          and remove old flyout integration (via ``readthedocs-doc-embed.js``).
+          Enabled only on projects that opted-in via the admin settings.
+
+        Note these headers will not be required anymore eventually
+        since all the project will be using the new addons once we fully roll them out.
+        """
         addons = False
         project_slug = getattr(request, "path_project_slug", "")
         version_slug = getattr(request, "path_version_slug", "")
 
-        if project_slug and version_slug:
-            addons = Version.objects.filter(
-                project__slug=project_slug,
-                slug=version_slug,
-                addons=True,
+        if project_slug:
+            force_addons = Project.objects.filter(
+                slug=project_slug,
+                addons__enabled=True,
             ).exists()
+            if force_addons:
+                response["X-RTD-Force-Addons"] = "true"
+                return
+
+            if version_slug:
+                addons = Version.objects.filter(
+                    project__slug=project_slug,
+                    slug=version_slug,
+                    addons=True,
+                ).exists()
+
             if addons:
                 response["X-RTD-Hosting-Integrations"] = "true"
 

readthedocs/proxito/views/hosting.py

@@ -239,6 +239,7 @@ def _v0(self, project, version, build, filename, user):
             project.translations.all().only("language").order_by("language")
         )
         # Make one DB query here and then check on Python code
+        # TODO: make usage of ``Project.addons.<name>_enabled`` to decide if enabled
         project_features = project.features.all().values_list("feature_id", flat=True)
 
         data = {