Collect build data (#9113)

Author: stsewd

readthedocs/api/v2/views/model_views.py

@@ -1,8 +1,8 @@
 """Endpoints for listing Projects, Versions, Builds, etc."""
 
 import json
-import structlog
 
+import structlog
 from allauth.socialaccount.models import SocialAccount
 from django.conf import settings
 from django.db.models import BooleanField, Case, Value, When

readthedocs/builds/admin.py

@@ -1,16 +1,7 @@
 """Django admin interface for `~builds.models.Build` and related models."""
 
-import json
 from django.contrib import admin, messages
-from django.utils.safestring import mark_safe
-from polymorphic.admin import (
-    PolymorphicChildModelAdmin,
-    PolymorphicParentModelAdmin,
-)
-
-from pygments import highlight
-from pygments.lexers import JsonLexer
-from pygments.formatters import HtmlFormatter
+from polymorphic.admin import PolymorphicChildModelAdmin, PolymorphicParentModelAdmin
 
 from readthedocs.builds.models import (
     Build,
@@ -20,33 +11,11 @@
     VersionAutomationRule,
 )
 from readthedocs.core.utils import trigger_build
+from readthedocs.core.utils.admin import pretty_json_field
 from readthedocs.projects.models import HTMLFile
 from readthedocs.search.utils import _indexing_helper
 
 
-def _pretty_config(instance):
-    """
-    Function to display pretty version of our data.
-
-    Thanks to PyDanny: https://www.pydanny.com/pretty-formatting-json-django-admin.html
-    """
-
-    # Convert the data to sorted, indented JSON
-    response = json.dumps(instance.config, sort_keys=True, indent=2)
-
-    # Get the Pygments formatter
-    formatter = HtmlFormatter()
-
-    # Highlight the data
-    response = highlight(response, JsonLexer(), formatter)
-
-    # Get the stylesheet
-    style = "<style>" + formatter.get_style_defs() + "</style><br>"
-
-    # Safe the output
-    return mark_safe(style + response)
-
-
 class BuildCommandResultInline(admin.TabularInline):
     model = BuildCommandResult
     fields = ('command', 'exit_code', 'output')
@@ -96,7 +65,7 @@ def version_slug(self, obj):
         return obj.version.slug
 
     def pretty_config(self, instance):
-        return _pretty_config(instance)
+        return pretty_json_field(instance, "config")
 
     pretty_config.short_description = 'Config File'
 
@@ -123,7 +92,7 @@ def project_slug(self, obj):
         return obj.project.slug
 
     def pretty_config(self, instance):
-        return _pretty_config(instance)
+        return pretty_json_field(instance, "config")
 
     pretty_config.short_description = 'Config File'
 

readthedocs/config/config.py

@@ -180,6 +180,7 @@ class BuildConfigBase:
     def __init__(self, env_config, raw_config, source_file):
         self.env_config = env_config
         self._raw_config = copy.deepcopy(raw_config)
+        self.source_config = copy.deepcopy(raw_config)
         self.source_file = source_file
         if os.path.isdir(self.source_file):
             self.base_path = self.source_file

readthedocs/core/db.py

@@ -0,0 +1,36 @@
+"""
+Custom database routers.
+
+https://docs.djangoproject.com/en/4.0/topics/db/multi-db/#automatic-database-routing
+"""
+
+from collections import defaultdict
+
+
+class MapAppsRouter:
+
+    """
+    Router to map Django applications to a specific database.
+
+    :py:attr:`apps_to_db` is used to map an application to a database,
+    if an application isn't listed here, it will use the ``default`` database.
+    """
+
+    def __init__(self):
+        self.apps_to_db = defaultdict(lambda: "default")
+        self.apps_to_db.update({"telemetry": "telemetry"})
+
+    def db_for_read(self, model, **hints):
+        return self.apps_to_db[model._meta.app_label]
+
+    def db_for_write(self, model, **hints):
+        return self.apps_to_db[model._meta.app_label]
+
+    def allow_relation(self, obj1, obj2, **hints):
+        return (
+            self.apps_to_db[obj1._meta.app_label]
+            == self.apps_to_db[obj2._meta.app_label]
+        )
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        return self.apps_to_db[app_label] == db

readthedocs/core/utils/admin.py

@@ -0,0 +1,28 @@
+import json
+
+from django.utils.safestring import mark_safe
+from pygments import highlight
+from pygments.formatters import HtmlFormatter
+from pygments.lexers import JsonLexer
+
+
+def pretty_json_field(instance, field):
+    """
+    Display a pretty version of a JSON field in the admin.
+
+    Thanks to PyDanny: https://www.pydanny.com/pretty-formatting-json-django-admin.html
+    """
+    # Convert the data to sorted, indented JSON
+    response = json.dumps(getattr(instance, field), sort_keys=True, indent=2)
+
+    # Get the Pygments formatter
+    formatter = HtmlFormatter()
+
+    # Highlight the data
+    response = highlight(response, JsonLexer(), formatter)
+
+    # Get the stylesheet
+    style = "<style>" + formatter.get_style_defs() + "</style><br>"
+
+    # Safe the output
+    return mark_safe(style + response)

readthedocs/projects/tasks/builds.py

@@ -46,6 +46,8 @@
     YAMLParseError,
 )
 from readthedocs.storage import build_media_storage
+from readthedocs.telemetry.collectors import BuildDataCollector
+from readthedocs.telemetry.tasks import save_build_data
 from readthedocs.worker import app
 
 from ..exceptions import (
@@ -346,6 +348,8 @@ def before_start(self, task_id, args, kwargs):
         # Reset any previous build error reported to the user
         self.data.build['error'] = ''
 
+        self.data.build_data = None
+
         # Also note there are builds that are triggered without a commit
         # because they just build the latest commit for that version
         self.data.build_commit = kwargs.get('build_commit')
@@ -540,6 +544,7 @@ def after_return(self, status, retval, task_id, args, kwargs, einfo):
         self.data.build['length'] = (timezone.now() - self.data.start_time).seconds
 
         self.update_build(BUILD_STATE_FINISHED)
+        self.save_build_data()
 
         build_complete.send(sender=Build, build=self.data.build)
 
@@ -595,13 +600,46 @@ def execute(self):
         # ``__exit__``
         self.data.build_director.create_build_environment()
         with self.data.build_director.build_environment:
-            # Installing
-            self.update_build(state=BUILD_STATE_INSTALLING)
-            self.data.build_director.setup_environment()
+            try:
+                # Installing
+                self.update_build(state=BUILD_STATE_INSTALLING)
+                self.data.build_director.setup_environment()
+
+                # Building
+                self.update_build(state=BUILD_STATE_BUILDING)
+                self.data.build_director.build()
+            finally:
+                self.data.build_data = self.collect_build_data()
+
+    def collect_build_data(self):
+        """
+        Collect data from the current build.
 
-            # Building
-            self.update_build(state=BUILD_STATE_BUILDING)
-            self.data.build_director.build()
+        The data is collected from inside the container,
+        so this must be called before killing the container.
+        """
+        try:
+            return BuildDataCollector(
+                self.data.build_director.build_environment
+            ).collect()
+        except Exception:
+            log.exception("Error while collecting build data")
+
+    def save_build_data(self):
+        """
+        Save the data collected from the build after it has ended.
+
+        This must be called after the build has finished updating its state,
+        otherwise some attributes like ``length`` won't be available.
+        """
+        try:
+            if self.data.build_data:
+                save_build_data.delay(
+                    build_id=self.data.build_pk,
+                    data=self.data.build_data,
+                )
+        except Exception:
+            log.exception("Error while saving build data")
 
     @staticmethod
     def get_project(project_pk):

readthedocs/projects/tests/test_build_tasks.py

@@ -17,11 +17,12 @@
 from readthedocs.projects.exceptions import RepositoryError
 from readthedocs.projects.models import EnvironmentVariable, Project, WebHookEvent
 from readthedocs.projects.tasks.builds import sync_repository_task, update_docs_task
+from readthedocs.telemetry.models import BuildData
 
 from .mockers import BuildEnvironmentMocker
 
 
-@pytest.mark.django_db
+@pytest.mark.django_db(databases="__all__")
 class BuildEnvironmentBase:
 
     # NOTE: `load_yaml_config` maybe be moved to the setup and assign to self.
@@ -258,6 +259,8 @@ def test_successful_build(
             }
         )
 
+        assert not BuildData.objects.all().exists()
+
         self._trigger_update_docs_task()
 
         # It has to be called twice, ``before_start`` and ``after_return``
@@ -393,6 +396,8 @@ def test_successful_build(
             "error": "",
         }
 
+        assert BuildData.objects.all().exists()
+
         self.mocker.mocks["build_media_storage"].sync_directory.assert_has_calls(
             [
                 mock.call(mock.ANY, "html/project/latest"),
@@ -418,6 +423,8 @@ def test_failed_build(
         send_external_build_status,
         build_complete,
     ):
+        assert not BuildData.objects.all().exists()
+
         # Force an exception from the execution of the task. We don't really
         # care "where" it was raised: setup, build, syncing directories, etc
         execute.side_effect = Exception('Force and exception here.')
@@ -449,6 +456,10 @@ def test_failed_build(
             build=mock.ANY,
         )
 
+        # The build data is None (we are failing the build before the environment is created)
+        # and the task won't be run.
+        assert not BuildData.objects.all().exists()
+
         # Test we are updating the DB by calling the API with the updated build object
         api_request = self.requests_mock.request_history[
             -1

readthedocs/settings/base.py

@@ -128,6 +128,7 @@ def SESSION_COOKIE_SAMESITE(self):
     DONT_HIT_API = False
     DONT_HIT_DB = True
     RTD_SAVE_BUILD_COMMANDS_TO_STORAGE = False
+    DATABASE_ROUTERS = ['readthedocs.core.db.MapAppsRouter']
 
     USER_MATURITY_DAYS = 7
 
@@ -199,6 +200,7 @@ def INSTALLED_APPS(self):  # noqa
             'readthedocs.sphinx_domains',
             'readthedocs.search',
             'readthedocs.embed',
+            'readthedocs.telemetry',
 
             # allauth
             'allauth',

readthedocs/settings/dev.py

@@ -18,7 +18,11 @@ def DATABASES(self):  # noqa
             'default': {
                 'ENGINE': 'django.db.backends.sqlite3',
                 'NAME': os.path.join(self.SITE_ROOT, 'dev.db'),
-            }
+            },
+            'telemetry': {
+                'ENGINE': 'django.db.backends.sqlite3',
+                'NAME': os.path.join(self.SITE_ROOT, 'telemetry.dev.db'),
+            },
         }
 
     DONT_HIT_DB = False

readthedocs/settings/docker_compose.py

@@ -121,7 +121,15 @@ def DATABASES(self):  # noqa
                 "PASSWORD": os.environ.get("DB_PWD", "docs_pwd"),
                 "HOST": os.environ.get("DB_HOST", "database"),
                 "PORT": "",
-            }
+            },
+            "telemetry": {
+                "ENGINE": "django.db.backends.postgresql_psycopg2",
+                "NAME": "telemetry",
+                "USER": os.environ.get("DB_USER", "docs_user"),
+                "PASSWORD": os.environ.get("DB_PWD", "docs_pwd"),
+                "HOST": os.environ.get("DB_HOST", "database"),
+                "PORT": "",
+            },
         }
 
     def show_debug_toolbar(request):

readthedocs/telemetry/__init__.py

@@ -0,0 +1,22 @@
+"""Telemetry admin."""
+
+
+from django.contrib import admin
+
+from readthedocs.core.utils.admin import pretty_json_field
+from readthedocs.telemetry.models import BuildData
+
+
+@admin.register(BuildData)
+class BuildDataAdmin(admin.ModelAdmin):
+
+    fields = ("created", "modified", "pretty_data")
+    readonly_fields = (
+        "created",
+        "modified",
+        "pretty_data",
+    )
+
+    # pylint: disable=no-self-use
+    def pretty_data(self, instance):
+        return pretty_json_field(instance, "data")

readthedocs/telemetry/admin.py

@@ -0,0 +1,15 @@
+"""
+Telemetry application.
+
+Collect relevant data to be analyzed later.
+"""
+
+from django.apps import AppConfig
+
+
+class TelemetryConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "readthedocs.telemetry"
+
+    def ready(self):
+        import readthedocs.telemetry.tasks  # noqa