Collect build data

Author: stsewd

readthedocs/api/v2/views/model_views.py

@@ -1,8 +1,8 @@
 """Endpoints for listing Projects, Versions, Builds, etc."""
 
 import json
-import structlog
 
+import structlog
 from allauth.socialaccount.models import SocialAccount
 from django.conf import settings
 from django.db.models import BooleanField, Case, Value, When
@@ -19,6 +19,7 @@
 from readthedocs.oauth.services import GitHubService, registry
 from readthedocs.projects.models import Domain, Project
 from readthedocs.storage import build_commands_storage
+from readthedocs.telemetry.models import BuildData
 
 from ..permissions import APIPermission, APIRestrictedPermission, IsOwner
 from ..serializers import (
@@ -285,6 +286,17 @@ def reset(self, request, **kwargs):
         instance.reset()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
+    @decorators.action(
+        detail=True,
+        permission_classes=[permissions.IsAdminUser],
+        methods=["post"],
+    )
+    def telemetry(self, request, **kwargs):
+        """Collect telemetry data from the build."""
+        build = self.get_object()
+        BuildData.objects.collect(build, request.data)
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
 
 class BuildCommandViewSet(DisableListEndpoint, UserSelectViewSet):
     parser_classes = [JSONParser, MultiPartParser]

readthedocs/builds/admin.py

@@ -1,16 +1,7 @@
 """Django admin interface for `~builds.models.Build` and related models."""
 
-import json
 from django.contrib import admin, messages
-from django.utils.safestring import mark_safe
-from polymorphic.admin import (
-    PolymorphicChildModelAdmin,
-    PolymorphicParentModelAdmin,
-)
-
-from pygments import highlight
-from pygments.lexers import JsonLexer
-from pygments.formatters import HtmlFormatter
+from polymorphic.admin import PolymorphicChildModelAdmin, PolymorphicParentModelAdmin
 
 from readthedocs.builds.models import (
     Build,
@@ -20,33 +11,11 @@
     VersionAutomationRule,
 )
 from readthedocs.core.utils import trigger_build
+from readthedocs.core.utils.admin import pretty_json_field
 from readthedocs.projects.models import HTMLFile
 from readthedocs.search.utils import _indexing_helper
 
 
-def _pretty_config(instance):
-    """
-    Function to display pretty version of our data.
-
-    Thanks to PyDanny: https://www.pydanny.com/pretty-formatting-json-django-admin.html
-    """
-
-    # Convert the data to sorted, indented JSON
-    response = json.dumps(instance.config, sort_keys=True, indent=2)
-
-    # Get the Pygments formatter
-    formatter = HtmlFormatter()
-
-    # Highlight the data
-    response = highlight(response, JsonLexer(), formatter)
-
-    # Get the stylesheet
-    style = "<style>" + formatter.get_style_defs() + "</style><br>"
-
-    # Safe the output
-    return mark_safe(style + response)
-
-
 class BuildCommandResultInline(admin.TabularInline):
     model = BuildCommandResult
     fields = ('command', 'exit_code', 'output')
@@ -96,7 +65,7 @@ def version_slug(self, obj):
         return obj.version.slug
 
     def pretty_config(self, instance):
-        return _pretty_config(instance)
+        return pretty_json_field(instance, "config")
 
     pretty_config.short_description = 'Config File'
 
@@ -123,7 +92,7 @@ def project_slug(self, obj):
         return obj.project.slug
 
     def pretty_config(self, instance):
-        return _pretty_config(instance)
+        return pretty_json_field(instance, "config")
 
     pretty_config.short_description = 'Config File'
 

readthedocs/config/config.py

@@ -180,6 +180,7 @@ class BuildConfigBase:
     def __init__(self, env_config, raw_config, source_file):
         self.env_config = env_config
         self._raw_config = copy.deepcopy(raw_config)
+        self.source_config = copy.deepcopy(raw_config)
         self.source_file = source_file
         if os.path.isdir(self.source_file):
             self.base_path = self.source_file

readthedocs/core/utils/admin.py

@@ -0,0 +1,28 @@
+import json
+
+from django.utils.safestring import mark_safe
+from pygments import highlight
+from pygments.formatters import HtmlFormatter
+from pygments.lexers import JsonLexer
+
+
+def pretty_json_field(instance, field):
+    """
+    Display a pretty version of a JSON field in the admin.
+
+    Thanks to PyDanny: https://www.pydanny.com/pretty-formatting-json-django-admin.html
+    """
+    # Convert the data to sorted, indented JSON
+    response = json.dumps(getattr(instance, field), sort_keys=True, indent=2)
+
+    # Get the Pygments formatter
+    formatter = HtmlFormatter()
+
+    # Highlight the data
+    response = highlight(response, JsonLexer(), formatter)
+
+    # Get the stylesheet
+    style = "<style>" + formatter.get_style_defs() + "</style><br>"
+
+    # Safe the output
+    return mark_safe(style + response)

readthedocs/projects/tasks/builds.py

@@ -46,6 +46,7 @@
     YAMLParseError,
 )
 from readthedocs.storage import build_media_storage
+from readthedocs.telemetry.collectors import BuildDataCollector
 from readthedocs.worker import app
 
 from ..exceptions import (
@@ -341,6 +342,8 @@ def before_start(self, task_id, args, kwargs):
         # Reset any previous build error reported to the user
         self.data.build['error'] = ''
 
+        self.data.build_data = None
+
         # Also note there are builds that are triggered without a commit
         # because they just build the latest commit for that version
         self.data.build_commit = kwargs.get('build_commit')
@@ -535,6 +538,7 @@ def after_return(self, status, retval, task_id, args, kwargs, einfo):
         self.data.build['length'] = (timezone.now() - self.data.start_time).seconds
 
         self.update_build(BUILD_STATE_FINISHED)
+        self.upload_build_data()
 
         build_complete.send(sender=Build, build=self.data.build)
 
@@ -590,13 +594,43 @@ def execute(self):
         # ``__exit__``
         self.data.build_director.create_build_environment()
         with self.data.build_director.build_environment:
-            # Installing
-            self.update_build(state=BUILD_STATE_INSTALLING)
-            self.data.build_director.setup_environment()
+            try:
+                # Installing
+                self.update_build(state=BUILD_STATE_INSTALLING)
+                self.data.build_director.setup_environment()
+
+                # Building
+                self.update_build(state=BUILD_STATE_BUILDING)
+                self.data.build_director.build()
+            finally:
+                self.data.build_data = self.collect_build_data()
+
+    def collect_build_data(self):
+        """
+        Collect data from the current build.
+
+        The data is collected from inside the container,
+        to this must be called before killing the container.
+        """
+        try:
+            return BuildDataCollector(
+                self.data.build_director.build_environment
+            ).collect()
+        except Exception:
+            log.exception("Error while collecting build data")
+
+    def upload_build_data(self):
+        """
+        Upload data collected from the build after the build has ended.
 
-            # Building
-            self.update_build(state=BUILD_STATE_BUILDING)
-            self.data.build_director.build()
+        This must be called after the build has finished updating its state,
+        otherwise some attributes like ``length`` won't be available.
+        """
+        try:
+            if self.data.build_data:
+                api_v2.build(self.data.build_pk).telemetry.post(self.data.build_data)
+        except Exception:
+            log.exception("Error while uploading build data")
 
     @staticmethod
     def get_project(project_pk):

readthedocs/settings/base.py

@@ -198,6 +198,7 @@ def INSTALLED_APPS(self):  # noqa
             'readthedocs.sphinx_domains',
             'readthedocs.search',
             'readthedocs.embed',
+            'readthedocs.telemetry',
 
             # allauth
             'allauth',

readthedocs/telemetry/__init__.py

@@ -0,0 +1,18 @@
+"""Telemetry admin."""
+
+
+from django.contrib import admin
+
+from readthedocs.core.utils.admin import pretty_json_field
+from readthedocs.telemetry.models import BuildData
+
+
+@admin.register(BuildData)
+class BuildDataAdmin(admin.ModelAdmin):
+
+    fields = ("pretty_data",)
+    readonly_fields = ("pretty_data",)
+
+    # pylint: disable=no-self-use
+    def pretty_data(self, instance):
+        return pretty_json_field(instance, "data")

readthedocs/telemetry/admin.py

@@ -0,0 +1,12 @@
+"""
+Telemetry application.
+
+Collect relevant data to be analyzed later.
+"""
+
+from django.apps import AppConfig
+
+
+class TelemetryConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "readthedocs.telemetry"