Allow large form posts via multipart encoded forms to command API (#5000)

This temporarily puts this under a feature flag, to make debugging a
little easier, but initial tests show things working as expected. This
allows for a large form post to be chunked up and POST, where we were
noticing a number of different issues with body size before.

Author: agjohnson

readthedocs/doc_builder/environments.py

@@ -17,6 +17,7 @@
 from docker.errors import APIError as DockerAPIError
 from docker.errors import DockerException
 from requests.exceptions import ConnectionError
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 from slumber.exceptions import HttpClientError
 
 from readthedocs.builds.constants import BUILD_STATE_FINISHED
@@ -266,7 +267,25 @@ def save(self):
             'start_time': self.start_time,
             'end_time': self.end_time,
         }
-        api_v2.command.post(data)
+
+        if self.build_env.project.has_feature(Feature.API_LARGE_DATA):
+            # Don't use slumber directly here. Slumber tries to enforce a string,
+            # which will break our multipart encoding here.
+            encoder = MultipartEncoder(
+                {key: str(value) for key, value in data.items()}
+            )
+            resource = api_v2.command
+            resp = resource._store['session'].post(
+                resource._store['base_url'] + '/',
+                data=encoder,
+                headers={
+                    'Content-Type': encoder.content_type,
+                }
+            )
+            log.info('Post response via multipart form: %s', resp)
+        else:
+            resp = api_v2.command.post(data)
+            log.info('Post response via JSON encoded data: %s', resp)
 
 
 class DockerBuildCommand(BuildCommand):

readthedocs/projects/models.py

@@ -1239,6 +1239,7 @@ def add_features(sender, **kwargs):
     DONT_OVERWRITE_SPHINX_CONTEXT = 'dont_overwrite_sphinx_context'
     ALLOW_V2_CONFIG_FILE = 'allow_v2_config_file'
     MKDOCS_THEME_RTD = 'mkdocs_theme_rtd'
+    API_LARGE_DATA = 'api_large_data'
     DONT_SHALLOW_CLONE = 'dont_shallow_clone'
     USE_TESTING_BUILD_IMAGE = 'use_testing_build_image'
 
@@ -1276,6 +1277,7 @@ def add_features(sender, **kwargs):
                 'Use Docker image labelled as `testing` to build the docs',
             ),
         ),
+        (API_LARGE_DATA, _('Try alternative method of posting large data'))
     )
 
     projects = models.ManyToManyField(

readthedocs/restapi/views/model_views.py

@@ -8,6 +8,7 @@
 from django.shortcuts import get_object_or_404
 from django.template.loader import render_to_string
 from rest_framework import decorators, permissions, status, viewsets
+from rest_framework.parsers import MultiPartParser, JSONParser, FormParser
 from rest_framework.renderers import BaseRenderer, JSONRenderer
 from rest_framework.response import Response
 
@@ -277,6 +278,7 @@ class BuildViewSet(SettingsOverrideObject):
 
 
 class BuildCommandViewSet(UserSelectViewSet):
+    parser_classes = [JSONParser, MultiPartParser]
     permission_classes = [APIRestrictedPermission]
     renderer_classes = (JSONRenderer,)
     serializer_class = BuildCommandSerializer