Save config on build model

readthedocs/builds/migrations/0005_add_config_field.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.13 on 2018-10-22 19:37
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import jsonfield.fields
+import readthedocs.builds.version_slug
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('builds', '0004_add-apiversion-proxy-model'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='build',
+            name='config',
+            field=jsonfield.fields.JSONField(default=dict, verbose_name='Configuration used in the build'),
+        ),
+        migrations.AlterField(
+            model_name='build',
+            name='error',
+            field=models.TextField(blank=True, default='', verbose_name='Error'),
+        ),
+        migrations.AlterField(
+            model_name='build',
+            name='output',
+            field=models.TextField(blank=True, default='', verbose_name='Output'),
+        ),
+        migrations.AlterField(
+            model_name='build',
+            name='state',
+            field=models.CharField(choices=[('triggered', 'Triggered'), ('cloning', 'Cloning'), ('installing', 'Installing'), ('building', 'Building'), ('finished', 'Finished')], default='finished', max_length=55, verbose_name='State'),
+        ),
+        migrations.AlterField(
+            model_name='build',
+            name='type',
+            field=models.CharField(choices=[('html', 'HTML'), ('pdf', 'PDF'), ('epub', 'Epub'), ('man', 'Manpage'), ('dash', 'Dash')], default='html', max_length=55, verbose_name='Type'),
+        ),
+        migrations.AlterField(
+            model_name='version',
+            name='privacy_level',
+            field=models.CharField(choices=[('public', 'Public'), ('protected', 'Protected'), ('private', 'Private')], default='public', help_text='Level of privacy for this Version.', max_length=20, verbose_name='Privacy Level'),
+        ),
+        migrations.AlterField(
+            model_name='version',
+            name='slug',
+            field=readthedocs.builds.version_slug.VersionSlugField(db_index=True, max_length=255, populate_from='verbose_name', verbose_name='Slug'),
+        ),
+        migrations.AlterField(
+            model_name='version',
+            name='type',
+            field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'),
+        ),
+        migrations.AlterField(
+            model_name='versionalias',
+            name='from_slug',
+            field=models.CharField(default='', max_length=255, verbose_name='From slug'),
+        ),
+        migrations.AlterField(
+            model_name='versionalias',
+            name='to_slug',
+            field=models.CharField(blank=True, default='', max_length=255, verbose_name='To slug'),
+        ),
+    ]

readthedocs/builds/models.py

@@ -2,36 +2,57 @@
 """Models for the builds app."""
 
 from __future__ import (
-    absolute_import, division, print_function, unicode_literals)
+    absolute_import,
+    division,
+    print_function,
+    unicode_literals,
+)
 
 import logging
 import os.path
 import re
-from builtins import object
 from shutil import rmtree
 
+from builtins import object
 from django.conf import settings
 from django.core.urlresolvers import reverse
 from django.db import models
+from django.utils import timezone
 from django.utils.encoding import python_2_unicode_compatible
-from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext
+from django.utils.translation import ugettext_lazy as _
 from guardian.shortcuts import assign
+from jsonfield import JSONField
 from taggit.managers import TaggableManager
 
 from readthedocs.core.utils import broadcast
 from readthedocs.projects.constants import (
-    BITBUCKET_URL, GITHUB_URL, GITLAB_URL, PRIVACY_CHOICES, PRIVATE)
+    BITBUCKET_URL,
+    GITHUB_URL,
+    GITLAB_URL,
+    PRIVACY_CHOICES,
+    PRIVATE,
+)
 from readthedocs.projects.models import APIProject, Project
 
 from .constants import (
-    BRANCH, BUILD_STATE, BUILD_STATE_FINISHED, BUILD_TYPES, LATEST,
-    NON_REPOSITORY_VERSIONS, STABLE, TAG, VERSION_TYPES)
+    BRANCH,
+    BUILD_STATE,
+    BUILD_STATE_FINISHED,
+    BUILD_TYPES,
+    LATEST,
+    NON_REPOSITORY_VERSIONS,
+    STABLE,
+    TAG,
+    VERSION_TYPES,
+)
 from .managers import VersionManager
 from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet
 from .utils import (
-    get_bitbucket_username_repo, get_github_username_repo,
-    get_gitlab_username_repo)
+    get_bitbucket_username_repo,
+    get_github_username_repo,
+    get_gitlab_username_repo,
+)
 from .version_slug import VersionSlugField
 
 DEFAULT_VERSION_PRIVACY_LEVEL = getattr(
@@ -109,6 +130,16 @@ def __str__(self):
                 pk=self.pk,
             ))
 
+    @property
+    def config(self):
+        """Returns the configuration used in the last successful build."""
+        last_build = (
+            self.builds.filter(state='finished', success=True)
+            .order_by('-date')
+            .first()
+        )
+        return last_build.get_config()
+
     @property
     def commit_name(self):
         """
@@ -448,6 +479,7 @@ class Build(models.Model):
     exit_code = models.IntegerField(_('Exit code'), null=True, blank=True)
     commit = models.CharField(
         _('Commit'), max_length=255, null=True, blank=True)
+    config = JSONField(_('Configuration used in the build'), default=dict)
 
     length = models.IntegerField(_('Build Length'), null=True, blank=True)
 
@@ -466,6 +498,47 @@ class Meta(object):
         get_latest_by = 'date'
         index_together = [['version', 'state', 'type']]
 
+    @property
+    def previous(self):
+        """Returns the previous build to this one."""
+        date = self.date or timezone.now()
+        if self.project is not None and self.version is not None:
+            return (
+                Build.objects
+                .filter(
+                    project=self.project,
+                    version=self.version,
+                    date__lt=date,
+                )
+                .order_by('-date')
+                .first()
+            )
+        return None
+
+    def get_config(self):
+        """Get the config from correct object."""
+        if '__config' in self.config:
+            return Build.objects.get(pk=self.config['__config']).config
+        return self.config
+
+    def save(self, *args, **kwargs):  # noqa
+        """
+        Save object.
+
+        To save space on the db we only save the config if it's different
+        from the previous one.
+
+        If the config is the same, we save the pk of the object
+        that has the **real** config under the ``__config`` key.
+        """
+        previous = self.previous
+        if previous is not None and self.config == previous.get_config():
+            previous_pk = previous.pk
+            if '__config' in previous.config:
+                previous_pk = previous.config['__config']
+            self.config = {'__config': previous_pk}
+        super(Build, self).save(*args, **kwargs)
+
     def __str__(self):
         return ugettext(
             'Build {project} for {usernames} ({pk})'.format(

readthedocs/config/config.py

@@ -233,6 +233,21 @@ def python_full_version(self):
             )
         return ver
 
+    def as_dict(self):
+        attributes = [
+            'version', 'formats', 'python',
+            'conda', 'build', 'doctype',
+            'sphinx', 'mkdocs', 'submodules',
+        ]
+        config = {}
+        for name in attributes:
+            attr = getattr(self, name)
+            if hasattr(attr, '_asdict'):
+                config[name] = attr._asdict()
+            else:
+                config[name] = attr
+        return config
+
     def __getattr__(self, name):
         """Raise an error for unknown attributes."""
         raise ConfigOptionNotSupportedError(name)

readthedocs/config/tests/test_config.py

@@ -841,6 +841,60 @@ def test_config_filenames_regex(correct_config_filename):
     assert re.match(CONFIG_FILENAME_REGEX, correct_config_filename)
 
 
+def test_as_dict(tmpdir):
+    apply_fs(tmpdir, {'requirements.txt': ''})
+    build = get_build_config(
+        {
+            'version': 1,
+            'formats': ['pdf'],
+            'python': {
+                'version': 3.5,
+            },
+            'requirements_file': 'requirements.txt',
+        },
+        get_env_config({
+            'defaults': {
+                'doctype': 'sphinx',
+                'sphinx_configuration': None,
+            },
+        }),
+        source_file=str(tmpdir.join('readthedocs.yml')),
+    )
+    build.validate()
+    expected_dict = {
+        'version': '1',
+        'formats': ['pdf'],
+        'python': {
+            'version': 3.5,
+            'requirements': 'requirements.txt',
+            'install_with_pip': False,
+            'install_with_setup': False,
+            'extra_requirements': [],
+            'use_system_site_packages': False,
+        },
+        'build': {
+            'image': 'readthedocs/build:2.0',
+        },
+        'conda': None,
+        'sphinx': {
+            'builder': 'sphinx',
+            'configuration': None,
+            'fail_on_warning': False,
+        },
+        'mkdocs': {
+            'configuration': None,
+            'fail_on_warning': False,
+        },
+        'doctype': 'sphinx',
+        'submodules': {
+            'include': ALL,
+            'exclude': [],
+            'recursive': True,
+        },
+    }
+    assert build.as_dict() == expected_dict
+
+
 class TestBuildConfigV2(object):
 
     def get_build_config(self, config, env_config=None,
@@ -1804,3 +1858,47 @@ def test_pop_config_raise_exception(self):
             build.pop_config('one.four', raise_ex=True)
         assert excinfo.value.value == 'four'
         assert excinfo.value.code == VALUE_NOT_FOUND
+
+    def test_as_dict(self, tmpdir):
+        apply_fs(tmpdir, {'requirements.txt': ''})
+        build = self.get_build_config(
+            {
+                'version': 2,
+                'formats': ['pdf'],
+                'python': {
+                    'version': 3.6,
+                    'requirements': 'requirements.txt',
+                },
+            },
+            source_file=str(tmpdir.join('readthedocs.yml')),
+        )
+        build.validate()
+        expected_dict = {
+            'version': '2',
+            'formats': ['pdf'],
+            'python': {
+                'version': 3.6,
+                'requirements': str(tmpdir.join('requirements.txt')),
+                'install_with_pip': False,
+                'install_with_setup': False,
+                'extra_requirements': [],
+                'use_system_site_packages': False,
+            },
+            'build': {
+                'image': 'readthedocs/build:latest',
+            },
+            'conda': None,
+            'sphinx': {
+                'builder': 'sphinx',
+                'configuration': None,
+                'fail_on_warning': False,
+            },
+            'mkdocs': None,
+            'doctype': 'sphinx',
+            'submodules': {
+                'include': [],
+                'exclude': ALL,
+                'recursive': False,
+            },
+        }
+        assert build.as_dict() == expected_dict

readthedocs/projects/tasks.py

@@ -405,8 +405,10 @@ def run_setup(self, record=True):
                 raise YAMLParseError(
                     YAMLParseError.GENERIC_WITH_PARSE_EXCEPTION.format(
                         exception=str(e),
-                    ))
+                    )
+                )
 
+            self.save_build_config()
             self.additional_vcs_operations()
 
         if self.setup_env.failure or self.config is None:
@@ -529,9 +531,14 @@ def get_build(build_pk):
         build = {}
         if build_pk:
             build = api_v2.build(build_pk).get()
-        return dict((key, val) for (key, val) in list(build.items())
-                    if key not in ['project', 'version', 'resource_uri',
-                                   'absolute_uri'])
+        private_keys = [
+            'project', 'version', 'resource_uri', 'absolute_uri'
+        ]
+        return {
+            key: val
+            for key, val in build.items()
+            if key not in private_keys
+        }
 
     def setup_vcs(self):
         """
@@ -592,6 +599,15 @@ def set_valid_clone(self):
         self.project.has_valid_clone = True
         self.version.project.has_valid_clone = True
 
+    def save_build_config(self):
+        """Save config in the build object."""
+        pk = self.build['id']
+        build_data = api_v2.build(pk).get()
+        config = self.config.as_dict()
+        build_data['config'] = config
+        api_v2.build(pk).put(build_data)
+        self.build['config'] = config
+
     def update_app_instances(self, html=False, localmedia=False, search=False,
                              pdf=False, epub=False):
         """

readthedocs/restapi/serializers.py

@@ -112,6 +112,7 @@ class BuildSerializer(serializers.ModelSerializer):
     version_slug = serializers.ReadOnlyField(source='version.slug')
     docs_url = serializers.ReadOnlyField(source='version.get_absolute_url')
     state_display = serializers.ReadOnlyField(source='get_state_display')
+    config = serializers.JSONField(required=False)
 
     class Meta(object):
         model = Build