diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index 25443f05322..6a2dd4d71f6 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -440,7 +440,7 @@ class BuildEnvironment(BaseEnvironment): ProjectBuildsSkippedError, YAMLParseError, BuildTimeoutError, - MkDocsYAMLParseError + MkDocsYAMLParseError, ) def __init__(self, project=None, version=None, build=None, config=None, @@ -466,7 +466,7 @@ def __exit__(self, exc_type, exc_value, tb): project=self.project.slug, version=self.version.slug, msg='Build finished', - ) + ), ) return ret diff --git a/readthedocs/projects/admin.py b/readthedocs/projects/admin.py index 35ba2bc92cd..f564b65297e 100644 --- a/readthedocs/projects/admin.py +++ b/readthedocs/projects/admin.py @@ -1,21 +1,35 @@ +# -*- coding: utf-8 -*- """Django administration interface for `projects.models`""" -from __future__ import absolute_import -from django.contrib import admin -from django.contrib import messages +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from django.contrib import admin, messages from django.contrib.admin.actions import delete_selected from django.utils.translation import ugettext_lazy as _ from guardian.admin import GuardedModelAdmin +from readthedocs.builds.models import Version from readthedocs.core.models import UserProfile from readthedocs.core.utils import broadcast -from readthedocs.builds.models import Version -from readthedocs.redirects.models import Redirect from readthedocs.notifications.views import SendNotificationView +from readthedocs.redirects.models import Redirect from .forms import FeatureForm -from .models import (Project, ImportedFile, Feature, - ProjectRelationship, EmailHook, WebHook, Domain) +from .models import ( + Domain, + EmailHook, + EnvironmentVariable, + Feature, + ImportedFile, + Project, + ProjectRelationship, + WebHook, +) from .notifications import ResourceUsageNotification from .tasks import remove_dir @@ -202,7 +216,14 @@ def project_count(self, feature): return feature.projects.count() +class EnvironmentVariableAdmin(admin.ModelAdmin): + model = EnvironmentVariable + list_display = ('name', 'value', 'project', 'created') + search_fields = ('name', 'project__slug') + + admin.site.register(Project, ProjectAdmin) +admin.site.register(EnvironmentVariable, EnvironmentVariableAdmin) admin.site.register(ImportedFile, ImportedFileAdmin) admin.site.register(Domain, DomainAdmin) admin.site.register(Feature, FeatureAdmin) diff --git a/readthedocs/projects/migrations/0033_add_environment_variables.py b/readthedocs/projects/migrations/0033_add_environment_variables.py new file mode 100644 index 00000000000..de9e3d18e5b --- /dev/null +++ b/readthedocs/projects/migrations/0033_add_environment_variables.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2018-11-12 13:57 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0032_increase_webhook_maxsize'), + ] + + operations = [ + migrations.CreateModel( + name='EnvironmentVariable', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), + ('name', models.CharField(help_text='Name of the environment variable', max_length=128)), + ('value', models.CharField(help_text='Value of the environment variable', max_length=256)), + ], + options={ + 'ordering': ('-modified', '-created'), + 'get_latest_by': 'modified', + 'abstract': False, + }, + ), + migrations.AddField( + model_name='environmentvariable', + name='project', + field=models.ForeignKey(help_text='Project where this variable will be used', on_delete=django.db.models.deletion.CASCADE, to='projects.Project'), + ), + ] diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 78523387234..2fc0dcf9e0f 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -15,6 +15,7 @@ from django.db import models from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ +from django_extensions.db.models import TimeStampedModel from future.backports.urllib.parse import urlparse # noqa from guardian.shortcuts import assign from taggit.managers import TaggableManager @@ -869,13 +870,27 @@ def show_advertising(self): """ Whether this project is ad-free - :return: ``True`` if advertising should be shown and ``False`` otherwise + :returns: ``True`` if advertising should be shown and ``False`` otherwise + :rtype: bool """ if self.ad_free or self.gold_owners.exists(): return False return True + @property + def environment_variables(self): + """ + Environment variables to build this particular project. + + :returns: dictionary with all the variables {name: value} + :rtype: dict + """ + return { + variable.name: variable.value + for variable in self.environmentvariable_set.all() + } + class APIProject(Project): @@ -899,6 +914,7 @@ class Meta: def __init__(self, *args, **kwargs): self.features = kwargs.pop('features', []) + environment_variables = kwargs.pop('environment_variables', {}) ad_free = (not kwargs.pop('show_advertising', True)) # These fields only exist on the API return, not on the model, so we'll # remove them to avoid throwing exceptions due to unexpected fields @@ -912,6 +928,7 @@ def __init__(self, *args, **kwargs): # Overwrite the database property with the value from the API self.ad_free = ad_free + self._environment_variables = environment_variables def save(self, *args, **kwargs): return 0 @@ -924,6 +941,10 @@ def show_advertising(self): """Whether this project is ad-free (don't access the database)""" return not self.ad_free + @property + def environment_variables(self): + return self._environment_variables + @python_2_unicode_compatible class ImportedFile(models.Model): @@ -1109,3 +1130,19 @@ def get_feature_display(self): implement this behavior. """ return dict(self.FEATURES).get(self.feature_id, self.feature_id) + + +class EnvironmentVariable(TimeStampedModel, models.Model): + name = models.CharField( + max_length=128, + help_text=_('Name of the environment variable'), + ) + value = models.CharField( + max_length=256, + help_text=_('Value of the environment variable'), + ) + project = models.ForeignKey( + Project, + on_delete=models.CASCADE, + help_text=_('Project where this variable will be used'), + ) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 7dd35b72a06..30495a23f93 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -548,7 +548,7 @@ def get_build(build_pk): if build_pk: build = api_v2.build(build_pk).get() private_keys = [ - 'project', 'version', 'resource_uri', 'absolute_uri' + 'project', 'version', 'resource_uri', 'absolute_uri', ] return { key: val @@ -605,7 +605,7 @@ def get_env_vars(self): env = { 'READTHEDOCS': True, 'READTHEDOCS_VERSION': self.version.slug, - 'READTHEDOCS_PROJECT': self.project.slug + 'READTHEDOCS_PROJECT': self.project.slug, } if self.config.conda is not None: @@ -616,7 +616,7 @@ def get_env_vars(self): self.project.doc_path, 'conda', self.version.slug, - 'bin' + 'bin', ), }) else: @@ -625,10 +625,13 @@ def get_env_vars(self): self.project.doc_path, 'envs', self.version.slug, - 'bin' + 'bin', ), }) + # Update environment from Project's specific environment variables + env.update(self.project.environment_variables) + return env def set_valid_clone(self): diff --git a/readthedocs/restapi/serializers.py b/readthedocs/restapi/serializers.py index 0cf346c6d8b..1f7607c04c3 100644 --- a/readthedocs/restapi/serializers.py +++ b/readthedocs/restapi/serializers.py @@ -43,11 +43,6 @@ class ProjectAdminSerializer(ProjectSerializer): slug_field='feature_id', ) - show_advertising = serializers.SerializerMethodField() - - def get_show_advertising(self, obj): - return obj.show_advertising - class Meta(ProjectSerializer.Meta): fields = ProjectSerializer.Meta.fields + ( 'enable_epub_build', @@ -68,6 +63,7 @@ class Meta(ProjectSerializer.Meta): 'has_valid_clone', 'has_valid_webhook', 'show_advertising', + 'environment_variables', ) diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index 2981bf5b137..d169f9bc51d 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -26,7 +26,7 @@ from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.integrations.models import Integration from readthedocs.oauth.models import RemoteOrganization, RemoteRepository -from readthedocs.projects.models import APIProject, Feature, Project +from readthedocs.projects.models import APIProject, Feature, Project, EnvironmentVariable from readthedocs.restapi.views.integrations import GitHubWebhookView from readthedocs.restapi.views.task_views import get_status_data @@ -704,6 +704,27 @@ def test_remote_organization_pagination(self): self.assertEqual(len(resp.data['results']), 25) # page_size self.assertIn('?page=2', resp.data['next']) + def test_project_environment_variables(self): + user = get(User, is_staff=True) + project = get(Project, main_language_project=None) + get( + EnvironmentVariable, + name='TOKEN', + value='a1b2c3', + project=project, + ) + + client = APIClient() + client.force_authenticate(user=user) + + resp = client.get('/api/v2/project/%s/' % (project.pk)) + self.assertEqual(resp.status_code, 200) + self.assertIn('environment_variables', resp.data) + self.assertEqual( + resp.data['environment_variables'], + {'TOKEN': 'a1b2c3'}, + ) + def test_init_api_project(self): project_data = { 'name': 'Test Project', @@ -716,13 +737,16 @@ def test_init_api_project(self): self.assertEqual(api_project.features, []) self.assertFalse(api_project.ad_free) self.assertTrue(api_project.show_advertising) + self.assertEqual(api_project.environment_variables, {}) project_data['features'] = ['test-feature'] project_data['show_advertising'] = False + project_data['environment_variables'] = {'TOKEN': 'a1b2c3'} api_project = APIProject(**project_data) self.assertEqual(api_project.features, ['test-feature']) self.assertTrue(api_project.ad_free) self.assertFalse(api_project.show_advertising) + self.assertEqual(api_project.environment_variables, {'TOKEN': 'a1b2c3'}) class APIImportTests(TestCase): @@ -1186,6 +1210,7 @@ def test_get_version_by_id(self): 'default_version': 'latest', 'description': '', 'documentation_type': 'sphinx', + 'environment_variables': {}, 'enable_epub_build': True, 'enable_pdf_build': True, 'features': ['allow_deprecated_webhooks'], diff --git a/readthedocs/rtd_tests/tests/test_builds.py b/readthedocs/rtd_tests/tests/test_builds.py index 6e91ffba12e..57939fad374 100644 --- a/readthedocs/rtd_tests/tests/test_builds.py +++ b/readthedocs/rtd_tests/tests/test_builds.py @@ -6,6 +6,7 @@ ) import mock +import os from django.test import TestCase from django_dynamic_fixture import fixture, get @@ -13,7 +14,7 @@ from readthedocs.doc_builder.config import load_yaml_config from readthedocs.doc_builder.environments import LocalBuildEnvironment from readthedocs.doc_builder.python_environments import Virtualenv -from readthedocs.projects.models import Project +from readthedocs.projects.models import Project, EnvironmentVariable from readthedocs.projects.tasks import UpdateDocsTaskStep from readthedocs.rtd_tests.tests.test_config_integration import create_load @@ -250,6 +251,55 @@ def test_save_config_in_build_model(self, load_config, api_v2): assert 'sphinx' in build_config assert build_config['doctype'] == 'sphinx' + def test_get_env_vars(self): + project = get( + Project, + slug='project', + documentation_type='sphinx', + ) + get( + EnvironmentVariable, + name='TOKEN', + value='a1b2c3', + project=project, + ) + build = get(Build) + version = get(Version, slug='1.8', project=project) + task = UpdateDocsTaskStep( + project=project, version=version, build={'id': build.pk}, + ) + + # mock this object to make sure that we are NOT in a conda env + task.config = mock.Mock(conda=None) + + env = { + 'READTHEDOCS': True, + 'READTHEDOCS_VERSION': version.slug, + 'READTHEDOCS_PROJECT': project.slug, + 'BIN_PATH': os.path.join( + project.doc_path, + 'envs', + version.slug, + 'bin', + ), + 'TOKEN': 'a1b2c3', + } + self.assertEqual(task.get_env_vars(), env) + + # mock this object to make sure that we are in a conda env + task.config = mock.Mock(conda=True) + env.update({ + 'CONDA_ENVS_PATH': os.path.join(project.doc_path, 'conda'), + 'CONDA_DEFAULT_ENV': version.slug, + 'BIN_PATH': os.path.join( + project.doc_path, + 'conda', + version.slug, + 'bin', + ), + }) + self.assertEqual(task.get_env_vars(), env) + class BuildModelTests(TestCase):