CRUD for EnvironmentVariables from Project's admin

docs/builds.rst

@@ -225,3 +225,8 @@ The *Sphinx* and *Mkdocs* builders set the following RTD-specific environment va
 +-------------------------+--------------------------------------------------+----------------------+
 | ``READTHEDOCS_PROJECT`` | The RTD name of the project which is being built | ``myexampleproject`` |
 +-------------------------+--------------------------------------------------+----------------------+
+
+.. tip::
+
+   In case extra environment variables are needed to the build process (like secrets, tokens, etc),
+   you can add them going to **Admin > Environment Variables** in your project. See :doc:`guides/environment-variables`.

docs/guides/environment-variables.rst

@@ -0,0 +1,37 @@
+I Need Secrets (or Environment Variables) in my Build
+=====================================================
+
+It may happen that your documentation depends on an authenticated service to be built properly.
+In this case, you will require some secrets to access these services.
+
+Read the Docs provides a way to define environment variables for your project to be used in the build process.
+All these variables will be exposed to all the commands executed when building your documentation.
+
+To define an environment variable, you need to
+
+#. Go to your project **Admin > Environment Variables**
+#. Click on "Add Environment Variable" button
+#. Input a ``Name`` and ``Value`` (your secret needed here)
+#. Click "Save" button
+
+.. note::
+
+   Values will never be exposed to users, even to owners of the project. Once you create an environment variable you won't be able to see its value anymore because of security purposes.
+
+After adding an environment variable from your project's admin, you can access it from your build process using Python, for example:
+
+.. code-block:: python
+
+   # conf.py
+   import os
+   import requests
+
+   # Access to our custom environment variables
+   username = os.environ.get('USERNAME')
+   password = os.environ.get('PASSWORD')
+
+   # Request a username/password protected URL
+   response = requests.get(
+       'https://httpbin.org/basic-auth/username/password',
+       auth=(username, password),
+   )

docs/guides/specifying-dependencies.rst

@@ -35,7 +35,7 @@ Using the project admin dashboard
 Once the requirements file has been created;
 
 - Login to Read the Docs and go to the project admin dashboard.
-- Go to ``Admin > Advanced Settings > Requirements file``.
+- Go to **Admin > Advanced Settings > Requirements file**.
 - Specify the path of the requirements file you just created. The path should be relative to the root directory of the documentation project.
 
 Using a conda environment file

readthedocs/projects/forms.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+
 """Project forms."""
 
 from __future__ import (
@@ -8,6 +9,18 @@
     unicode_literals,
 )
 
+try:
+    # TODO: remove this when we deprecate Python2
+    # re.fullmatch is >= Py3.4 only
+    from re import fullmatch
+except ImportError:
+    # https://stackoverflow.com/questions/30212413/backport-python-3-4s-regular-expression-fullmatch-to-python-2
+    import re
+
+    def fullmatch(regex, string, flags=0):
+        """Emulate python-3.4 re.fullmatch()."""  # noqa
+        return re.match("(?:" + regex + r")\Z", string, flags=flags)
+
 from random import choice
 
 from builtins import object
@@ -27,11 +40,11 @@
 from readthedocs.integrations.models import Integration
 from readthedocs.oauth.models import RemoteRepository
 from readthedocs.projects import constants
-from readthedocs.projects.constants import PUBLIC
 from readthedocs.projects.exceptions import ProjectSpamError
 from readthedocs.projects.models import (
     Domain,
     EmailHook,
+    EnvironmentVariable,
     Feature,
     Project,
     ProjectRelationship,
@@ -758,3 +771,49 @@ class Meta(object):
     def __init__(self, *args, **kwargs):
         super(FeatureForm, self).__init__(*args, **kwargs)
         self.fields['feature_id'].choices = Feature.FEATURES
+
+
+class EnvironmentVariableForm(forms.ModelForm):
+
+    """
+    Form to add an EnvironmentVariable to a Project.
+
+    This limits the name of the variable.
+    """
+
+    project = forms.CharField(widget=forms.HiddenInput(), required=False)
+
+    class Meta(object):
+        model = EnvironmentVariable
+        fields = ('name', 'value', 'project')
+
+    def __init__(self, *args, **kwargs):
+        self.project = kwargs.pop('project', None)
+        super(EnvironmentVariableForm, self).__init__(*args, **kwargs)
+
+    def clean_project(self):
+        return self.project
+
+    def clean_name(self):
+        name = self.cleaned_data['name']
+        if name.startswith('__'):
+            raise forms.ValidationError(
+                _("Variable name can't start with __ (double underscore)"),
+            )
+        elif name.startswith('READTHEDOCS'):
+            raise forms.ValidationError(
+                _("Variable name can't start with READTHEDOCS"),
+            )
+        elif self.project.environmentvariable_set.filter(name=name).exists():
+            raise forms.ValidationError(
+                _('There is already a variable with this name for this project'),
+            )
+        elif ' ' in name:
+            raise forms.ValidationError(
+                _("Variable name can't contain spaces"),
+            )
+        elif not fullmatch('[a-zA-Z0-9_]+', name):
+            raise forms.ValidationError(
+                _('Only letters, numbers and underscore are allowed'),
+            )
+        return name

readthedocs/projects/models.py

@@ -8,6 +8,7 @@
 import logging
 import os
 from builtins import object  # pylint: disable=redefined-builtin
+from six.moves import shlex_quote
 
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -1064,6 +1065,7 @@ def get_feature_display(self):
         return dict(self.FEATURES).get(self.feature_id, self.feature_id)
 
 
+@python_2_unicode_compatible
 class EnvironmentVariable(TimeStampedModel, models.Model):
     name = models.CharField(
         max_length=128,
@@ -1078,3 +1080,10 @@ class EnvironmentVariable(TimeStampedModel, models.Model):
         on_delete=models.CASCADE,
         help_text=_('Project where this variable will be used'),
     )
+
+    def __str__(self):
+        return self.name
+
+    def save(self, *args, **kwargs):  # pylint: disable=arguments-differ
+        self.value = shlex_quote(self.value)
+        return super(EnvironmentVariable, self).save(*args, **kwargs)

readthedocs/projects/urls/private.py

@@ -1,18 +1,38 @@
-"""Project URLs for authenticated users"""
+"""Project URLs for authenticated users."""
+
+from __future__ import (
+    absolute_import,
+    division,
+    print_function,
+    unicode_literals,
+)
 
-from __future__ import absolute_import
 from django.conf.urls import url
 
 from readthedocs.constants import pattern_opts
+from readthedocs.projects.backends.views import ImportDemoView, ImportWizardView
 from readthedocs.projects.views import private
 from readthedocs.projects.views.private import (
-    ProjectDashboard, ImportView,
-    ProjectUpdate, ProjectAdvancedUpdate,
-    DomainList, DomainCreate, DomainDelete, DomainUpdate,
-    IntegrationList, IntegrationCreate, IntegrationDetail, IntegrationDelete,
-    IntegrationExchangeDetail, IntegrationWebhookSync, ProjectAdvertisingUpdate)
-from readthedocs.projects.backends.views import ImportWizardView, ImportDemoView
-
+    DomainCreate,
+    DomainDelete,
+    DomainList,
+    DomainUpdate,
+    EnvironmentVariableCreate,
+    EnvironmentVariableDelete,
+    EnvironmentVariableList,
+    EnvironmentVariableDetail,
+    ImportView,
+    IntegrationCreate,
+    IntegrationDelete,
+    IntegrationDetail,
+    IntegrationExchangeDetail,
+    IntegrationList,
+    IntegrationWebhookSync,
+    ProjectAdvancedUpdate,
+    ProjectAdvertisingUpdate,
+    ProjectDashboard,
+    ProjectUpdate,
+)
 
 urlpatterns = [
     url(r'^$',
@@ -171,3 +191,20 @@
 ]
 
 urlpatterns += subproject_urls
+
+environmentvariable_urls = [
+    url(r'^(?P<project_slug>[-\w]+)/environmentvariables/$',
+        EnvironmentVariableList.as_view(),
+        name='projects_environmentvariables'),
+    url(r'^(?P<project_slug>[-\w]+)/environmentvariables/create/$',
+        EnvironmentVariableCreate.as_view(),
+        name='projects_environmentvariables_create'),
+    url(r'^(?P<project_slug>[-\w]+)/environmentvariables/(?P<environmentvariable_pk>[-\w]+)/$',
+        EnvironmentVariableDetail.as_view(),
+        name='projects_environmentvariables_detail'),
+    url(r'^(?P<project_slug>[-\w]+)/environmentvariables/(?P<environmentvariable_pk>[-\w]+)/delete/$',
+        EnvironmentVariableDelete.as_view(),
+        name='projects_environmentvariables_delete'),
+]
+
+urlpatterns += environmentvariable_urls

readthedocs/projects/views/private.py

@@ -43,6 +43,7 @@
 from readthedocs.projects.forms import (
     DomainForm,
     EmailHookForm,
+    EnvironmentVariableForm,
     IntegrationForm,
     ProjectAdvancedForm,
     ProjectAdvertisingForm,
@@ -59,6 +60,7 @@
 from readthedocs.projects.models import (
     Domain,
     EmailHook,
+    EnvironmentVariable,
     Project,
     ProjectRelationship,
     WebHook,
@@ -875,3 +877,37 @@ def get_queryset(self):
 
     def get_success_url(self):
         return reverse('projects_advertising', args=[self.object.slug])
+
+
+class EnvironmentVariableMixin(ProjectAdminMixin, PrivateViewMixin):
+
+    """Environment Variables to be added when building the Project."""
+
+    model = EnvironmentVariable
+    form_class = EnvironmentVariableForm
+    lookup_url_kwarg = 'environmentvariable_pk'
+
+    def get_success_url(self):
+        return reverse(
+            'projects_environmentvariables',
+            args=[self.get_project().slug],
+        )
+
+
+class EnvironmentVariableList(EnvironmentVariableMixin, ListView):
+    pass
+
+
+class EnvironmentVariableCreate(EnvironmentVariableMixin, CreateView):
+    pass
+
+
+class EnvironmentVariableDetail(EnvironmentVariableMixin, DetailView):
+    pass
+
+
+class EnvironmentVariableDelete(EnvironmentVariableMixin, DeleteView):
+
+    # This removes the delete confirmation
+    def get(self, request, *args, **kwargs):
+        return self.http_method_not_allowed(request, *args, **kwargs)

readthedocs/restapi/serializers.py

@@ -43,6 +43,12 @@ class ProjectAdminSerializer(ProjectSerializer):
         slug_field='feature_id',
     )
 
+    def get_environment_variables(self, obj):
+        return {
+            variable.name: variable.value
+            for variable in obj.environmentvariable_set.all()
+        }
+
     class Meta(ProjectSerializer.Meta):
         fields = ProjectSerializer.Meta.fields + (
             'enable_epub_build',

readthedocs/rtd_tests/tests/test_privacy_urls.py

@@ -14,7 +14,7 @@
 from readthedocs.builds.models import Build, BuildCommandResult
 from readthedocs.core.utils.tasks import TaskNoPermission
 from readthedocs.integrations.models import HttpExchange, Integration
-from readthedocs.projects.models import Project, Domain
+from readthedocs.projects.models import Project, Domain, EnvironmentVariable
 from readthedocs.oauth.models import RemoteRepository, RemoteOrganization
 from readthedocs.rtd_tests.utils import create_user
 
@@ -150,6 +150,7 @@ def setUp(self):
             status_code=200,
         )
         self.domain = get(Domain, url='http://docs.foobar.com', project=self.pip)
+        self.environment_variable = get(EnvironmentVariable, project=self.pip)
         self.default_kwargs = {
             'project_slug': self.pip.slug,
             'subproject_slug': self.subproject.slug,
@@ -162,6 +163,7 @@ def setUp(self):
             'domain_pk': self.domain.pk,
             'integration_pk': self.integration.pk,
             'exchange_pk': self.webhook_exchange.pk,
+            'environmentvariable_pk': self.environment_variable.pk,
         }
 
 
@@ -241,11 +243,13 @@ class PrivateProjectAdminAccessTest(PrivateProjectMixin, TestCase):
         '/dashboard/pip/integrations/sync/': {'status_code': 405},
         '/dashboard/pip/integrations/{integration_id}/sync/': {'status_code': 405},
         '/dashboard/pip/integrations/{integration_id}/delete/': {'status_code': 405},
+        '/dashboard/pip/environmentvariables/{environmentvariable_id}/delete/': {'status_code': 405},
     }
 
     def get_url_path_ctx(self):
         return {
             'integration_id': self.integration.id,
+            'environmentvariable_id': self.environment_variable.id,
         }
 
     def login(self):
@@ -275,6 +279,7 @@ class PrivateProjectUserAccessTest(PrivateProjectMixin, TestCase):
         '/dashboard/pip/integrations/sync/': {'status_code': 405},
         '/dashboard/pip/integrations/{integration_id}/sync/': {'status_code': 405},
         '/dashboard/pip/integrations/{integration_id}/delete/': {'status_code': 405},
+        '/dashboard/pip/environmentvariables/{environmentvariable_id}/delete/': {'status_code': 405},
     }
 
     # Filtered out by queryset on projects that we don't own.
@@ -283,6 +288,7 @@ class PrivateProjectUserAccessTest(PrivateProjectMixin, TestCase):
     def get_url_path_ctx(self):
         return {
             'integration_id': self.integration.id,
+            'environmentvariable_id': self.environment_variable.id,
         }
 
     def login(self):