Merge branch 'master' of github.com:readthedocs/readthedocs.org into humitos/remove-web-related-tasks

humitos · humitos · commit 4338b99cee2e · 2020-05-04T14:34:42.000+02:00
diff --git a/dockerfiles/nginx/proxito.conf b/dockerfiles/nginx/proxito.conf
@@ -56,8 +56,10 @@ server {
         add_header X-RTD-Path $rtd_path always;
         set $rtd_domain $upstream_http_x_rtd_domain;
         add_header X-RTD-Domain $rtd_domain always;
-        set $rtd_method $upstream_http_x_rtd_version_method;
-        add_header X-RTD-Version-Method $rtd_method always;
+        set $rtd_version_method $upstream_http_x_rtd_version_method;
+        add_header X-RTD-Version-Method $rtd_version_method always;
+        set $rtd_project_method $upstream_http_x_rtd_project_method;
+        add_header X-RTD-Project-Method $rtd_project_method always;
         set $rtd_redirect $upstream_http_x_rtd_redirect;
         add_header X-RTD-Redirect $rtd_redirect always;
     }
diff --git a/docs/custom_domains.rst b/docs/custom_domains.rst
@@ -40,22 +40,14 @@ You can also host your documentation from your own domain.
       The SSL certificate issuance can take about one hour,
       you can see the status of the certificate on the domain page in your project.
 
-      For example, https://pip.pypa.io resolves, but is hosted on our infrastructure.
-      As another example, fabric's dig record looks like this:
+      As an example, fabric's dig record looks like this:
 
       .. prompt:: bash $, auto
 
-         $ dig docs.fabfile.org
-         ...
-         ;; ANSWER SECTION:
-         docs.fabfile.org.   7200    IN  CNAME   readthedocs.io.
-
-      .. note::
-
-         Some older setups configured a CNAME record pointing to ``readthedocs.org`` or another variation.
-         While these continue to resolve,
-         they do not yet allow us to acquire SSL certificates for those domains.
-         Follow the new setup to have a SSL certificate.
+         $ dig +short docs.fabfile.org
+         readthedocs.io.
+         104.17.33.82
+         104.17.32.82
 
       .. admonition:: Certificate Authority Authorization (CAA)
 
diff --git a/docs/guides/autobuild-docs-for-pull-requests.rst b/docs/guides/autobuild-docs-for-pull-requests.rst
@@ -42,7 +42,8 @@ you might hit one of these issues:
 
 #. **Build status is not being reported on your Pull/Merge Request**.
    You need to make sure that you have granted access to the Read the Docs
-   `OAuth App`_ to your/organizations GitHub account.
+   `OAuth App`_ to your/organizations GitHub account. If you do not see "Read the Docs"
+   among the `OAuth App`_, you might need to disconnect and reconnect to GitHub service.
    Also make sure your webhook is properly setup
    to handle events. You can setup or ``re-sync`` the webhook from your projects admin dashboard.
    Learn more about setting up webhooks from our :doc:`Webhook Documentation </webhooks>`.
diff --git a/docs/guides/feature-flags.rst b/docs/guides/feature-flags.rst
@@ -37,6 +37,8 @@ e.g. python-reno release notes manager is known to do that
 
 ``EXTERNAL_VERSION_BUILD``: :featureflags:`EXTERNAL_VERSION_BUILD`
 
+``LIST_PACKAGES_INSTALLED_ENV``: :featureflags:`LIST_PACKAGES_INSTALLED_ENV`
+
 ``SHARE_SPHINX_DOCTREE``: :featureflags:`SHARE_SPHINX_DOCTREE`
 
 By default, when Read the Docs runs Sphinx it passes a different output directory for the generated/parsed doctrees
diff --git a/readthedocs/conftest.py b/readthedocs/conftest.py
@@ -1,32 +1,6 @@
 import pytest
 from rest_framework.test import APIClient
 
-
-try:
-    # TODO: this file is read/executed even when called from ``readthedocsinc``,
-    # so it's overriding the options that we are defining in the ``conftest.py``
-    # from the corporate site. We need to find a better way to avoid this.
-    import readthedocsinc
-    PYTEST_OPTIONS = ()
-except ImportError:
-    PYTEST_OPTIONS = (
-        # Options to set test environment
-        ('community', True),
-        ('corporate', False),
-        ('environment', 'readthedocs'),
-    )
-
-
-def pytest_configure(config):
-    for option, value in PYTEST_OPTIONS:
-        setattr(config.option, option, value)
-
-
-@pytest.fixture(autouse=True)
-def settings_modification(settings):
-    settings.CELERY_ALWAYS_EAGER = True
-
-
 @pytest.fixture
 def api_client():
     return APIClient()
diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py
@@ -425,9 +425,23 @@ def install_requirements_file(self, install):
             self.build_env.run(
                 *args,
                 cwd=self.checkout_path,
-                bin_path=self.venv_bin()  # noqa - no comma here in py27 :/
+                bin_path=self.venv_bin(),
             )
 
+    def list_packages_installed(self):
+        """List packages installed in pip."""
+        args = [
+            self.venv_bin(filename='python'),
+            '-m',
+            'pip',
+            'list',
+        ]
+        self.build_env.run(
+            *args,
+            cwd=self.checkout_path,
+            bin_path=self.venv_bin(),
+        )
+
 
 class Conda(PythonEnvironment):
 
@@ -627,3 +641,15 @@ def install_requirements_file(self, install):
         # as the conda environment was created by using the ``environment.yml``
         # defined by the user, there is nothing to update at this point
         pass
+
+    def list_packages_installed(self):
+        """List packages installed in conda."""
+        args = [
+            'conda',
+            'list',
+        ]
+        self.build_env.run(
+            *args,
+            cwd=self.checkout_path,
+            bin_path=self.venv_bin(),
+        )
diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py
@@ -1516,6 +1516,8 @@ def add_features(sender, **kwargs):
     CACHED_ENVIRONMENT = 'cached_environment'
     CELERY_ROUTER = 'celery_router'
     LIMIT_CONCURRENT_BUILDS = 'limit_concurrent_builds'
+    LIST_PACKAGES_INSTALLED_ENV = 'list_packages_installed_env'
+    VCS_REMOTE_LISTING = 'vcs_remote_listing'
 
     FEATURES = (
         (USE_SPHINX_LATEST, _('Use latest version of Sphinx')),
@@ -1594,6 +1596,17 @@ def add_features(sender, **kwargs):
             LIMIT_CONCURRENT_BUILDS,
             _('Limit the amount of concurrent builds'),
         ),
+        (
+            LIST_PACKAGES_INSTALLED_ENV,
+            _(
+                'List packages installed in the environment ("pip list" or "conda list") '
+                'on build\'s output',
+            ),
+        ),
+        (
+            VCS_REMOTE_LISTING,
+            _('Use remote listing in VCS (e.g. git ls-remote) if supported for sync versions'),
+        ),
     )
 
     projects = models.ManyToManyField(
diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py
@@ -245,24 +245,37 @@ def sync_versions(self, version_repo):
         ``sync_versions`` endpoint.
         """
         version_post_data = {'repo': version_repo.repo_url}
+        tags = None
+        branches = None
+        if all([
+                version_repo.supports_lsremote,
+                not version_repo.repo_exists(),
+                self.project.has_feature(Feature.VCS_REMOTE_LISTING),
+        ]):
+            # Do not use ``ls-remote`` if the VCS does not support it or if we
+            # have already cloned the repository locally. The latter happens
+            # when triggering a normal build.
+            branches, tags = version_repo.lsremote
 
         if all([
             version_repo.supports_tags,
             not self.project.has_feature(Feature.SKIP_SYNC_TAGS)
         ]):
+            tags = tags or version_repo.tags
             version_post_data['tags'] = [{
                 'identifier': v.identifier,
                 'verbose_name': v.verbose_name,
-            } for v in version_repo.tags]
+            } for v in tags]
 
         if all([
             version_repo.supports_branches,
             not self.project.has_feature(Feature.SKIP_SYNC_BRANCHES)
         ]):
+            branches = branches or version_repo.branches
             version_post_data['branches'] = [{
                 'identifier': v.identifier,
                 'verbose_name': v.verbose_name,
-            } for v in version_repo.branches]
+            } for v in branches]
 
         self.validate_duplicate_reserved_versions(version_post_data)
 
@@ -372,7 +385,7 @@ def run(self, version_pk):  # pylint: disable=arguments-differ
                     # all the other cached things (Python packages, Sphinx,
                     # virtualenv, etc)
                     self.pull_cached_environment()
-                    self.sync_repo(environment)
+                    self.update_versions_from_repository(environment)
             return True
         except RepositoryError:
             # Do not log as ERROR handled exceptions
@@ -401,6 +414,24 @@ def run(self, version_pk):  # pylint: disable=arguments-differ
         # Always return False for any exceptions
         return False
 
+    def update_versions_from_repository(self, environment):
+        """
+        Update Read the Docs versions from VCS repository.
+
+        Depending if the VCS backend supports remote listing, we just list its branches/tags
+        remotely or we do a full clone and local listing of branches/tags.
+        """
+        version_repo = self.get_vcs_repo(environment)
+        if any([
+                not version_repo.supports_lsremote,
+                not self.project.has_feature(Feature.VCS_REMOTE_LISTING),
+        ]):
+            log.info('Syncing repository via full clone. project=%s', self.projec.slug)
+            self.sync_repo(environment)
+        else:
+            log.info('Syncing repository via remote listing. project=%s', self.projec.slug)
+            self.sync_versions(version_repo)
+
 
 @app.task(
     bind=True,
@@ -1116,6 +1147,8 @@ def setup_python_environment(self):
         self.python_env.save_environment_json()
         self.python_env.install_core_requirements()
         self.python_env.install_requirements()
+        if self.project.has_feature(Feature.LIST_PACKAGES_INSTALLED_ENV):
+            self.python_env.list_packages_installed()
 
     def build_docs(self):
         """
@@ -1170,7 +1203,7 @@ def build_docs_search(self):
            For MkDocs search is indexed from its ``html`` artifacts.
            And in sphinx is run using the rtd-sphinx-extension.
         """
-        return self.is_type_sphinx() and self.version.type != EXTERNAL
+        return self.is_type_sphinx()
 
     def build_docs_localmedia(self):
         """Get local media files with separate build."""
diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
@@ -132,10 +132,30 @@ def process_request(self, request):  # noqa
         return None
 
     def process_response(self, request, response):  # noqa
-        """Set the Strict-Transport-Security (HSTS) header for a custom domain if max-age>0."""
-        if hasattr(request, 'domain'):
+        """
+        Set the Strict-Transport-Security (HSTS) header for docs sites.
+
+        * For the public domain, set the HSTS header if settings.PUBLIC_DOMAIN_USES_HTTPS
+        * For custom domains, check the HSTS values on the Domain object.
+          The domain object should be saved already in request.domain.
+        """
+        host = request.get_host().lower().split(':')[0]
+        public_domain = settings.PUBLIC_DOMAIN.lower().split(':')[0]
+
+        hsts_header_values = []
+
+        if not request.is_secure():
+            # Only set the HSTS header if the request is over HTTPS
+            return response
+
+        if settings.PUBLIC_DOMAIN_USES_HTTPS and public_domain in host:
+            hsts_header_values = [
+                'max-age=31536000',
+                'includeSubDomains',
+                'preload',
+            ]
+        elif hasattr(request, 'domain'):
             domain = request.domain
-            hsts_header_values = []
             if domain.hsts_max_age:
                 hsts_header_values.append(f'max-age={domain.hsts_max_age}')
                 # These other options don't make sense without max_age > 0
@@ -144,6 +164,8 @@ def process_response(self, request, response):  # noqa
                 if domain.hsts_preload:
                     hsts_header_values.append('preload')
 
-                # See https://tools.ietf.org/html/rfc6797
-                response['Strict-Transport-Security'] = '; '.join(hsts_header_values)
+        if hsts_header_values:
+            # See https://tools.ietf.org/html/rfc6797
+            response['Strict-Transport-Security'] = '; '.join(hsts_header_values)
+
         return response
diff --git a/readthedocs/proxito/tests/test_full.py b/readthedocs/proxito/tests/test_full.py
@@ -198,7 +198,25 @@ def test_valid_project_as_invalid_subproject(self):
         resp = self.client.get(url, HTTP_HOST=host)
         self.assertEqual(resp.status_code, 404)
 
-    def test_response_hsts(self):
+    def test_public_domain_hsts(self):
+        host = 'project.dev.readthedocs.io'
+        response = self.client.get('/', HTTP_HOST=host)
+        self.assertFalse('strict-transport-security' in response)
+
+        response = self.client.get("/", HTTP_HOST=host, secure=True)
+        self.assertFalse('strict-transport-security' in response)
+
+        with override_settings(PUBLIC_DOMAIN_USES_HTTPS=True):
+            response = self.client.get('/', HTTP_HOST=host)
+            self.assertFalse('strict-transport-security' in response)
+
+            response = self.client.get("/", HTTP_HOST=host, secure=True)
+            self.assertEqual(
+                response['strict-transport-security'],
+                'max-age=31536000; includeSubDomains; preload',
+            )
+
+    def test_custom_domain_response_hsts(self):
         hostname = 'docs.random.com'
         domain = fixture.get(
             Domain,
@@ -212,10 +230,16 @@ def test_response_hsts(self):
         response = self.client.get("/", HTTP_HOST=hostname)
         self.assertFalse('strict-transport-security' in response)
 
+        response = self.client.get("/", HTTP_HOST=hostname, secure=True)
+        self.assertFalse('strict-transport-security' in response)
+
         domain.hsts_max_age = 3600
         domain.save()
 
         response = self.client.get("/", HTTP_HOST=hostname)
+        self.assertFalse('strict-transport-security' in response)
+
+        response = self.client.get("/", HTTP_HOST=hostname, secure=True)
         self.assertTrue('strict-transport-security' in response)
         self.assertEqual(
             response['strict-transport-security'], 'max-age=3600',
@@ -225,7 +249,7 @@ def test_response_hsts(self):
         domain.hsts_preload = True
         domain.save()
 
-        response = self.client.get("/", HTTP_HOST=hostname)
+        response = self.client.get("/", HTTP_HOST=hostname, secure=True)
         self.assertTrue('strict-transport-security' in response)
         self.assertEqual(
             response['strict-transport-security'], 'max-age=3600; includeSubDomains; preload',
diff --git a/readthedocs/proxito/views/mixins.py b/readthedocs/proxito/views/mixins.py
@@ -129,13 +129,15 @@ def _serve_docs_nginx(self, request, final_project, version_slug, path, download
         # Include the project & project-version so we can do larger purges if needed
         response['Cache-Tag'] = f'{final_project.slug}-{version_slug},{final_project.slug}'
         if hasattr(request, 'rtdheader'):
-            response['X-RTD-Version-Method'] = 'rtdheader'
-        if hasattr(request, 'subdomain'):
-            response['X-RTD-Version-Method'] = 'subdomain'
+            response['X-RTD-Project-Method'] = 'rtdheader'
+        elif hasattr(request, 'subdomain'):
+            response['X-RTD-Project-Method'] = 'subdomain'
+        elif hasattr(request, 'cname'):
+            response['X-RTD-Project-Method'] = 'cname'
         if hasattr(request, 'external_domain'):
-            response['X-RTD-Version-Method'] = 'external_domain'
-        if hasattr(request, 'cname'):
-            response['X-RTD-Version-Method'] = 'cname'
+            response['X-RTD-Version-Method'] = 'domain'
+        else:
+            response['X-RTD-Version-Method'] = 'path'
 
         return response
 
diff --git a/readthedocs/rtd_tests/tests/test_backend.py b/readthedocs/rtd_tests/tests/test_backend.py
diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py
diff --git a/readthedocs/settings/test.py b/readthedocs/settings/test.py
diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py
diff --git a/readthedocs/vcs_support/base.py b/readthedocs/vcs_support/base.py
