Add temporary method for skipping submodule checkout (#3821)

* Add temporary method for skipping submodule checkout

This adds a project feature that allows for a project to specify that they would
like to skip submodule installation. Currently we are forcing all submodules to
be checked out, so this fails on private submodules.

Refs readthedocs/readthedocs-build#30

* Also protect git clone

* Lint fixes

Author: agjohnson

readthedocs/projects/models.py

@@ -1067,12 +1067,14 @@ def add_features(sender, **kwargs):
     USE_SETUPTOOLS_LATEST = 'use_setuptools_latest'
     ALLOW_DEPRECATED_WEBHOOKS = 'allow_deprecated_webhooks'
     PIP_ALWAYS_UPGRADE = 'pip_always_upgrade'
+    SKIP_SUBMODULES = 'skip_submodules'
 
     FEATURES = (
         (USE_SPHINX_LATEST, _('Use latest version of Sphinx')),
         (USE_SETUPTOOLS_LATEST, _('Use latest version of setuptools')),
         (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')),
         (PIP_ALWAYS_UPGRADE, _('Always run pip install --upgrade')),
+        (SKIP_SUBMODULES, _('Skip git submodule checkout')),
     )
 
     projects = models.ManyToManyField(

readthedocs/rtd_tests/tests/test_backend.py

@@ -2,8 +2,9 @@
 from os.path import exists
 
 from django.contrib.auth.models import User
+import django_dynamic_fixture as fixture
 
-from readthedocs.projects.models import Project
+from readthedocs.projects.models import Project, Feature
 from readthedocs.rtd_tests.base import RTDTestCase
 
 from readthedocs.rtd_tests.utils import make_test_git, make_test_hg
@@ -82,11 +83,23 @@ def test_check_for_submodules(self):
         repo = self.project.vcs_repo()
 
         repo.checkout()
-        self.assertFalse(repo.submodules_exists())
+        self.assertFalse(repo.are_submodules_available())
 
         # The submodule branch contains one submodule
         repo.checkout('submodule')
-        self.assertTrue(repo.submodules_exists())
+        self.assertTrue(repo.are_submodules_available())
+
+    def test_skip_submodule_checkout(self):
+        repo = self.project.vcs_repo()
+        repo.checkout('submodule')
+        self.assertTrue(repo.are_submodules_available())
+        feature = fixture.get(
+            Feature,
+            projects=[self.project],
+            feature_id=Feature.SKIP_SUBMODULES,
+        )
+        self.assertTrue(self.project.has_feature(Feature.SKIP_SUBMODULES))
+        self.assertFalse(repo.are_submodules_available())
 
 
 class TestHgBackend(RTDTestCase):

readthedocs/vcs_support/backends/git.py

@@ -54,7 +54,20 @@ def repo_exists(self):
         code, _, _ = self.run('git', 'status', record=False)
         return code == 0
 
-    def submodules_exists(self):
+    def are_submodules_available(self):
+        """
+        Test whether git submodule checkout step should be performed.
+
+        .. note::
+
+            Temporarily, we support skipping these steps as submodule step can
+            fail if using private submodules. This will eventually be
+            configureable with our YAML config.
+        """
+        # TODO remove with https://github.com/rtfd/readthedocs-build/issues/30
+        from readthedocs.projects.models import Feature
+        if self.project.has_feature(Feature.SKIP_SUBMODULES):
+            return False
         code, out, _ = self.run('git', 'submodule', 'status', record=False)
         return code == 0 and bool(out)
 
@@ -74,7 +87,22 @@ def checkout_revision(self, revision=None):
         return [code, out, err]
 
     def clone(self):
-        code, _, _ = self.run('git', 'clone', '--recursive', self.repo_url, '.')
+        """
+        Clone the repository.
+
+        .. note::
+
+            Temporarily, we support skipping submodule recursive clone via a
+            feature flag. This will eventually be configureable with our YAML
+            config.
+        """
+        # TODO remove with https://github.com/rtfd/readthedocs-build/issues/30
+        from readthedocs.projects.models import Feature
+        cmd = ['git', 'clone']
+        if not self.project.has_feature(Feature.SKIP_SUBMODULES):
+            cmd.append('--recursive')
+        cmd.extend([self.repo_url, '.'])
+        code, _, _ = self.run(*cmd)
         if code != 0:
             raise RepositoryError
 
@@ -199,8 +227,9 @@ def checkout(self, identifier=None):
         # Clean any remains of previous checkouts
         self.run('git', 'clean', '-d', '-f', '-f')
 
-        # Update submodules
-        if self.submodules_exists():
+        # Update submodules, temporarily allow for skipping submodule checkout
+        # step for projects need more submodule configuration.
+        if self.are_submodules_available():
             self.run('git', 'submodule', 'sync')
             self.run(
                 'git',

readthedocs/vcs_support/base.py

@@ -52,6 +52,7 @@ class BaseVCS(object):
     # pylint: disable=unused-argument
     def __init__(self, project, version_slug, environment=None, **kwargs):
         self.default_branch = project.default_branch
+        self.project = project
         self.name = project.name
         self.repo_url = project.clean_repo
         self.working_dir = project.checkout_path(version_slug)