Call python scripts as argument to virtualenv python command

This fixes #994, where the 127 character limit on shebang line lengths was
triggering failures of commands inside virtualenvs that had long names or
long branch names. This PR adds some path environment variable handling to
allow for shorter commands, and makes all python script call wrapped by a
call to the virtual env symlinked python binary.

Author: agjohnson

readthedocs/doc_builder/backends/mkdocs.py

@@ -130,14 +130,19 @@ def append_conf(self, **kwargs):
     def build(self, **kwargs):
         checkout_path = self.project.checkout_path(self.version.slug)
         build_command = [
+            'python',
             self.project.venv_bin(version=self.version.slug, bin='mkdocs'),
             self.builder,
             '--clean',
             '--site-dir', self.build_dir,
         ]
         if self.use_theme:
             build_command.extend(['--theme', 'readthedocs'])
-        cmd_ret = self.run(*build_command, cwd=checkout_path)
+        cmd_ret = self.run(
+            *build_command,
+            cwd=checkout_path,
+            bin_path=self.project.venv_bin(version=self.version.slug, bin=None)
+        )
         return cmd_ret.successful
 
 

readthedocs/doc_builder/backends/sphinx.py

@@ -126,6 +126,7 @@ def build(self, **kwargs):
         self.clean()
         project = self.project
         build_command = [
+            'python',
             project.venv_bin(version=self.version.slug, bin='sphinx-build'),
             '-T'
         ]
@@ -138,8 +139,11 @@ def build(self, **kwargs):
             '.',
             self.sphinx_build_dir
         ])
-        cmd_ret = self.run(*build_command,
-                           cwd=project.conf_dir(self.version.slug))
+        cmd_ret = self.run(
+            *build_command,
+            cwd=project.conf_dir(self.version.slug),
+            bin_path=project.venv_bin(version=self.version.slug, bin=None)
+        )
         return cmd_ret.successful
 
 
@@ -232,13 +236,15 @@ def build(self, **kwargs):
 
         # Default to this so we can return it always.
         self.run(
+            'python',
             self.project.venv_bin(version=self.version.slug, bin='sphinx-build'),
             '-b', 'latex',
             '-D', 'language={lang}'.format(lang=self.project.language),
             '-d', '_build/doctrees',
             '.',
             '_build/latex',
-            cwd=cwd
+            cwd=cwd,
+            bin_path=self.project.venv_bin(version=self.version.slug, bin=None)
         )
         latex_cwd = os.path.join(cwd, '_build', 'latex')
         tex_files = glob(os.path.join(latex_cwd, '*.tex'))

readthedocs/doc_builder/environments.py

@@ -44,7 +44,8 @@ class BuildCommand(object):
 
     # TODO add short name here for reporting
     def __init__(self, command, cwd=None, shell=False, environment=None,
-                 combine_output=True, input_data=None, build_env=None):
+                 combine_output=True, input_data=None, build_env=None,
+                 bin_path=None):
         self.command = command
         self.shell = shell
         if cwd is None:
@@ -55,6 +56,7 @@ def __init__(self, command, cwd=None, shell=False, environment=None,
             self.environment.update(environment)
         self.combine_output = combine_output
         self.build_env = build_env
+        self.bin_path = bin_path
         self.status = None
         self.input_data = input_data
         self.output = None
@@ -91,6 +93,10 @@ def run(self):
             del environment['DJANGO_SETTINGS_MODULE']
         if 'PYTHONPATH' in environment:
             del environment['PYTHONPATH']
+        if self.bin_path is not None:
+            env_paths = environment.get('PATH', '').split(':')
+            env_paths.insert(0, self.bin_path)
+            environment['PATH'] = ':'.join(env_paths)
 
         try:
             proc = subprocess.Popen(

readthedocs/projects/models.py

@@ -485,6 +485,14 @@ def single_version_symlink_path(self):
     #
 
     def venv_bin(self, version=LATEST, bin='python'):
+        """Return path to the virtualenv bin path, or a specific binary
+
+        By default, return the path to the ``python`` binary in the virtual
+        environment path. If ``bin`` is :py:data:`None`, then return the path to
+        the virtual env path.
+        """
+        if bin is None:
+            bin = ''
         return os.path.join(self.venv_path(version), 'bin', bin)
 
     def full_doc_path(self, version=LATEST):

readthedocs/projects/tasks.py

@@ -267,6 +267,7 @@ def setup_environment(self):
         ]
 
         cmd = [
+            'python',
             self.project.venv_bin(version=self.version.slug, bin='pip'),
             'install',
             '--use-wheel',
@@ -280,7 +281,10 @@ def setup_environment(self):
             # --system-site-packages is used)
             cmd.append('-I')
         cmd.extend(requirements)
-        self.build_env.run(*cmd)
+        self.build_env.run(
+            *cmd,
+            bin_path=self.project.venv_bin(version=self.version.slug, bin=None)
+        )
 
         # Handle requirements
         requirements_file_path = self.project.requirements_file
@@ -298,11 +302,14 @@ def setup_environment(self):
 
         if requirements_file_path:
             self.build_env.run(
+                'python',
                 self.project.venv_bin(version=self.version.slug, bin='pip'),
                 'install',
                 '--exists-action=w',
                 '-r{0}'.format(requirements_file_path),
-                cwd=checkout_path
+                cwd=checkout_path,
+                bin_path=self.project.venv_bin(version=self.version.slug,
+                                               bin=None)
             )
 
         # Handle setup.py
@@ -311,21 +318,24 @@ def setup_environment(self):
         if os.path.isfile(setup_path):
             if getattr(settings, 'USE_PIP_INSTALL', False):
                 self.build_env.run(
-                    self.project.venv_bin(version=self.version.slug,
-                                          bin='pip'),
+                    'python',
+                    self.project.venv_bin(version=self.version.slug, bin='pip'),
                     'install',
                     '--ignore-installed',
                     '.',
-                    cwd=checkout_path
+                    cwd=checkout_path,
+                    bin_path=self.project.venv_bin(version=self.version.slug,
+                                                   bin=None)
                 )
             else:
                 self.build_env.run(
-                    self.project.venv_bin(version=self.version.slug,
-                                          bin='python'),
+                    'python',
                     'setup.py',
                     'install',
                     '--force',
-                    cwd=checkout_path
+                    cwd=checkout_path,
+                    bin_path=self.project.venv_bin(version=self.version.slug,
+                                                   bin=None)
                 )
 
     def build_docs(self):

readthedocs/rtd_tests/tests/test_builds.py

@@ -46,7 +46,8 @@ def test_build(self):
         # Get command and check first part of command list is a call to sphinx
         self.assertEqual(self.mocks.popen.call_count, 1)
         cmd = self.mocks.popen.call_args_list[0][0]
-        self.assertRegexpMatches(cmd[0][0], r'sphinx-build')
+        self.assertRegexpMatches(cmd[0][0], r'python')
+        self.assertRegexpMatches(cmd[0][1], r'sphinx-build')
 
     def test_build_respects_pdf_flag(self):
         '''Build output format control'''