From be8f4de475b4a434ed9d595680025b06b9ff4bfc Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:07:54 -0700 Subject: [PATCH 1/9] Sync External versions to our database This is just an initial proof of concept. We still need to update the External type properly, and do lots of other work. This is just a start to show how we can sync and build them. --- readthedocs/projects/constants.py | 5 +++++ readthedocs/vcs_support/backends/git.py | 11 ++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 7c83cffd7ea..1a97a89a397 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -333,3 +333,8 @@ 'https://gitlab.com/{user}/{repo}/' '{action}/{version}{docroot}{path}{source_suffix}' ) + +DEFAULT_GIT_PATTERN = 'refs/heads/*:refs/remotes/origin/*' +GITHUB_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' +GITLAB_GIT_PATTERN = 'refs/merge-requests/*/head:refs/remotes/origin/external/*' +BITBUCKET_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index aa3adc6c2bc..5619ca44e13 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -11,6 +11,7 @@ from git.exc import BadName, InvalidGitRepositoryError from readthedocs.config import ALL +from readthedocs.projects.constants import GITHUB_GIT_PATTERN, DEFAULT_GIT_PATTERN from readthedocs.projects.exceptions import RepositoryError from readthedocs.projects.validators import validate_submodule_url from readthedocs.vcs_support.base import BaseVCS, VCSVersion @@ -55,9 +56,12 @@ def update(self): super().update() if self.repo_exists(): self.set_remote_url(self.repo_url) - return self.fetch() + self.fetch() + return self.make_clean_working_dir() - return self.clone() + self.clone() + # Do a fetch to make sure we get all external versions + self.fetch() def repo_exists(self): try: @@ -144,7 +148,8 @@ def use_shallow_clone(self): return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE) def fetch(self): - cmd = ['git', 'fetch', '--tags', '--prune', '--prune-tags'] + cmd = ['git', 'fetch', 'origin', DEFAULT_GIT_PATTERN, GITHUB_GIT_PATTERN, + '--tags', '--prune', '--prune-tags'] if self.use_shallow_clone(): cmd.extend(['--depth', str(self.repo_depth)]) From 30ac6bab3a2fea67e7b8b16090c6c2f505855c22 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:12:07 -0700 Subject: [PATCH 2/9] Remove unused patterns --- readthedocs/projects/constants.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 1a97a89a397..de8151b993d 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -335,6 +335,5 @@ ) DEFAULT_GIT_PATTERN = 'refs/heads/*:refs/remotes/origin/*' +# https://help.github.com/en/articles/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally GITHUB_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' -GITLAB_GIT_PATTERN = 'refs/merge-requests/*/head:refs/remotes/origin/external/*' -BITBUCKET_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' From df122daf4baecf6c28416d95a51f40f6b27bf39e Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:12:35 -0700 Subject: [PATCH 3/9] Small whitespace cleanup --- readthedocs/vcs_support/backends/git.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 5619ca44e13..c61fff5cc51 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -148,7 +148,8 @@ def use_shallow_clone(self): return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE) def fetch(self): - cmd = ['git', 'fetch', 'origin', DEFAULT_GIT_PATTERN, GITHUB_GIT_PATTERN, + cmd = ['git', 'fetch', 'origin', + DEFAULT_GIT_PATTERN, GITHUB_GIT_PATTERN, '--tags', '--prune', '--prune-tags'] if self.use_shallow_clone(): From da4f71d8ae3a731705c4be7c48d88211bb55a510 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:14:25 -0700 Subject: [PATCH 4/9] Clean up fetch logic a bit --- readthedocs/vcs_support/backends/git.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index c61fff5cc51..8bcb2c09a51 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -56,11 +56,9 @@ def update(self): super().update() if self.repo_exists(): self.set_remote_url(self.repo_url) - self.fetch() - return - self.make_clean_working_dir() - self.clone() - # Do a fetch to make sure we get all external versions + else: + self.make_clean_working_dir() + self.clone() self.fetch() def repo_exists(self): From 1207afb700e2c1534b3b7ce123b491616f981a3b Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:16:55 -0700 Subject: [PATCH 5/9] Add Comment --- readthedocs/vcs_support/backends/git.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 8bcb2c09a51..1718b4401cc 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -59,6 +59,7 @@ def update(self): else: self.make_clean_working_dir() self.clone() + # A fetch is always required to get external versions properly self.fetch() def repo_exists(self): From c74d29414bcd047dc61db0d29eb3bdbea5eff307 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 20 Jun 2019 00:29:01 +0600 Subject: [PATCH 6/9] Initial changes for PR Builder --- readthedocs/api/v2/utils.py | 33 ++++++++++++++++++++++++ readthedocs/api/v2/views/integrations.py | 7 +++++ readthedocs/api/v2/views/model_views.py | 24 ++++++++++++++--- readthedocs/projects/tasks.py | 6 +++++ readthedocs/vcs_support/backends/git.py | 26 ++++++++++++++++--- 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/readthedocs/api/v2/utils.py b/readthedocs/api/v2/utils.py index a9fcffea6d8..30184cefa13 100644 --- a/readthedocs/api/v2/utils.py +++ b/readthedocs/api/v2/utils.py @@ -13,7 +13,9 @@ STABLE, STABLE_VERBOSE_NAME, TAG, + EXTERNAL, ) +from readthedocs.core.utils import trigger_build from readthedocs.builds.models import Version @@ -77,6 +79,15 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin version_name, version_id, ) + elif type == EXTERNAL: + created_version = Version.objects.create( + project=project, + type=type, + identifier=version_id, + verbose_name=version_name, + ) + added.add(created_version.slug) + else: # New Version created_version = Version.objects.create( @@ -135,6 +146,9 @@ def delete_versions(project, version_data): versions_tags = [ version['verbose_name'] for version in version_data.get('tags', []) ] + external_versions = [ + version['verbose_name'] for version in version_data.get('external_branches', []) + ] versions_branches = [ version['identifier'] for version in version_data.get('branches', []) ] @@ -147,6 +161,10 @@ def delete_versions(project, version_data): type=BRANCH, identifier__in=versions_branches, ) + to_delete_qs = to_delete_qs.exclude( + type=EXTERNAL, + verbose_name__in=external_versions, + ) to_delete_qs = to_delete_qs.exclude(uploaded=True) to_delete_qs = to_delete_qs.exclude(active=True) to_delete_qs = to_delete_qs.exclude(slug__in=NON_REPOSITORY_VERSIONS) @@ -176,6 +194,21 @@ def run_automation_rules(project, versions_slug): rule.run(version) +def trigger_external_build(project, version_list): + """ + Trigger Builds for all external versions provided. + + The rules are sorted by priority. + + """ + for version_slug in version_list: + version = project.versions(manager=EXTERNAL).get(slug=version_slug) + version.active = True + version.save() + + trigger_build(project=project, version=version) + + class RemoteOrganizationPagination(PageNumberPagination): page_size = 25 diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 4424238a78b..75979a450ba 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -29,6 +29,9 @@ GITHUB_EVENT_HEADER = 'HTTP_X_GITHUB_EVENT' GITHUB_SIGNATURE_HEADER = 'HTTP_X_HUB_SIGNATURE' GITHUB_PUSH = 'push' +GITHUB_PULL_REQUEST = 'pull_request' +GITHUB_PULL_REQUEST_OPEN = 'open' +GITHUB_PULL_REQUEST_SYNC = 'synchronize' GITHUB_CREATE = 'create' GITHUB_DELETE = 'delete' GITLAB_TOKEN_HEADER = 'HTTP_X_GITLAB_TOKEN' @@ -271,6 +274,10 @@ def handle_webhook(self): raise ParseError('Parameter "ref" is required') if event in (GITHUB_CREATE, GITHUB_DELETE): return self.sync_versions(self.project) + + if event == GITHUB_PULL_REQUEST: + return self.sync_versions(self.project) + return None def _normalize_ref(self, ref): diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index fca91b32626..ecc57a0a8f0 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG, INTERNAL +from readthedocs.builds.constants import BRANCH, TAG, INTERNAL, EXTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -189,6 +189,7 @@ def sync_versions(self, request, **kwargs): # noqa: D205 # Update All Versions data = request.data added_versions = set() + added_external_versions = set() if 'tags' in data: ret_set = api_utils.sync_versions( project=project, @@ -203,6 +204,15 @@ def sync_versions(self, request, **kwargs): # noqa: D205 type=BRANCH, ) added_versions.update(ret_set) + + if 'external_branches' in data: + ret_set = api_utils.sync_versions( + project=project, + versions=data['external_branches'], + type=EXTERNAL, + ) + added_external_versions.update(ret_set) + deleted_versions = api_utils.delete_versions(project, data) except Exception as e: log.exception('Sync Versions Error') @@ -213,11 +223,13 @@ def sync_versions(self, request, **kwargs): # noqa: D205 status=status.HTTP_400_BAD_REQUEST, ) + all_added_versions = added_versions | added_external_versions + try: # The order of added_versions isn't deterministic. # We don't track the commit time or any other metadata. # We usually have one version added per webhook. - api_utils.run_automation_rules(project, added_versions) + api_utils.run_automation_rules(project, all_added_versions) except Exception: # Don't interrupt the request if something goes wrong # in the automation rules. @@ -226,6 +238,12 @@ def sync_versions(self, request, **kwargs): # noqa: D205 project.slug, added_versions ) + if added_external_versions: + api_utils.trigger_external_build( + project=project, + version_list=added_external_versions + ) + # TODO: move this to an automation rule promoted_version = project.update_stable_version() new_stable = project.get_stable_version() @@ -250,7 +268,7 @@ def sync_versions(self, request, **kwargs): # noqa: D205 trigger_build(project=project, version=promoted_version) return Response({ - 'added_versions': added_versions, + 'added_versions': all_added_versions, 'deleted_versions': deleted_versions, }) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 7efcd5a762a..1bf721249cb 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -165,6 +165,12 @@ def sync_versions(self, version_repo): 'verbose_name': v.verbose_name, } for v in version_repo.branches] + if version_repo.supports_external_branches: + version_post_data['external_branches'] = [{ + 'identifier': v.identifier, + 'verbose_name': v.verbose_name, + } for v in version_repo.external_branches] + self.validate_duplicate_reserved_versions(version_post_data) try: diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 1718b4401cc..193ebc2d07a 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -26,6 +26,7 @@ class Backend(BaseVCS): supports_tags = True supports_branches = True + supports_external_branches = True supports_submodules = True fallback_branch = 'master' # default branch repo_depth = 50 @@ -213,11 +214,30 @@ def branches(self): for branch in branches: verbose_name = branch.name - if verbose_name.startswith('origin/'): + if not verbose_name.startswith('origin/external/'): + if verbose_name.startswith('origin/'): + verbose_name = verbose_name.replace('origin/', '') + if verbose_name == 'HEAD': + continue + versions.append(VCSVersion(self, str(branch), verbose_name)) + return versions + + @property + def external_branches(self): + repo = git.Repo(self.working_dir) + versions = [] + branches = [] + + # ``repo.remotes.origin.refs`` returns remote branches + if repo.remotes: + branches += repo.remotes.origin.refs + + for branch in branches: + verbose_name = branch.name + if verbose_name.startswith('origin/external/'): verbose_name = verbose_name.replace('origin/', '') - if verbose_name == 'HEAD': + versions.append(VCSVersion(self, str(branch.commit), verbose_name)) continue - versions.append(VCSVersion(self, str(branch), verbose_name)) return versions @property From 5fb2e4eba6da0190180dcb626c47a4690758ed1c Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 21 Jun 2019 18:11:27 +0600 Subject: [PATCH 7/9] Only build external version from webhook payload --- readthedocs/api/v2/utils.py | 33 ------------------ readthedocs/api/v2/views/integrations.py | 34 +++++++++++++++--- readthedocs/api/v2/views/model_views.py | 26 +++----------- readthedocs/core/views/hooks.py | 20 +++++++++++ readthedocs/projects/constants.py | 4 +-- readthedocs/projects/tasks.py | 8 +---- readthedocs/vcs_support/backends/bzr.py | 2 +- readthedocs/vcs_support/backends/git.py | 44 ++++++++++-------------- readthedocs/vcs_support/backends/hg.py | 2 +- readthedocs/vcs_support/backends/svn.py | 2 +- 10 files changed, 77 insertions(+), 98 deletions(-) diff --git a/readthedocs/api/v2/utils.py b/readthedocs/api/v2/utils.py index 30184cefa13..a9fcffea6d8 100644 --- a/readthedocs/api/v2/utils.py +++ b/readthedocs/api/v2/utils.py @@ -13,9 +13,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, - EXTERNAL, ) -from readthedocs.core.utils import trigger_build from readthedocs.builds.models import Version @@ -79,15 +77,6 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin version_name, version_id, ) - elif type == EXTERNAL: - created_version = Version.objects.create( - project=project, - type=type, - identifier=version_id, - verbose_name=version_name, - ) - added.add(created_version.slug) - else: # New Version created_version = Version.objects.create( @@ -146,9 +135,6 @@ def delete_versions(project, version_data): versions_tags = [ version['verbose_name'] for version in version_data.get('tags', []) ] - external_versions = [ - version['verbose_name'] for version in version_data.get('external_branches', []) - ] versions_branches = [ version['identifier'] for version in version_data.get('branches', []) ] @@ -161,10 +147,6 @@ def delete_versions(project, version_data): type=BRANCH, identifier__in=versions_branches, ) - to_delete_qs = to_delete_qs.exclude( - type=EXTERNAL, - verbose_name__in=external_versions, - ) to_delete_qs = to_delete_qs.exclude(uploaded=True) to_delete_qs = to_delete_qs.exclude(active=True) to_delete_qs = to_delete_qs.exclude(slug__in=NON_REPOSITORY_VERSIONS) @@ -194,21 +176,6 @@ def run_automation_rules(project, versions_slug): rule.run(version) -def trigger_external_build(project, version_list): - """ - Trigger Builds for all external versions provided. - - The rules are sorted by priority. - - """ - for version_slug in version_list: - version = project.versions(manager=EXTERNAL).get(slug=version_slug) - version.active = True - version.save() - - trigger_build(project=project, version=version) - - class RemoteOrganizationPagination(PageNumberPagination): page_size = 25 diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 75979a450ba..d194623b7b9 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -19,7 +19,11 @@ webhook_github, webhook_gitlab, ) -from readthedocs.core.views.hooks import build_branches, sync_versions +from readthedocs.core.views.hooks import ( + build_branches, + sync_versions, + get_or_create_external_version, +) from readthedocs.integrations.models import HttpExchange, Integration from readthedocs.projects.models import Project @@ -30,7 +34,7 @@ GITHUB_SIGNATURE_HEADER = 'HTTP_X_HUB_SIGNATURE' GITHUB_PUSH = 'push' GITHUB_PULL_REQUEST = 'pull_request' -GITHUB_PULL_REQUEST_OPEN = 'open' +GITHUB_PULL_REQUEST_OPEN = 'opened' GITHUB_PULL_REQUEST_SYNC = 'synchronize' GITHUB_CREATE = 'create' GITHUB_DELETE = 'delete' @@ -113,6 +117,10 @@ def handle_webhook(self): """Handle webhook payload.""" raise NotImplementedError + def get_external_version_data(self): + """Get External Version data from payload.""" + raise NotImplementedError + def is_payload_valid(self): """Validates the webhook's payload using the integration's secret.""" return False @@ -221,6 +229,13 @@ def get_data(self): pass return super().get_data() + def get_external_version_data(self): + """Get Commit Sha and pull request number from payload""" + identifier = self.data['pull_request']['head']['sha'] + verbose_name = str(self.data['number']) + + return identifier, verbose_name + def is_payload_valid(self): """ GitHub use a HMAC hexdigest hash to sign the payload. @@ -275,8 +290,19 @@ def handle_webhook(self): if event in (GITHUB_CREATE, GITHUB_DELETE): return self.sync_versions(self.project) - if event == GITHUB_PULL_REQUEST: - return self.sync_versions(self.project) + if ( + event == GITHUB_PULL_REQUEST and + self.data['action'] in [GITHUB_PULL_REQUEST_OPEN, GITHUB_PULL_REQUEST_SYNC] + ): + try: + identifier, verbose_name = self.get_external_version_data() + external_version = get_or_create_external_version( + self.project, identifier, verbose_name + ) + return self.get_response_push(self.project, external_version.verbose_name) + + except KeyError: + raise ParseError('Parameters "sha" and "number" are required') return None diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index ecc57a0a8f0..af41f28ab84 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG, INTERNAL, EXTERNAL +from readthedocs.builds.constants import BRANCH, TAG from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions(manager=INTERNAL).filter(active=True) + versions = project.versions.filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) @@ -189,7 +189,6 @@ def sync_versions(self, request, **kwargs): # noqa: D205 # Update All Versions data = request.data added_versions = set() - added_external_versions = set() if 'tags' in data: ret_set = api_utils.sync_versions( project=project, @@ -204,15 +203,6 @@ def sync_versions(self, request, **kwargs): # noqa: D205 type=BRANCH, ) added_versions.update(ret_set) - - if 'external_branches' in data: - ret_set = api_utils.sync_versions( - project=project, - versions=data['external_branches'], - type=EXTERNAL, - ) - added_external_versions.update(ret_set) - deleted_versions = api_utils.delete_versions(project, data) except Exception as e: log.exception('Sync Versions Error') @@ -223,13 +213,11 @@ def sync_versions(self, request, **kwargs): # noqa: D205 status=status.HTTP_400_BAD_REQUEST, ) - all_added_versions = added_versions | added_external_versions - try: # The order of added_versions isn't deterministic. # We don't track the commit time or any other metadata. # We usually have one version added per webhook. - api_utils.run_automation_rules(project, all_added_versions) + api_utils.run_automation_rules(project, added_versions) except Exception: # Don't interrupt the request if something goes wrong # in the automation rules. @@ -238,12 +226,6 @@ def sync_versions(self, request, **kwargs): # noqa: D205 project.slug, added_versions ) - if added_external_versions: - api_utils.trigger_external_build( - project=project, - version_list=added_external_versions - ) - # TODO: move this to an automation rule promoted_version = project.update_stable_version() new_stable = project.get_stable_version() @@ -268,7 +250,7 @@ def sync_versions(self, request, **kwargs): # noqa: D205 trigger_build(project=project, version=promoted_version) return Response({ - 'added_versions': all_added_versions, + 'added_versions': added_versions, 'deleted_versions': deleted_versions, }) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index a3273a88871..bd5b8c40cb0 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -2,6 +2,8 @@ import logging +from readthedocs.builds.constants import EXTERNAL +from readthedocs.builds.models import Version from readthedocs.core.utils import trigger_build from readthedocs.projects.tasks import sync_repository_task @@ -88,3 +90,21 @@ def sync_versions(project): except Exception: log.exception('Unknown sync versions exception') return None + + +def get_or_create_external_version(project, identifier, verbose_name): + external_version = project.versions(manager=EXTERNAL).filter(verbose_name=verbose_name).first() + if external_version: + if external_version.identifier != identifier: + external_version.identifier = identifier + external_version.save() + else: + created_external_version = Version.objects.create( + project=project, + type=EXTERNAL, + identifier=identifier, + verbose_name=verbose_name, + active=True + ) + return created_external_version + return external_version diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index de8151b993d..8b770830f79 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -334,6 +334,4 @@ '{action}/{version}{docroot}{path}{source_suffix}' ) -DEFAULT_GIT_PATTERN = 'refs/heads/*:refs/remotes/origin/*' -# https://help.github.com/en/articles/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally -GITHUB_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' +GITHUB_GIT_PATTERN = 'pull/{id}/head:external-{id}' diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 1bf721249cb..2d0e087b2a2 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -138,7 +138,7 @@ def sync_repo(self): } ) version_repo = self.get_vcs_repo() - version_repo.update() + version_repo.update(version=self.version) self.sync_versions(version_repo) version_repo.checkout(self.version.identifier) finally: @@ -165,12 +165,6 @@ def sync_versions(self, version_repo): 'verbose_name': v.verbose_name, } for v in version_repo.branches] - if version_repo.supports_external_branches: - version_post_data['external_branches'] = [{ - 'identifier': v.identifier, - 'verbose_name': v.verbose_name, - } for v in version_repo.external_branches] - self.validate_duplicate_reserved_versions(version_post_data) try: diff --git a/readthedocs/vcs_support/backends/bzr.py b/readthedocs/vcs_support/backends/bzr.py index e228ac720d3..99029d6dc82 100644 --- a/readthedocs/vcs_support/backends/bzr.py +++ b/readthedocs/vcs_support/backends/bzr.py @@ -17,7 +17,7 @@ class Backend(BaseVCS): supports_tags = True fallback_branch = '' - def update(self): + def update(self, version=None): # pylint: disable=arguments-differ super().update() retcode = self.run('bzr', 'status', record=False)[0] if retcode == 0: diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 193ebc2d07a..ba265fe3704 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -10,8 +10,9 @@ from django.core.exceptions import ValidationError from git.exc import BadName, InvalidGitRepositoryError +from readthedocs.builds.constants import EXTERNAL from readthedocs.config import ALL -from readthedocs.projects.constants import GITHUB_GIT_PATTERN, DEFAULT_GIT_PATTERN +from readthedocs.projects.constants import GITHUB_GIT_PATTERN from readthedocs.projects.exceptions import RepositoryError from readthedocs.projects.validators import validate_submodule_url from readthedocs.vcs_support.base import BaseVCS, VCSVersion @@ -52,16 +53,21 @@ def _get_clone_url(self): def set_remote_url(self, url): return self.run('git', 'remote', 'set-url', 'origin', url) - def update(self): + def update(self, version=None): # pylint: disable=arguments-differ """Clone or update the repository.""" super().update() if self.repo_exists(): self.set_remote_url(self.repo_url) - else: - self.make_clean_working_dir() - self.clone() + # A fetch is always required to get external versions properly + if version and version.type == EXTERNAL: + return self.fetch(version.verbose_name) + return self.fetch() + self.make_clean_working_dir() # A fetch is always required to get external versions properly - self.fetch() + if version and version.type == EXTERNAL: + self.clone() + return self.fetch(version.verbose_name) + return self.clone() def repo_exists(self): try: @@ -147,11 +153,15 @@ def use_shallow_clone(self): from readthedocs.projects.models import Feature return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE) - def fetch(self): + def fetch(self, verbose_name=None): cmd = ['git', 'fetch', 'origin', - DEFAULT_GIT_PATTERN, GITHUB_GIT_PATTERN, '--tags', '--prune', '--prune-tags'] + if verbose_name and 'github.com' in self.repo_url: + cmd.append( + GITHUB_GIT_PATTERN.format(id=verbose_name) + ) + if self.use_shallow_clone(): cmd.extend(['--depth', str(self.repo_depth)]) @@ -222,24 +232,6 @@ def branches(self): versions.append(VCSVersion(self, str(branch), verbose_name)) return versions - @property - def external_branches(self): - repo = git.Repo(self.working_dir) - versions = [] - branches = [] - - # ``repo.remotes.origin.refs`` returns remote branches - if repo.remotes: - branches += repo.remotes.origin.refs - - for branch in branches: - verbose_name = branch.name - if verbose_name.startswith('origin/external/'): - verbose_name = verbose_name.replace('origin/', '') - versions.append(VCSVersion(self, str(branch.commit), verbose_name)) - continue - return versions - @property def commit(self): if self.repo_exists(): diff --git a/readthedocs/vcs_support/backends/hg.py b/readthedocs/vcs_support/backends/hg.py index 0361bfa462c..9b44faeed2e 100644 --- a/readthedocs/vcs_support/backends/hg.py +++ b/readthedocs/vcs_support/backends/hg.py @@ -13,7 +13,7 @@ class Backend(BaseVCS): supports_branches = True fallback_branch = 'default' - def update(self): + def update(self, version=None): # pylint: disable=arguments-differ super().update() retcode = self.run('hg', 'status', record=False)[0] if retcode == 0: diff --git a/readthedocs/vcs_support/backends/svn.py b/readthedocs/vcs_support/backends/svn.py index b1a945aec2c..50bab45a512 100644 --- a/readthedocs/vcs_support/backends/svn.py +++ b/readthedocs/vcs_support/backends/svn.py @@ -27,7 +27,7 @@ def __init__(self, *args, **kwargs): else: self.base_url = self.repo_url - def update(self): + def update(self, version=None): # pylint: disable=arguments-differ super().update() # For some reason `svn status` gives me retcode 0 in non-svn # directories that's why I use `svn info` here. From a859eacc5d331c679c9722cee521aec66d2ffb48 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 21 Jun 2019 18:17:19 +0600 Subject: [PATCH 8/9] fix --- readthedocs/api/v2/views/model_views.py | 4 ++-- readthedocs/vcs_support/backends/git.py | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index af41f28ab84..fca91b32626 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG +from readthedocs.builds.constants import BRANCH, TAG, INTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions.filter(active=True) + versions = project.versions(manager=INTERNAL).filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index ba265fe3704..506143132ed 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -224,12 +224,11 @@ def branches(self): for branch in branches: verbose_name = branch.name - if not verbose_name.startswith('origin/external/'): - if verbose_name.startswith('origin/'): - verbose_name = verbose_name.replace('origin/', '') - if verbose_name == 'HEAD': - continue - versions.append(VCSVersion(self, str(branch), verbose_name)) + if verbose_name.startswith('origin/'): + verbose_name = verbose_name.replace('origin/', '') + if verbose_name == 'HEAD': + continue + versions.append(VCSVersion(self, str(branch), verbose_name)) return versions @property From 5b883df743276aebb41c1765e54f04c2f2a3ab03 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 21 Jun 2019 18:50:37 +0600 Subject: [PATCH 9/9] need to pass verbose_name as a list for get_response_push --- readthedocs/api/v2/views/integrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index d194623b7b9..1567e71b18e 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -299,7 +299,7 @@ def handle_webhook(self): external_version = get_or_create_external_version( self.project, identifier, verbose_name ) - return self.get_response_push(self.project, external_version.verbose_name) + return self.get_response_push(self.project, [external_version.verbose_name]) except KeyError: raise ParseError('Parameters "sha" and "number" are required')