From 926f95dced231fc432aa3caf250131fde3f7299b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 22 Jan 2025 15:47:20 -0500 Subject: [PATCH 01/92] Migrate GitHub OAuth App to GitHub App Ref https://github.com/readthedocs/readthedocs.org/issues/11780 --- .../allauth/providers/githubapp/__init__.py | 0 .../allauth/providers/githubapp/provider.py | 10 ++++++++++ .../allauth/providers/githubapp/urls.py | 6 ++++++ .../allauth/providers/githubapp/views.py | 13 ++++++++++++ readthedocs/core/adapters.py | 18 +++++++++++++++++ readthedocs/settings/base.py | 20 ++++++++++++------- 6 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 readthedocs/allauth/providers/githubapp/__init__.py create mode 100644 readthedocs/allauth/providers/githubapp/provider.py create mode 100644 readthedocs/allauth/providers/githubapp/urls.py create mode 100644 readthedocs/allauth/providers/githubapp/views.py diff --git a/readthedocs/allauth/providers/githubapp/__init__.py b/readthedocs/allauth/providers/githubapp/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/allauth/providers/githubapp/provider.py b/readthedocs/allauth/providers/githubapp/provider.py new file mode 100644 index 00000000000..cf5958a7826 --- /dev/null +++ b/readthedocs/allauth/providers/githubapp/provider.py @@ -0,0 +1,10 @@ +from allauth.socialaccount.providers.github.provider import GitHubProvider +from readthedocs.allauth.providers.githubapp.views import GitHubAppOAuth2Adapter + + +class GitHubAppProvider(GitHubProvider): + id = "githubapp" + name = "GitHub App" + oauth2_adapter_class = GitHubAppOAuth2Adapter + +provider_classes = [GitHubAppProvider] diff --git a/readthedocs/allauth/providers/githubapp/urls.py b/readthedocs/allauth/providers/githubapp/urls.py new file mode 100644 index 00000000000..3ab15feaae2 --- /dev/null +++ b/readthedocs/allauth/providers/githubapp/urls.py @@ -0,0 +1,6 @@ +from allauth.socialaccount.providers.oauth2.urls import default_urlpatterns + +from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider + + +urlpatterns = default_urlpatterns(GitHubAppProvider) diff --git a/readthedocs/allauth/providers/githubapp/views.py b/readthedocs/allauth/providers/githubapp/views.py new file mode 100644 index 00000000000..a5d1377c648 --- /dev/null +++ b/readthedocs/allauth/providers/githubapp/views.py @@ -0,0 +1,13 @@ +from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter + +from allauth.socialaccount.providers.oauth2.views import ( + OAuth2CallbackView, + OAuth2LoginView, +) + +class GitHubAppOAuth2Adapter(GitHubOAuth2Adapter): + provider_id = 'githubapp' + + +oauth2_login = OAuth2LoginView.adapter_view(GitHubAppOAuth2Adapter) +oauth2_callback = OAuth2CallbackView.adapter_view(GitHubAppOAuth2Adapter) diff --git a/readthedocs/core/adapters.py b/readthedocs/core/adapters.py index 56edf345691..c2f8005271d 100644 --- a/readthedocs/core/adapters.py +++ b/readthedocs/core/adapters.py @@ -1,9 +1,14 @@ """Allauth overrides.""" +from allauth.socialaccount.adapter import DefaultSocialAccountAdapter +from allauth.socialaccount.models import SocialAccount +from allauth.socialaccount.providers.github.provider import GitHubProvider + import structlog from allauth.account.adapter import DefaultAccountAdapter from django.utils.encoding import force_str +from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider from readthedocs.core.utils import send_email_from_object from readthedocs.invitations.models import Invitation @@ -50,3 +55,16 @@ def save_user(self, request, user, form, commit=True): invitation.delete() else: log.info("Invitation not found", invitation_pk=invitation_pk) + + +class SocialAccountAdapter(DefaultSocialAccountAdapter): + + def pre_social_login(self, request, sociallogin): + provider = sociallogin.account.get_provider() + if provider.id == GitHubAppProvider.id and not sociallogin.is_existing: + social_ccount = SocialAccount.objects.filter( + provider=GitHubProvider.id, + uid=sociallogin.account.uid, + ).first() + if social_ccount: + sociallogin.connect(request, social_ccount.user) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index b98cc1ea04f..7e7e6ca9038 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -2,19 +2,17 @@ import os import re -import subprocess import socket +import subprocess import structlog - from celery.schedules import crontab +from corsheaders.defaults import default_headers +from django.conf.global_settings import PASSWORD_HASHERS +from readthedocs.builds import constants_docker from readthedocs.core.logs import shared_processors -from corsheaders.defaults import default_headers from readthedocs.core.settings import Settings -from readthedocs.builds import constants_docker - -from django.conf.global_settings import PASSWORD_HASHERS try: import readthedocsext.cdn # noqa @@ -73,7 +71,7 @@ def _show_debug_toolbar(request): # It's a "known issue/bug" and there is no solution as far as we can tell. "debug_toolbar.panels.sql.SQLPanel", "debug_toolbar.panels.templates.TemplatesPanel", - ] + ], } @property @@ -294,6 +292,7 @@ def INSTALLED_APPS(self): # noqa "allauth.account", "allauth.socialaccount", "allauth.socialaccount.providers.github", + "readthedocs.allauth.providers.githubapp", "allauth.socialaccount.providers.gitlab", "allauth.socialaccount.providers.bitbucket_oauth2", "allauth.mfa", @@ -679,6 +678,7 @@ def DOCKER_LIMITS(self): # Allauth ACCOUNT_ADAPTER = "readthedocs.core.adapters.AccountAdapter" + SOCIALACCOUNT_ADAPTER = 'readthedocs.core.adapters.SocialAccountAdapter' ACCOUNT_EMAIL_REQUIRED = True # By preventing enumeration, we will always send an email, # even if the email is not registered, that's hurting @@ -709,6 +709,12 @@ def DOCKER_LIMITS(self): "repo:status", ], }, + "githubapp": { + "APPS": [ + {"client_id": "123", "secret": "456", "key": ""}, + ], + "SCOPE": [], + }, "gitlab": { "APPS": [ {"client_id": "123", "secret": "456", "key": ""}, From 1314a9cb2ca40939304669f623f96345a58a2b1e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 30 Jan 2025 17:55:12 -0500 Subject: [PATCH 02/92] wip --- readthedocs/oauth/models.py | 52 ++++++- readthedocs/oauth/services/githubapp.py | 185 ++++++++++++++++++++++++ readthedocs/oauth/urls.py | 7 + readthedocs/oauth/views.py | 153 ++++++++++++++++++++ readthedocs/settings/base.py | 7 + readthedocs/urls.py | 1 + requirements/pip.in | 3 + 7 files changed, 407 insertions(+), 1 deletion(-) create mode 100644 readthedocs/oauth/services/githubapp.py create mode 100644 readthedocs/oauth/urls.py create mode 100644 readthedocs/oauth/views.py diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 4810943d046..67fee5a90a6 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -15,8 +15,47 @@ from .querysets import RemoteOrganizationQuerySet, RemoteRepositoryQuerySet -class RemoteOrganization(TimeStampedModel): +class GitHubAccountType(models.TextChoices): + USER = "User", _("User") + ORGANIZATION = "Organization", _("Organization") + + +class GitHubAppInstallation(TimeStampedModel): + # Should we just use big int? + installation_id = models.CharField( + help_text=_("The application installation ID"), + max_length=255, + unique=True, + db_index=True, + ) + target_id = models.CharField( + help_text=_("A GitHub account ID, it can be from a user or an organization"), + max_length=255, + ) + target_type = models.CharField( + help_text=_( + "Account type that the target_id belongs to (user or organization)" + ), + choices=GitHubAccountType.choices, + max_length=255, + ) + extra_data = models.JSONField( + help_text=_( + "Extra data returned by the webhook when the installation is created" + ), + default=dict, + ) + class Meta(TimeStampedModel.Meta): + constraints = [ + models.UniqueConstraint( + fields=["target_id", "target_type"], + name="unique_target_id_target_type", + ) + ] + + +class RemoteOrganization(TimeStampedModel): """ Organization from remote service. @@ -174,6 +213,17 @@ class RemoteRepository(TimeStampedModel): _("VCS provider"), choices=VCS_PROVIDER_CHOICES, max_length=32 ) + github_app_installation = models.ForeignKey( + GitHubAppInstallation, + verbose_name=_("GitHub App Installation"), + related_name="repositories", + null=True, + blank=True, + # Delete the repository if the installation is deleted? + # or keep the repository and just remove the installation? + on_delete=models.SET_NULL, + ) + objects = RemoteRepositoryQuerySet.as_manager() class Meta: diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py new file mode 100644 index 00000000000..2bf7cd0efeb --- /dev/null +++ b/readthedocs/oauth/services/githubapp.py @@ -0,0 +1,185 @@ +import structlog +from allauth.socialaccount.models import SocialAccount +from django.conf import settings +from github import Auth, GithubIntegration +from github.Installation import Installation as GHInstallation +from github.NamedUser import NamedUser as GHNamedUser +from github.Repository import Repository as GHRepository + +from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider +from readthedocs.oauth.constants import GITHUB +from readthedocs.oauth.models import ( + GitHubAccountType, + GitHubAppInstallation, + RemoteOrganization, + RemoteRepository, + RemoteRepositoryRelation, +) + +log = structlog.get_logger(__name__) + +class GitHubAppService: + vcs_provider_slug = GITHUB + def __init__(self, installation: GitHubAppInstallation): + self.installation = installation + self._client = None + + def _get_auth(self): + app_auth = Auth.AppAuth( + app_id=settings.GITHUB_APP_CLIENT_ID, + private_key=settings.GITHUB_APP_PRIVATE_KEY, + # 10 minutes is the maximum allowed by GitHub. + # PyGithub will handle the token expiration and renew it automatically. + jwt_expiry=60 * 10, + ) + return app_auth + + @property + def client(self): + """Return a client authenticated as the GitHub App to interact with most of the GH API""" + if self._client is None: + self._client = self.integration_client.get_github_for_installation(self.installation.installation_id) + return self._client + + @property + def integration_client(self): + """Return a client authenticated as the GitHub App to interact with the installation API""" + if self._integration_client is None: + self._integration_client = GithubIntegration(auth=self._get_auth()) + return self._integration_client + + @property + def app_installation(self) -> GHInstallation: + return self.integration_client.get_app_installation(self.installation.installation_id) + + def sync_repositories(self): + if self.installation.target_type != GitHubAccountType.USER: + return + return self._sync_installation_repositories() + + def _sync_installation_repositories(self): + remote_repositories = [] + for repo in self.app_installation.get_repos(): + remote_repo = self.create_or_update_repository(repo) + if remote_repo: + remote_repositories.append(remote_repo) + + # Remove repositories that are no longer in the list. + RemoteRepository.objects.filter( + github_app_installation=self.installation, + vcs_provider=self.vcs_provider_slug, + ).exclude( + pk__in=[repo.pk for repo in remote_repositories], + ).delete() + + def add_repositories(self, repository_ids: list[int]): + for repository_id in repository_ids: + repo = self.client.get_repo(repository_id) + self.create_or_update_repository(repo) + + def remove_repositories(self, repository_ids: list[int]): + RemoteRepository.objects.filter( + github_app_installation=self.installation, + vcs_provider=self.vcs_provider_slug, + remote_id__in=repository_ids, + ).delete() + + def create_or_update_repository(self, repo: GHRepository) -> RemoteRepository | None + if not settings.ALLOW_PRIVATE_REPOS and repo.private: + return + + remote_repo, _ = RemoteRepository.objects.get_or_create( + remote_id=str(repo.id), + vcs_provider=self.vcs_provider_slug, + ) + + remote_repo.name = repo.name + remote_repo.full_name = repo.full_name + remote_repo.description = repo.description + remote_repo.avatar_url = repo.owner.avatar_url + remote_repo.ssh_url = repo.ssh_url + remote_repo.html_url = repo.html_url + remote_repo.private = repo.private + remote_repo.default_branch = repo.default_branch + + # TODO: Do we need the SSH URL for private repositories now that we can clone using a token? + remote_repo.clone_url = repo.ssh_url if repo.private else repo.clone_url + + # NOTE: Only one installation of our APP should give access to a repository. + # This should only happen if our data is out of sync. + if remote_repo.github_app_installation and remote_repo.github_app_installation != self.installation: + log.info( + "Repository linked to another installation", + repository_id=remote_repo.remote_id, + old_installation_id=remote_repo.github_app_installation.installation_id, + new_installation_id=self.installation.installation_id, + ) + remote_repo.github_app_installation = self.installation + + remote_repo.organization = None + if repo.owner.type == GitHubAccountType.ORGANIZATION: + remote_repo.organization = self.create_or_update_organization(repo.owner) + + self._resync_collaborators(repo, remote_repo) + # What about memmbers of the organization? do we care? + # I think all of our permissions are based on the collaborators of the repository, + # not the members of the organization. + remote_repo.save() + return remote_repo + + def create_or_update_organization(self, org: GHNamedUser) -> RemoteOrganization: + remote_org, _ = RemoteOrganization.objects.get_or_create( + remote_id=str(org.id), + vcs_provider=self.vcs_provider_slug, + ) + remote_org.slug = org.login + remote_org.name = org.name + # NOTE: do we need the email of the organization? + remote_org.email = org.email + remote_org.avatar_url = org.avatar_url + remote_org.url = org.html_url + remote_org.save() + return remote_org + + def _resync_collaborators(self, repo: GHRepository, remote_repo: RemoteRepository): + """ + Sync collaborators of a repository with the database. + + This method will remove collaborators that are no longer in the list. + """ + collaborators = { + collaborator.id: collaborator + # Return all collaborators or just the ones with admin permission? + for collaborator in repo.get_collaborators() + } + remote_repo_relations_ids = [] + for account in self._get_social_accounts(collaborators.keys()): + remote_repo_relation, _ = RemoteRepositoryRelation.objects.get_or_create( + remote_repository=remote_repo, + account=account, + ) + remote_repo_relation.user = account.user + remote_repo_relation.admin = collaborators[account.uid].permissions.admin + remote_repo_relation.save() + remote_repo_relations_ids.append(remote_repo_relation.pk) + + # Remove collaborators that are no longer in the list. + RemoteRepositoryRelation.objects.filter( + remote_repository=remote_repo, + ).exclude( + pk__in=remote_repo_relations_ids, + ).delete() + + def _get_social_account(self, id): + return self._get_social_accounts([id]).first() + + def _get_social_accounts(self, ids): + return SocialAccount.objects.filter( + uid__in=ids, + provider=GitHubAppProvider.id, + ).select_related("user") + + def sync_organizations(self): + expected_organization = self.app_installation.account + if self.installation.target_type != GitHubAccountType.ORGANIZATION: + return diff --git a/readthedocs/oauth/urls.py b/readthedocs/oauth/urls.py new file mode 100644 index 00000000000..a464d303c16 --- /dev/null +++ b/readthedocs/oauth/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from readthedocs.oauth.views import GitHubAppWebhookView + +urlpatterns = [ + path("githubapp/", GitHubAppWebhookView.as_view(), name="github_app_webhook"), +] diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py new file mode 100644 index 00000000000..82c6b842a5b --- /dev/null +++ b/readthedocs/oauth/views.py @@ -0,0 +1,153 @@ +import hmac + +import structlog +from django.conf import settings +from rest_framework.exceptions import ValidationError +from rest_framework.response import Response +from rest_framework.views import APIView + +from readthedocs.api.v2.views.integrations import ( + GITHUB_EVENT_HEADER, + GITHUB_SIGNATURE_HEADER, + WebhookMixin, +) +from readthedocs.oauth.models import GitHubAppInstallation + +log = structlog.get_logger(__name__) + + +class GitHubAppWebhookView(APIView): + authentication_classes = [] + + def post(self, request): + if not self._is_payload_signature_valid(): + raise ValidationError("Invalid webhook signature") + + event = self.request.headers.get(GITHUB_EVENT_HEADER) + + event_handlers = { + "installation": self._handle_installation_event, + } + if event in event_handlers: + event_handlers[event]() + return Response(status=200) + raise ValidationError(f"Unsupported event: {event}") + + def _handle_installation_event(self): + """ + + .. code-block:: json + + { + "action": "created", + "installation": { + "id": 1234, + "client_id": "12345", + "account": { + "login": "user", + "id": 12345, + "type": "User" + }, + "repository_selection": "selected", + "html_url": "https://github.com/settings/installations/1234", + "app_id": 12345, + "app_slug": "app-slug", + "target_id": 12345, + "target_type": "User", + "permissions": { + "contents": "read", + "metadata": "read", + "pull_requests": "write", + "statuses": "write" + }, + "events": [ + "create", + "delete", + "public", + "pull_request", + "push" + ], + "created_at": "2025-01-29T12:04:11.000-05:00", + "updated_at": "2025-01-29T12:04:12.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [], + "suspended_by": null, + "suspended_at": null + }, + "repositories": [ + { + "id": 770738492, + "name": "test-public", + "full_name": "user/test-public", + "private": false + }, + { + "id": 917825438, + "name": "test-private", + "full_name": "user/test-private", + "private": true + } + ], + "requester": null, + "sender": { + "login": "user", + "id": 1234, + "type": "User" + } + } + + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#installation. + """ + data = self.request.data + + action = data.get("action") + installation = data.get("installation", {}) + + if action == "created": + gha__installation, created = GitHubAppInstallation.objects.get_or_create( + installation_id=installation["id"], + target_id=installation["target_id"], + target_type=installation["target_type"], + extra_data=data, + ) + # Sync repositories with the installation + # Do we do an incremental sync or a full sync? + elif action == "deleted": + gha_installation = GitHubAppInstallation.objects.filter( + installation_id=installation["id"] + ).first() + if not gha_installation: + raise ValidationError(f"Installation {installation['id']} not found") + + raise ValidationError(f"Unsupported action: {action}") + + def _sync_installation_repositories(self, installation): + pass + + def _add_installation_repositories( + self, installation: GitHubAppInstallation, repositories: list[dict] + ): + pass + + def _remove_installation_repositories(self, installation, repositories): + pass + + def _is_payload_signature_valid(self): + """ + GitHub uses a HMAC hexdigest hash to sign the payload. + + It is sent in the request's header. + See https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries. + """ + signature = self.request.headers.get(GITHUB_SIGNATURE_HEADER) + if not signature: + return False + + msg = self.request.body.decode() + secret = settings.GITHUB_APP_WEBHOOK_SECRET + digest = WebhookMixin.get_digest(secret, msg) + return hmac.compare_digest( + f"sha256={digest}".encode(), + signature.encode(), + ) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index aa7a688a516..7bb1d32ace9 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -748,6 +748,13 @@ def SOCIALACCOUNT_PROVIDERS(self): "signup": "readthedocs.forms.SignupFormWithNewsletter", } + GITHUB_APP_ID = 1234 + GITHUB_APP_PRIVATE_KEY = "" + GITHUB_APP_WEBHOOK_SECRET = "" + + def GITHUB_APP_CLIENT_ID(self): + return self.SOCIALACCOUNT_PROVIDERS["githubapp"]["APPS"][0]["client_id"] + # CORS # Don't allow sending cookies in cross-domain requests, this is so we can # relax our CORS headers for more views, but at the same time not opening diff --git a/readthedocs/urls.py b/readthedocs/urls.py index db638596019..f9329ba41a9 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -56,6 +56,7 @@ path("builds/", include("readthedocs.builds.urls")), # Put this as a unique path for the webhook, so we don't clobber existing Stripe URL's path("djstripe/", include("djstripe.urls", namespace="djstripe")), + path("webhook/", include("readthedocs.oauth.urls")), ] project_urls = [ diff --git a/requirements/pip.in b/requirements/pip.in index b37876668ac..da7a8f66dd8 100644 --- a/requirements/pip.in +++ b/requirements/pip.in @@ -48,6 +48,9 @@ pyyaml Pygments pydantic +# Used for GitHub API integration +PyGithub + dnspython # Used for Redis cache Django backend (`django.core.cache.backends.redis.RedisCache`) From 22de5d648c6679033b450ef1637953e5b3078de3 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 3 Feb 2025 17:05:30 -0500 Subject: [PATCH 03/92] Migration and stuff --- .../oauth/migrations/0017_githubapp.py | 39 +++++ readthedocs/oauth/models.py | 18 +-- readthedocs/oauth/services/githubapp.py | 72 ++++++--- readthedocs/oauth/views.py | 149 ++++++++++++++++-- requirements/deploy.txt | 19 +++ requirements/docker.txt | 19 +++ requirements/pip.in | 2 +- requirements/pip.txt | 19 ++- requirements/testing.txt | 19 +++ 9 files changed, 304 insertions(+), 52 deletions(-) create mode 100644 readthedocs/oauth/migrations/0017_githubapp.py diff --git a/readthedocs/oauth/migrations/0017_githubapp.py b/readthedocs/oauth/migrations/0017_githubapp.py new file mode 100644 index 00000000000..9f4300c8223 --- /dev/null +++ b/readthedocs/oauth/migrations/0017_githubapp.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.18 on 2025-02-03 21:58 +from django_safemigrate import Safe + +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields + + +class Migration(migrations.Migration): + + safe = Safe.before_deploy + + dependencies = [ + ('oauth', '0016_deprecate_old_vcs'), + ] + + operations = [ + migrations.CreateModel( + name='GitHubAppInstallation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), + ('installation_id', models.PositiveBigIntegerField(db_index=True, help_text='The application installation ID', unique=True)), + ('target_id', models.PositiveBigIntegerField(help_text='A GitHub account ID, it can be from a user or an organization')), + ('target_type', models.CharField(choices=[('User', 'User'), ('Organization', 'Organization')], help_text='Account type that the target_id belongs to (user or organization)', max_length=255)), + ('extra_data', models.JSONField(default=dict, help_text='Extra data returned by the webhook when the installation is created')), + ], + options={ + 'get_latest_by': 'modified', + 'abstract': False, + }, + ), + migrations.AddField( + model_name='remoterepository', + name='github_app_installation', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='repositories', to='oauth.githubappinstallation', verbose_name='GitHub App Installation'), + ), + ] diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 67fee5a90a6..7cbc45c997e 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -21,16 +21,13 @@ class GitHubAccountType(models.TextChoices): class GitHubAppInstallation(TimeStampedModel): - # Should we just use big int? - installation_id = models.CharField( + installation_id = models.PositiveBigIntegerField( help_text=_("The application installation ID"), - max_length=255, unique=True, db_index=True, ) - target_id = models.CharField( + target_id = models.PositiveBigIntegerField( help_text=_("A GitHub account ID, it can be from a user or an organization"), - max_length=255, ) target_type = models.CharField( help_text=_( @@ -47,12 +44,7 @@ class GitHubAppInstallation(TimeStampedModel): ) class Meta(TimeStampedModel.Meta): - constraints = [ - models.UniqueConstraint( - fields=["target_id", "target_type"], - name="unique_target_id_target_type", - ) - ] + pass class RemoteOrganization(TimeStampedModel): @@ -137,7 +129,6 @@ class Meta: class RemoteRepository(TimeStampedModel): - """ Remote importable repositories. @@ -221,6 +212,9 @@ class RemoteRepository(TimeStampedModel): blank=True, # Delete the repository if the installation is deleted? # or keep the repository and just remove the installation? + # I think we should keep the repository, but only if it's linked to a project, + # since a user could re-install the app, they shouldn't need to + # manually link each project to the repository again. on_delete=models.SET_NULL, ) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 2bf7cd0efeb..a1adf34e04b 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -1,3 +1,5 @@ +from functools import cached_property + import structlog from allauth.socialaccount.models import SocialAccount from django.conf import settings @@ -18,11 +20,12 @@ log = structlog.get_logger(__name__) + class GitHubAppService: vcs_provider_slug = GITHUB + def __init__(self, installation: GitHubAppInstallation): self.installation = installation - self._client = None def _get_auth(self): app_auth = Auth.AppAuth( @@ -34,27 +37,27 @@ def _get_auth(self): ) return app_auth - @property + @cached_property def client(self): """Return a client authenticated as the GitHub App to interact with most of the GH API""" - if self._client is None: - self._client = self.integration_client.get_github_for_installation(self.installation.installation_id) - return self._client + return self.integration_client.get_github_for_installation( + self.installation.installation_id + ) - @property + @cached_property def integration_client(self): """Return a client authenticated as the GitHub App to interact with the installation API""" - if self._integration_client is None: - self._integration_client = GithubIntegration(auth=self._get_auth()) - return self._integration_client + return GithubIntegration(auth=self._get_auth()) - @property + @cached_property def app_installation(self) -> GHInstallation: - return self.integration_client.get_app_installation(self.installation.installation_id) + return self.integration_client.get_app_installation( + self.installation.installation_id + ) def sync_repositories(self): - if self.installation.target_type != GitHubAccountType.USER: - return + # if self.installation.target_type != GitHubAccountType.USER: + # return return self._sync_installation_repositories() def _sync_installation_repositories(self): @@ -64,10 +67,12 @@ def _sync_installation_repositories(self): if remote_repo: remote_repositories.append(remote_repo) - # Remove repositories that are no longer in the list. + # Remove repositories that are no longer in the list, + # and that are not linked to a project. RemoteRepository.objects.filter( github_app_installation=self.installation, vcs_provider=self.vcs_provider_slug, + projects=None, ).exclude( pk__in=[repo.pk for repo in remote_repositories], ).delete() @@ -78,16 +83,40 @@ def add_repositories(self, repository_ids: list[int]): self.create_or_update_repository(repo) def remove_repositories(self, repository_ids: list[int]): + """ + Remove repositories from the given list that are not linked to a project. + + We don't remove repositories that are linked to a project, since a user could + grant access to the repository again, and we don't want users having to manually + link the project to the repository again. + """ RemoteRepository.objects.filter( github_app_installation=self.installation, vcs_provider=self.vcs_provider_slug, remote_id__in=repository_ids, + projects=None, ).delete() - def create_or_update_repository(self, repo: GHRepository) -> RemoteRepository | None + def create_or_update_repository( + self, repo: GHRepository + ) -> RemoteRepository | None: if not settings.ALLOW_PRIVATE_REPOS and repo.private: return + target_id = self.installation.target_id + target_type = self.installation.target_type + # NOTE: All the repositories should be owned by the installation account. + # This should never happen, unless our assumptions are wrong. + if repo.owner.id != target_id or repo.owner.type != target_type: + log.exception( + "Repository owner does not match the installation account", + repository_id=repo.id, + repository_owner_id=repo.owner.id, + installation_target_id=target_id, + installation_target_type=target_type, + ) + return + remote_repo, _ = RemoteRepository.objects.get_or_create( remote_id=str(repo.id), vcs_provider=self.vcs_provider_slug, @@ -99,7 +128,7 @@ def create_or_update_repository(self, repo: GHRepository) -> RemoteRepository | remote_repo.avatar_url = repo.owner.avatar_url remote_repo.ssh_url = repo.ssh_url remote_repo.html_url = repo.html_url - remote_repo.private = repo.private + remote_repo.private = repo.private remote_repo.default_branch = repo.default_branch # TODO: Do we need the SSH URL for private repositories now that we can clone using a token? @@ -107,7 +136,10 @@ def create_or_update_repository(self, repo: GHRepository) -> RemoteRepository | # NOTE: Only one installation of our APP should give access to a repository. # This should only happen if our data is out of sync. - if remote_repo.github_app_installation and remote_repo.github_app_installation != self.installation: + if ( + remote_repo.github_app_installation + and remote_repo.github_app_installation != self.installation + ): log.info( "Repository linked to another installation", repository_id=remote_repo.remote_id, @@ -121,7 +153,7 @@ def create_or_update_repository(self, repo: GHRepository) -> RemoteRepository | remote_repo.organization = self.create_or_update_organization(repo.owner) self._resync_collaborators(repo, remote_repo) - # What about memmbers of the organization? do we care? + # What about members of the organization? Do we care? # I think all of our permissions are based on the collaborators of the repository, # not the members of the organization. remote_repo.save() @@ -172,7 +204,7 @@ def _resync_collaborators(self, repo: GHRepository, remote_repo: RemoteRepositor def _get_social_account(self, id): return self._get_social_accounts([id]).first() - + def _get_social_accounts(self, ids): return SocialAccount.objects.filter( uid__in=ids, @@ -180,6 +212,6 @@ def _get_social_accounts(self, ids): ).select_related("user") def sync_organizations(self): - expected_organization = self.app_installation.account if self.installation.target_type != GitHubAccountType.ORGANIZATION: return + return self._sync_installation_repositories() diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 82c6b842a5b..038746548b2 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -12,6 +12,7 @@ WebhookMixin, ) from readthedocs.oauth.models import GitHubAppInstallation +from readthedocs.oauth.services.githubapp import GitHubAppService log = structlog.get_logger(__name__) @@ -27,6 +28,9 @@ def post(self, request): event_handlers = { "installation": self._handle_installation_event, + "installation_repositories": self._handle_installation_repositories_event, + # Hmm, don't think we need this one. + "installation_target": self._handle_installation_target_event, } if event in event_handlers: event_handlers[event]() @@ -35,6 +39,9 @@ def post(self, request): def _handle_installation_event(self): """ + Handle the installation event. + + Triggered when a user installs or uninstalls the GitHub App under an account (user or organization). .. code-block:: json @@ -100,38 +107,146 @@ def _handle_installation_event(self): See https://docs.github.com/en/webhooks/webhook-events-and-payloads#installation. """ data = self.request.data - action = data.get("action") installation = data.get("installation", {}) if action == "created": - gha__installation, created = GitHubAppInstallation.objects.get_or_create( - installation_id=installation["id"], - target_id=installation["target_id"], - target_type=installation["target_type"], - extra_data=data, - ) - # Sync repositories with the installation - # Do we do an incremental sync or a full sync? + gha_installation, _ = self._get_or_create_installation(installation, data) + # We do a full sync, since it's a new installation. + GitHubAppService(gha_installation).sync_repositories() elif action == "deleted": gha_installation = GitHubAppInstallation.objects.filter( installation_id=installation["id"] ).first() if not gha_installation: + # If we never created the installation, we can ignore the event. + # Maybe don't raise an error? raise ValidationError(f"Installation {installation['id']} not found") + gha_installation.delete() + # NOTE: should we handle the suspended/unsuspended/new_permissions_accepted actions? raise ValidationError(f"Unsupported action: {action}") - def _sync_installation_repositories(self, installation): - pass + def _handle_installation_repositories_event(self): + """ + Handle the installation_repositories event. + + Triggered when a repository is added or removed from an installation. - def _add_installation_repositories( - self, installation: GitHubAppInstallation, repositories: list[dict] - ): - pass + .. code-block:: json + { + "action": "added", + "installation": { + "id": 1234, + "client_id": "1234", + "account": { + "login": "user", + "id": 12345, + "type": "User" + }, + "repository_selection": "selected", + "html_url": "https://github.com/settings/installations/60240360", + "app_id": 12345, + "app_slug": "app-slug", + "target_id": 12345, + "target_type": "User", + "permissions": { + "contents": "read", + "metadata": "read", + "pull_requests": "write", + "statuses": "write" + }, + "events": ["create", "delete", "public", "pull_request", "push"], + "created_at": "2025-01-29T12:04:11.000-05:00", + "updated_at": "2025-01-29T16:05:45.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [], + "suspended_by": null, + "suspended_at": null + }, + "repository_selection": "selected", + "repositories_added": [ + { + "id": 258664698, + "name": "test-public", + "full_name": "user/test-public", + "private": false + } + ], + "repositories_removed": [], + "requester": null, + "sender": { + "login": "user", + "id": 4975310, + "type": "User" + } + } - def _remove_installation_repositories(self, installation, repositories): - pass + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#installation_repositories. + """ + data = self.request.data + action = data.get("action") + installation = data.get("installation", {}) + gha_installation, created = self._get_or_create_installation(installation, data) + service = GitHubAppService(gha_installation) + + if created: + # If we didn't have the installation, we do a full sync. + service.sync_repositories() + return + + if action == "added": + if data["repository_selection"] == "all": + service.sync_repositories() + else: + service.add_repositories( + [repo["id"] for repo in data["repositories_added"]] + ) + elif action == "removed": + service.remove_repositories( + [repo["id"] for repo in data["repositories_removed"]] + ) + + raise ValidationError(f"Unsupported action: {action}") + + def _handle_installation_target_event(self): + """ + Handle the installation_target event. + + Triggered when the target of an installation changes, + like when the user or organization changes its username/slug. + """ + + def _get_or_create_installation(self, installation: dict, extra_data: dict): + target_id = installation["target_id"] + target_type = installation["target_type"] + installation, created = GitHubAppInstallation.objects.get_or_create( + installation_id=installation["id"], + defaults={ + "extra_data": extra_data, + "target_id": target_id, + "target_type": target_type, + }, + ) + # NOTE: An installation can't change its target_id or target_type. + # This should never happen, unless our assumptions are wrong. + if ( + installation.target_id != target_id + or installation.target_type != target_type + ): + log.exception( + "Installation target_id or target_type changed", + installation_id=installation.installation_id, + target_id=installation.target_id, + target_type=installation.target_type, + new_target_id=target_id, + new_target_type=target_type, + ) + installation.target_id = target_id + installation.target_type = target_type + installation.save() + return installation, created def _is_payload_signature_valid(self): """ diff --git a/requirements/deploy.txt b/requirements/deploy.txt index 9de4267fad8..9e7ac53fc01 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -53,6 +53,7 @@ cffi==1.17.1 # via # -r requirements/pip.txt # cryptography + # pynacl charset-normalizer==3.4.1 # via # -r requirements/pip.txt @@ -96,6 +97,10 @@ cssselect==1.2.0 # pyquery decorator==5.1.1 # via ipython +deprecated==1.2.18 + # via + # -r requirements/pip.txt + # pygithub distlib==0.3.9 # via # -r requirements/pip.txt @@ -322,6 +327,8 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic +pygithub==2.5.0 + # via -r requirements/pip.txt pygments==2.19.1 # via # -r requirements/pip.txt @@ -330,6 +337,11 @@ pyjwt[crypto]==2.10.1 # via # -r requirements/pip.txt # django-allauth + # pygithub +pynacl==1.5.0 + # via + # -r requirements/pip.txt + # pygithub pyquery==2.0.1 # via -r requirements/pip.txt python-crontab==3.2.0 @@ -367,6 +379,7 @@ requests==2.30.0 # -r requirements/pip.txt # django-allauth # docker + # pygithub # requests-oauthlib # requests-toolbelt # slumber @@ -436,6 +449,7 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # pygithub tzdata==2025.1 # via # -r requirements/pip.txt @@ -461,6 +475,7 @@ urllib3==2.3.0 # botocore # docker # elastic-transport + # pygithub # requests # sentry-sdk user-agents==2.2.0 @@ -481,6 +496,10 @@ websocket-client==1.8.0 # via # -r requirements/pip.txt # docker +wrapt==1.17.2 + # via + # -r requirements/pip.txt + # deprecated xmlsec==1.3.14 # via # -r requirements/pip.txt diff --git a/requirements/docker.txt b/requirements/docker.txt index ddee33eb4ab..9e3c924b92b 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -56,6 +56,7 @@ cffi==1.17.1 # via # -r requirements/pip.txt # cryptography + # pynacl chardet==5.2.0 # via tox charset-normalizer==3.4.1 @@ -106,6 +107,10 @@ decorator==5.1.1 # via # ipdb # ipython +deprecated==1.2.18 + # via + # -r requirements/pip.txt + # pygithub distlib==0.3.9 # via # -r requirements/pip.txt @@ -346,6 +351,8 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic +pygithub==2.5.0 + # via -r requirements/pip.txt pygments==2.19.1 # via # -r requirements/pip.txt @@ -356,6 +363,11 @@ pyjwt[crypto]==2.10.1 # via # -r requirements/pip.txt # django-allauth + # pygithub +pynacl==1.5.0 + # via + # -r requirements/pip.txt + # pygithub pyproject-api==1.9.0 # via tox pyquery==2.0.1 @@ -397,6 +409,7 @@ requests==2.30.0 # -r requirements/pip.txt # django-allauth # docker + # pygithub # requests-oauthlib # requests-toolbelt # slumber @@ -468,6 +481,7 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # pygithub # rich # tox tzdata==2025.1 @@ -495,6 +509,7 @@ urllib3==2.3.0 # botocore # docker # elastic-transport + # pygithub # requests user-agents==2.2.0 # via -r requirements/pip.txt @@ -518,6 +533,10 @@ websocket-client==1.8.0 # docker wmctrl==0.5 # via pdbpp +wrapt==1.17.2 + # via + # -r requirements/pip.txt + # deprecated xmlsec==1.3.14 # via # -r requirements/pip.txt diff --git a/requirements/pip.in b/requirements/pip.in index da7a8f66dd8..02f93849f3c 100644 --- a/requirements/pip.in +++ b/requirements/pip.in @@ -48,7 +48,7 @@ pyyaml Pygments pydantic -# Used for GitHub API integration +# Used to interact with the GitHub API PyGithub dnspython diff --git a/requirements/pip.txt b/requirements/pip.txt index 37d0352affe..c9d0c10caab 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -34,7 +34,9 @@ certifi==2024.12.14 # elastic-transport # requests cffi==1.17.1 - # via cryptography + # via + # cryptography + # pynacl charset-normalizer==3.4.1 # via requests click==8.1.8 @@ -60,6 +62,8 @@ cryptography==44.0.0 # pyjwt cssselect==1.2.0 # via pyquery +deprecated==1.2.18 + # via pygithub distlib==0.3.9 # via virtualenv dj-pagination==2.5.0 @@ -230,10 +234,16 @@ pydantic==2.10.6 # via -r requirements/pip.in pydantic-core==2.27.2 # via pydantic +pygithub==2.5.0 + # via -r requirements/pip.in pygments==2.19.1 # via -r requirements/pip.in pyjwt[crypto]==2.10.1 - # via django-allauth + # via + # django-allauth + # pygithub +pynacl==1.5.0 + # via pygithub pyquery==2.0.1 # via -r requirements/pip.in python-crontab==3.2.0 @@ -265,6 +275,7 @@ requests==2.30.0 # -r requirements/pip.in # django-allauth # docker + # pygithub # requests-oauthlib # requests-toolbelt # slumber @@ -313,6 +324,7 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # pygithub tzdata==2025.1 # via # -r requirements/pip.in @@ -331,6 +343,7 @@ urllib3==2.3.0 # botocore # docker # elastic-transport + # pygithub # requests user-agents==2.2.0 # via -r requirements/pip.in @@ -345,6 +358,8 @@ wcwidth==0.2.13 # via prompt-toolkit websocket-client==1.8.0 # via docker +wrapt==1.17.2 + # via deprecated xmlsec==1.3.14 # via python3-saml diff --git a/requirements/testing.txt b/requirements/testing.txt index eccee1e0f20..f6eada4e71a 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -54,6 +54,7 @@ cffi==1.17.1 # via # -r requirements/pip.txt # cryptography + # pynacl charset-normalizer==3.4.1 # via # -r requirements/pip.txt @@ -101,6 +102,10 @@ cython==3.0.11 # via sphinx defusedxml==0.7.1 # via sphinx +deprecated==1.2.18 + # via + # -r requirements/pip.txt + # pygithub distlib==0.3.9 # via # -r requirements/pip.txt @@ -324,6 +329,8 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic +pygithub==2.5.0 + # via -r requirements/pip.txt pygments==2.19.1 # via # -r requirements/pip.txt @@ -332,6 +339,11 @@ pyjwt[crypto]==2.10.1 # via # -r requirements/pip.txt # django-allauth + # pygithub +pynacl==1.5.0 + # via + # -r requirements/pip.txt + # pygithub pyquery==2.0.1 # via -r requirements/pip.txt pytest==8.3.4 @@ -385,6 +397,7 @@ requests==2.30.0 # -r requirements/pip.txt # django-allauth # docker + # pygithub # requests-mock # requests-oauthlib # requests-toolbelt @@ -465,6 +478,7 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # pygithub # sphinx tzdata==2025.1 # via @@ -491,6 +505,7 @@ urllib3==2.3.0 # botocore # docker # elastic-transport + # pygithub # requests user-agents==2.2.0 # via -r requirements/pip.txt @@ -510,6 +525,10 @@ websocket-client==1.8.0 # via # -r requirements/pip.txt # docker +wrapt==1.17.2 + # via + # -r requirements/pip.txt + # deprecated xmlsec==1.3.14 # via # -r requirements/pip.txt From 5119e483f1d06a745d311aebd0d885fb66b25502 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 3 Feb 2025 17:48:36 -0500 Subject: [PATCH 04/92] Fixes --- readthedocs/oauth/admin.py | 3 +++ readthedocs/oauth/services/githubapp.py | 8 ++++---- readthedocs/oauth/views.py | 10 ++++++++-- readthedocs/settings/base.py | 1 + readthedocs/settings/docker_compose.py | 11 +++++++++++ 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/readthedocs/oauth/admin.py b/readthedocs/oauth/admin.py index 7782468219a..7c3da78a4d7 100644 --- a/readthedocs/oauth/admin.py +++ b/readthedocs/oauth/admin.py @@ -3,12 +3,15 @@ from django.contrib import admin from .models import ( + GitHubAppInstallation, RemoteOrganization, RemoteOrganizationRelation, RemoteRepository, RemoteRepositoryRelation, ) +admin.site.register(GitHubAppInstallation) + @admin.register(RemoteRepository) class RemoteRepositoryAdmin(admin.ModelAdmin): diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index a1adf34e04b..a6e59d42f32 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -188,10 +188,13 @@ def _resync_collaborators(self, repo: GHRepository, remote_repo: RemoteRepositor for account in self._get_social_accounts(collaborators.keys()): remote_repo_relation, _ = RemoteRepositoryRelation.objects.get_or_create( remote_repository=remote_repo, + user=account.user, account=account, ) remote_repo_relation.user = account.user - remote_repo_relation.admin = collaborators[account.uid].permissions.admin + remote_repo_relation.admin = collaborators[ + int(account.uid) + ].permissions.admin remote_repo_relation.save() remote_repo_relations_ids.append(remote_repo_relation.pk) @@ -202,9 +205,6 @@ def _resync_collaborators(self, repo: GHRepository, remote_repo: RemoteRepositor pk__in=remote_repo_relations_ids, ).delete() - def _get_social_account(self, id): - return self._get_social_accounts([id]).first() - def _get_social_accounts(self, ids): return SocialAccount.objects.filter( uid__in=ids, diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 038746548b2..5b0f371c293 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -114,7 +114,9 @@ def _handle_installation_event(self): gha_installation, _ = self._get_or_create_installation(installation, data) # We do a full sync, since it's a new installation. GitHubAppService(gha_installation).sync_repositories() - elif action == "deleted": + return + + if action == "deleted": gha_installation = GitHubAppInstallation.objects.filter( installation_id=installation["id"] ).first() @@ -123,6 +125,7 @@ def _handle_installation_event(self): # Maybe don't raise an error? raise ValidationError(f"Installation {installation['id']} not found") gha_installation.delete() + return # NOTE: should we handle the suspended/unsuspended/new_permissions_accepted actions? raise ValidationError(f"Unsupported action: {action}") @@ -203,10 +206,13 @@ def _handle_installation_repositories_event(self): service.add_repositories( [repo["id"] for repo in data["repositories_added"]] ) - elif action == "removed": + return + + if action == "removed": service.remove_repositories( [repo["id"] for repo in data["repositories_removed"]] ) + return raise ValidationError(f"Unsupported action: {action}") diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 7bb1d32ace9..8adac805996 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -752,6 +752,7 @@ def SOCIALACCOUNT_PROVIDERS(self): GITHUB_APP_PRIVATE_KEY = "" GITHUB_APP_WEBHOOK_SECRET = "" + @property def GITHUB_APP_CLIENT_ID(self): return self.SOCIALACCOUNT_PROVIDERS["githubapp"]["APPS"][0]["client_id"] diff --git a/readthedocs/settings/docker_compose.py b/readthedocs/settings/docker_compose.py index 579d4bcf874..a7c38a96c9e 100644 --- a/readthedocs/settings/docker_compose.py +++ b/readthedocs/settings/docker_compose.py @@ -1,5 +1,6 @@ import os import socket +from pathlib import Path from .base import CommunityBaseSettings @@ -225,6 +226,16 @@ def SOCIALACCOUNT_PROVIDERS(self): pass return providers + GITHUB_APP_ID = os.environ.get("RTD_GITHUB_APP_ID") + GITHUB_APP_WEBHOOK_SECRET = os.environ.get("RTD_GITHUB_APP_WEBHOOK_SECRET") + + @property + def GITHUB_APP_PRIVATE_KEY(self): + pem_file = os.environ.get("RTD_GITHUB_APP_PRIVATE_KEY_PATH") + if not pem_file: + return "" + return Path(pem_file).read_text() + RTD_SAVE_BUILD_COMMANDS_TO_STORAGE = True RTD_BUILD_COMMANDS_STORAGE = "readthedocs.storage.s3_storage.S3BuildCommandsStorage" BUILD_COLD_STORAGE_URL = "http://storage:9000/builds" From a854c7e6604e92b03af948129532a1e69cffd9fb Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 4 Feb 2025 13:56:26 -0500 Subject: [PATCH 05/92] More hooks! --- readthedocs/core/views/hooks.py | 42 ++++- readthedocs/oauth/models.py | 9 ++ readthedocs/oauth/services/githubapp.py | 2 +- readthedocs/oauth/views.py | 206 ++++++++++++++++++++---- readthedocs/projects/models.py | 34 +++- readthedocs/settings/docker_compose.py | 9 +- 6 files changed, 258 insertions(+), 44 deletions(-) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 3f9897a9c57..f77414e99bb 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -15,7 +15,7 @@ log = structlog.get_logger(__name__) -def _build_version(project, slug, already_built=()): +def _build_version(project, version): """ Where we actually trigger builds for a project and slug. @@ -27,15 +27,14 @@ def _build_version(project, slug, already_built=()): # Previously we were building the latest version (inactive or active) # when building the default version, # some users may have relied on this to update the version list #4450 - version = project.versions.filter(active=True, slug=slug).first() - if version and slug not in already_built: + if version.active: log.info( "Building.", project_slug=project.slug, version_slug=version.slug, ) trigger_build(project=project, version=version) - return slug + return version.slug log.info("Not building.", version_slug=slug) return None @@ -45,6 +44,10 @@ def build_branches(project, branch_list): """ Build the branches for a specific project. + .. warning:: + + Deprecated, use ``build_versions_from_names`` instead. + Returns: to_build - a list of branches that were built not_building - a list of branches that we won't build @@ -59,7 +62,9 @@ def build_branches(project, branch_list): project_slug=project.slug, version_slug=version.slug, ) - ret = _build_version(project, version.slug, already_built=to_build) + if version.slug in to_build: + continue + ret = _build_version(project, version) if ret: to_build.add(ret) else: @@ -67,6 +72,33 @@ def build_branches(project, branch_list): return (to_build, not_building) +def build_versions_from_names(project, version_names: list[tuple[str, str]]): + """ + Build the branches or tags from the project. + + :param project: Project instance + :param version_names: A list of tuples with the version name and type. + :returns: A tuple with the versions that were built and the versions that were not built. + """ + to_build = set() + not_building = set() + for version_name, version_type in version_names: + for version in project.versions_from_name(version_name, version_type): + log.debug( + "Processing.", + project_slug=project.slug, + version_slug=version.slug, + ) + if version.slug in to_build: + continue + triggered = _build_version(project, version) + if triggered: + to_build.add(triggered) + else: + not_building.add(version.slug) + return to_build, not_building + + def trigger_sync_versions(project): """ Sync the versions of a repo using its latest version. diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 7cbc45c997e..6b07742791e 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -1,5 +1,7 @@ """OAuth service models.""" +from functools import cached_property + from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import User from django.core.validators import URLValidator @@ -46,6 +48,13 @@ class GitHubAppInstallation(TimeStampedModel): class Meta(TimeStampedModel.Meta): pass + @cached_property + def service(self): + """Return the service for this installation.""" + from readthedocs.oauth.services.githubapp import GitHubAppService + + return GitHubAppService(self) + class RemoteOrganization(TimeStampedModel): """ diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index a6e59d42f32..fc91d1300da 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -106,7 +106,7 @@ def create_or_update_repository( target_id = self.installation.target_id target_type = self.installation.target_type # NOTE: All the repositories should be owned by the installation account. - # This should never happen, unless our assumptions are wrong. + # This should never happen, unless this assumption is wrong. if repo.owner.id != target_id or repo.owner.type != target_type: log.exception( "Repository owner does not match the installation account", diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 5b0f371c293..54bb194a32c 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -9,10 +9,19 @@ from readthedocs.api.v2.views.integrations import ( GITHUB_EVENT_HEADER, GITHUB_SIGNATURE_HEADER, + ExternalVersionData, WebhookMixin, ) +from readthedocs.builds.constants import BRANCH, TAG +from readthedocs.core.views.hooks import ( + build_external_version, + build_versions_from_names, + close_external_version, + get_or_create_external_version, + trigger_sync_versions, +) from readthedocs.oauth.models import GitHubAppInstallation -from readthedocs.oauth.services.githubapp import GitHubAppService +from readthedocs.projects.models import Project log = structlog.get_logger(__name__) @@ -31,6 +40,12 @@ def post(self, request): "installation_repositories": self._handle_installation_repositories_event, # Hmm, don't think we need this one. "installation_target": self._handle_installation_target_event, + # "create": self._handle_create_event, + # "delete": self._handle_delete_event, + # TODO: this triggers when a branch or tag is deleted, + # do we need to handle the delete and create events as well? + "push": self._handle_push_event, + "pull_request": self._handle_pull_request_event, } if event in event_handlers: event_handlers[event]() @@ -107,13 +122,16 @@ def _handle_installation_event(self): See https://docs.github.com/en/webhooks/webhook-events-and-payloads#installation. """ data = self.request.data - action = data.get("action") - installation = data.get("installation", {}) + action = data["action"] + installation = data["installation"] if action == "created": - gha_installation, _ = self._get_or_create_installation(installation, data) - # We do a full sync, since it's a new installation. - GitHubAppService(gha_installation).sync_repositories() + gha_installation, created = self._get_or_create_installation() + if not created: + log.info( + "Installation already exists", installation_id=installation["id"] + ) + return if action == "deleted": @@ -189,27 +207,24 @@ def _handle_installation_repositories_event(self): See https://docs.github.com/en/webhooks/webhook-events-and-payloads#installation_repositories. """ data = self.request.data - action = data.get("action") - installation = data.get("installation", {}) - gha_installation, created = self._get_or_create_installation(installation, data) - service = GitHubAppService(gha_installation) + action = data["action"] + gha_installation, created = self._get_or_create_installation() + # If we didn't have the installation, all repositories were synced on creation. if created: - # If we didn't have the installation, we do a full sync. - service.sync_repositories() return if action == "added": if data["repository_selection"] == "all": - service.sync_repositories() + gha_installation.service.sync_repositories() else: - service.add_repositories( + gha_installation.service.add_repositories( [repo["id"] for repo in data["repositories_added"]] ) return if action == "removed": - service.remove_repositories( + gha_installation.service.remove_repositories( [repo["id"] for repo in data["repositories_removed"]] ) return @@ -224,35 +239,168 @@ def _handle_installation_target_event(self): like when the user or organization changes its username/slug. """ - def _get_or_create_installation(self, installation: dict, extra_data: dict): + def _handle_create_event(self): + """ + Handle the create event. + + Triggered when a branch or tag is created. + + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#create. + """ + self._sync_repo_versions() + + def _handle_delete_event(self): + """ + Handle the delete event. + + Triggered when a branch or tag is deleted. + + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#delete. + """ + self._sync_repo_versions() + + def _sync_repo_versions(self): + for project in self._get_projects(): + trigger_sync_versions(project) + + def _handle_push_event(self): + """ + Handle the push event. + + Triggered when a commit is pushed, when a commit tag is pushed, + when a branch is deleted, when a tag is deleted, + or when a repository is created from a template. + + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#push. + """ + data = self.request.data + created = data.get("created", False) + deleted = data.get("deleted", False) + if created or deleted: + self._sync_repo_versions() + return + + version_name, version_type = self._parse_version_from_ref(data["ref"]) + for project in self._get_projects(): + build_versions_from_names(project, [(version_name, version_type)]) + + def _parse_version_from_ref(self, ref: str): + """ + Parse the version name and type from a GitHub ref. + + The ref can be a branch or a tag. + + :param ref: The ref to parse. + :returns: A tuple with the version name and type. + """ + heads_prefix = "refs/heads/" + tags_prefix = "refs/tags/" + if ref.startswith(heads_prefix): + return ref.removeprefix(heads_prefix), BRANCH + if ref.startswith(tags_prefix): + return ref.removeprefix(tags_prefix), TAG + + # NOTE: this should never happen. + raise ValidationError(f"Invalid ref: {ref}") + + def _handle_pull_request_event(self): + """ + Handle the pull_request event. + + Triggered when there is activity on a pull request. + + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#pull_request. + """ + data = self.request.data + action = data["action"] + + pr = data["pull_request"] + ExternalVersionData( + id=str(pr["number"]), + commit=pr["head"]["sha"], + source_branch=pr["head"]["ref"], + base_branch=pr["base"]["ref"], + ) + + if action in ("opened", "reopened", "synchronize"): + for project in self._get_projects(): + external_version = get_or_create_external_version( + project=project, + version_data=ExternalVersionData, + ) + build_external_version(project, external_version) + return + + if action == "closed": + # Queue the external version for deletion. + for project in self._get_projects(): + close_external_version( + project=project, + version_data=ExternalVersionData, + ) + return + + # We don't need to handle the other actions. + return + + def _get_projects(self): + remote_repository = self._get_remote_repository() + if not remote_repository: + return Project.objects.none() + return remote_repository.projects.all() + + def _get_remote_repository(self): + """ + Get the remote repository from the request data. + + If the repository doesn't exist, return None. + """ + data = self.request.data + remote_id = data["repository"]["id"] + gha_installation, _ = self._get_or_create_installation() + return gha_installation.repositories.filter(remote_id=remote_id).first() + + def _get_or_create_installation(self, sync_repositories_on_create: bool = True): + """ + Get or create the GitHub App installation. + + If the installation didn't exist, and `sync_repositories_on_create` is True, + we sync the repositories. + """ + data = self.request.data + # All webhook payloads should have an installation object. + installation = data["installation"] + installation_id = installation["id"] target_id = installation["target_id"] target_type = installation["target_type"] - installation, created = GitHubAppInstallation.objects.get_or_create( - installation_id=installation["id"], + gha_installation, created = GitHubAppInstallation.objects.get_or_create( + installation_id=installation_id, defaults={ - "extra_data": extra_data, + "extra_data": data, "target_id": target_id, "target_type": target_type, }, ) # NOTE: An installation can't change its target_id or target_type. - # This should never happen, unless our assumptions are wrong. + # This should never happen, unless this assumption is wrong. if ( - installation.target_id != target_id - or installation.target_type != target_type + gha_installation.target_id != target_id + or gha_installation.target_type != target_type ): log.exception( "Installation target_id or target_type changed", - installation_id=installation.installation_id, - target_id=installation.target_id, - target_type=installation.target_type, + installation_id=gha_installation.installation_id, + target_id=gha_installation.target_id, + target_type=gha_installation.target_type, new_target_id=target_id, new_target_type=target_type, ) - installation.target_id = target_id - installation.target_type = target_type - installation.save() - return installation, created + gha_installation.target_id = target_id + gha_installation.target_type = target_type + gha_installation.save() + if created and sync_repositories_on_create: + gha_installation.service.sync_repositories() + return gha_installation, created def _is_payload_signature_valid(self): """ diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index d016dcc7b80..7544a3bea34 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -13,6 +13,7 @@ from django.contrib.contenttypes.fields import GenericRelation from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +from django.db.models import Q from django.urls import reverse from django.utils import timezone from django.utils.crypto import get_random_string @@ -1229,13 +1230,44 @@ def update_stable_version(self): ) return new_stable + def versions_from_name(self, name, type=None): + """ + Get all versions attached to the branch or tag name. + + Normally, only one version should be returned, but since LATEST and STABLE + are aliases for the branch/tag, they may be returned as well. + """ + queryset = self.versions(manager=INTERNAL) + queryset = queryset.filter( + # Normal branches + Q(verbose_name=name, machine=False) + # Latest and stable have the name of the branch/tag in the identifier. + # NOTE: if stable is a branch, identifier will be the commit hash, + # so we don't have a way to match it by name. + # We should do another lookup to get the original stable version, + # or change our logic to store the tags name in the identifier of stable. + | Q(identifier=name, machine=True) + ) + + if type: + queryset = queryset.filter(type=type) + + return queryset.distinct() + def versions_from_branch_name(self, branch): + """ + Get all versions attached to the branch or tag name. + + .. warning:: + + Deprecated, use ``versions_from_name`` instead. + """ return ( self.versions.filter(identifier=branch) | self.versions.filter(identifier="remotes/origin/%s" % branch) | self.versions.filter(identifier="origin/%s" % branch) | self.versions.filter(verbose_name=branch) - ) + ).distinct() def get_default_version(self): """ diff --git a/readthedocs/settings/docker_compose.py b/readthedocs/settings/docker_compose.py index a7c38a96c9e..f1607fdb659 100644 --- a/readthedocs/settings/docker_compose.py +++ b/readthedocs/settings/docker_compose.py @@ -1,6 +1,5 @@ import os import socket -from pathlib import Path from .base import CommunityBaseSettings @@ -228,13 +227,7 @@ def SOCIALACCOUNT_PROVIDERS(self): GITHUB_APP_ID = os.environ.get("RTD_GITHUB_APP_ID") GITHUB_APP_WEBHOOK_SECRET = os.environ.get("RTD_GITHUB_APP_WEBHOOK_SECRET") - - @property - def GITHUB_APP_PRIVATE_KEY(self): - pem_file = os.environ.get("RTD_GITHUB_APP_PRIVATE_KEY_PATH") - if not pem_file: - return "" - return Path(pem_file).read_text() + GITHUB_APP_PRIVATE_KEY = os.environ.get("RTD_GITHUB_APP_PRIVATE_KEY") RTD_SAVE_BUILD_COMMANDS_TO_STORAGE = True RTD_BUILD_COMMANDS_STORAGE = "readthedocs.storage.s3_storage.S3BuildCommandsStorage" From 7235cd8b935e980270726bcc62fe38c183b391b7 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 4 Feb 2025 14:48:24 -0500 Subject: [PATCH 06/92] Bug fixes and refactor --- readthedocs/core/views/hooks.py | 2 +- readthedocs/oauth/services/githubapp.py | 36 +++++++++++++------------ readthedocs/oauth/views.py | 16 +++++++++-- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index f77414e99bb..7165e7400e1 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -36,7 +36,7 @@ def _build_version(project, version): trigger_build(project=project, version=version) return version.slug - log.info("Not building.", version_slug=slug) + log.info("Not building.", version_slug=version.slug) return None diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index fc91d1300da..167ad94caa4 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -21,11 +21,9 @@ log = structlog.get_logger(__name__) -class GitHubAppService: - vcs_provider_slug = GITHUB - - def __init__(self, installation: GitHubAppInstallation): - self.installation = installation +class GitHubAppClient: + def __init__(self, installation_id: int): + self.installation_id = installation_id def _get_auth(self): app_auth = Auth.AppAuth( @@ -37,23 +35,27 @@ def _get_auth(self): ) return app_auth - @cached_property - def client(self): - """Return a client authenticated as the GitHub App to interact with most of the GH API""" - return self.integration_client.get_github_for_installation( - self.installation.installation_id - ) - @cached_property def integration_client(self): """Return a client authenticated as the GitHub App to interact with the installation API""" return GithubIntegration(auth=self._get_auth()) + @cached_property + def client(self): + """Return a client authenticated as the GitHub App to interact with most of the GH API""" + return self.integration_client.get_github_for_installation(self.installation_id) + @cached_property def app_installation(self) -> GHInstallation: - return self.integration_client.get_app_installation( - self.installation.installation_id - ) + return self.integration_client.get_app_installation(self.installation_id) + + +class GitHubAppService: + vcs_provider_slug = GITHUB + + def __init__(self, installation: GitHubAppInstallation): + self.installation = installation + self.gha_client = GitHubAppClient(self.installation.installation_id) def sync_repositories(self): # if self.installation.target_type != GitHubAccountType.USER: @@ -62,7 +64,7 @@ def sync_repositories(self): def _sync_installation_repositories(self): remote_repositories = [] - for repo in self.app_installation.get_repos(): + for repo in self.gha_client.app_installation.get_repos(): remote_repo = self.create_or_update_repository(repo) if remote_repo: remote_repositories.append(remote_repo) @@ -79,7 +81,7 @@ def _sync_installation_repositories(self): def add_repositories(self, repository_ids: list[int]): for repository_id in repository_ids: - repo = self.client.get_repo(repository_id) + repo = self.gha_client.client.get_repo(repository_id) self.create_or_update_repository(repo) def remove_repositories(self, repository_ids: list[int]): diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 54bb194a32c..84ba38a7a5d 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -21,6 +21,7 @@ trigger_sync_versions, ) from readthedocs.oauth.models import GitHubAppInstallation +from readthedocs.oauth.services.githubapp import GitHubAppClient from readthedocs.projects.models import Project log = structlog.get_logger(__name__) @@ -371,8 +372,19 @@ def _get_or_create_installation(self, sync_repositories_on_create: bool = True): # All webhook payloads should have an installation object. installation = data["installation"] installation_id = installation["id"] - target_id = installation["target_id"] - target_type = installation["target_type"] + + # These fields are not always present in all payloads. + target_id = installation.get("target_id") + target_type = installation.get("target_type") + # If they aren't present, fetch them from the API, + # so we can create the installation object if needed. + if not target_id or not target_type: + installation = GitHubAppClient(installation_id).app_installation + target_id = installation.target_id + target_type = installation.target_type + data = data.copy() + data["installation"] = installation.raw_data + gha_installation, created = GitHubAppInstallation.objects.get_or_create( installation_id=installation_id, defaults={ From 632e2e505d87a0e3b11352d4a858217718cc83d2 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 4 Feb 2025 14:49:20 -0500 Subject: [PATCH 07/92] Format --- readthedocs/allauth/providers/githubapp/provider.py | 2 ++ readthedocs/allauth/providers/githubapp/urls.py | 1 - readthedocs/allauth/providers/githubapp/views.py | 4 ++-- readthedocs/core/adapters.py | 7 ++----- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/readthedocs/allauth/providers/githubapp/provider.py b/readthedocs/allauth/providers/githubapp/provider.py index cf5958a7826..f785635ceb7 100644 --- a/readthedocs/allauth/providers/githubapp/provider.py +++ b/readthedocs/allauth/providers/githubapp/provider.py @@ -1,4 +1,5 @@ from allauth.socialaccount.providers.github.provider import GitHubProvider + from readthedocs.allauth.providers.githubapp.views import GitHubAppOAuth2Adapter @@ -7,4 +8,5 @@ class GitHubAppProvider(GitHubProvider): name = "GitHub App" oauth2_adapter_class = GitHubAppOAuth2Adapter + provider_classes = [GitHubAppProvider] diff --git a/readthedocs/allauth/providers/githubapp/urls.py b/readthedocs/allauth/providers/githubapp/urls.py index 3ab15feaae2..49d8a5f9514 100644 --- a/readthedocs/allauth/providers/githubapp/urls.py +++ b/readthedocs/allauth/providers/githubapp/urls.py @@ -2,5 +2,4 @@ from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider - urlpatterns = default_urlpatterns(GitHubAppProvider) diff --git a/readthedocs/allauth/providers/githubapp/views.py b/readthedocs/allauth/providers/githubapp/views.py index a5d1377c648..b0165884cae 100644 --- a/readthedocs/allauth/providers/githubapp/views.py +++ b/readthedocs/allauth/providers/githubapp/views.py @@ -1,12 +1,12 @@ from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter - from allauth.socialaccount.providers.oauth2.views import ( OAuth2CallbackView, OAuth2LoginView, ) + class GitHubAppOAuth2Adapter(GitHubOAuth2Adapter): - provider_id = 'githubapp' + provider_id = "githubapp" oauth2_login = OAuth2LoginView.adapter_view(GitHubAppOAuth2Adapter) diff --git a/readthedocs/core/adapters.py b/readthedocs/core/adapters.py index 479befe0285..0e2b6d27b20 100644 --- a/readthedocs/core/adapters.py +++ b/readthedocs/core/adapters.py @@ -1,12 +1,10 @@ """Allauth overrides.""" -from allauth.socialaccount.adapter import DefaultSocialAccountAdapter -from allauth.socialaccount.models import SocialAccount -from allauth.socialaccount.providers.github.provider import GitHubProvider - import structlog from allauth.account.adapter import DefaultAccountAdapter from allauth.socialaccount.adapter import DefaultSocialAccountAdapter +from allauth.socialaccount.models import SocialAccount +from allauth.socialaccount.providers.github.provider import GitHubProvider from django.utils.encoding import force_str from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider @@ -69,7 +67,6 @@ def pre_social_login(self, request, sociallogin): """ sociallogin.email_addresses = [ email for email in sociallogin.email_addresses if email.primary - ] provider = sociallogin.account.get_provider() From b0e2181fe0d9fa640b024940f9782aa57a36c230 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 4 Feb 2025 18:24:11 -0500 Subject: [PATCH 08/92] More events --- readthedocs/oauth/services/githubapp.py | 3 + readthedocs/oauth/views.py | 101 +++++++++++++++++++----- 2 files changed, 85 insertions(+), 19 deletions(-) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 167ad94caa4..45784aa5516 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -102,6 +102,9 @@ def remove_repositories(self, repository_ids: list[int]): def create_or_update_repository( self, repo: GHRepository ) -> RemoteRepository | None: + # What about a project that is public, and then becomes private? + # I think we should allow creating remote repositories for these, + # but block import/clone and other operations. if not settings.ALLOW_PRIVATE_REPOS and repo.private: return diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 84ba38a7a5d..b90b6317988 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -41,12 +41,11 @@ def post(self, request): "installation_repositories": self._handle_installation_repositories_event, # Hmm, don't think we need this one. "installation_target": self._handle_installation_target_event, - # "create": self._handle_create_event, - # "delete": self._handle_delete_event, - # TODO: this triggers when a branch or tag is deleted, - # do we need to handle the delete and create events as well? "push": self._handle_push_event, "pull_request": self._handle_pull_request_event, + "repository": self._handle_repository_event, + "organization": self._handle_organization_event, + "member": self._handle_member_event, } if event in event_handlers: event_handlers[event]() @@ -146,8 +145,10 @@ def _handle_installation_event(self): gha_installation.delete() return - # NOTE: should we handle the suspended/unsuspended/new_permissions_accepted actions? - raise ValidationError(f"Unsupported action: {action}") + # Ignore other actions: + # - new_permissions_accepted: We don't need to do anything here for now. + # - suspended/unsuspended: We don't do anything with suspended installations. + return def _handle_installation_repositories_event(self): """ @@ -230,6 +231,7 @@ def _handle_installation_repositories_event(self): ) return + # NOTE: this should never happen. raise ValidationError(f"Unsupported action: {action}") def _handle_installation_target_event(self): @@ -240,25 +242,33 @@ def _handle_installation_target_event(self): like when the user or organization changes its username/slug. """ - def _handle_create_event(self): + def _handle_repository_event(self): """ - Handle the create event. + Handle the repository event. - Triggered when a branch or tag is created. + Triggered when a repository is created, deleted, or updated. - See https://docs.github.com/en/webhooks/webhook-events-and-payloads#create. + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#repository. """ - self._sync_repo_versions() + data = self.request.data + action = data["action"] - def _handle_delete_event(self): - """ - Handle the delete event. + gha_installation, created = self._get_or_create_installation() - Triggered when a branch or tag is deleted. + # If the installation was just created, we already synced the repositories. + if created: + return - See https://docs.github.com/en/webhooks/webhook-events-and-payloads#delete. - """ - self._sync_repo_versions() + if action in ("edited", "privatized", "publicized", "renamed", "trasferred"): + gha_installation.service.add_repositories([data["repository"]["id"]]) + return + + # Ignore other actions: + # - created: If the user granted access to all repositories, + # GH will trigger an installation_repositories event. + # - deleted: If the repository was linked to an installation, + # GH will be trigger an installation_repositories event. + # - archived/unarchived: We don't do anything with archived repositories. def _sync_repo_versions(self): for project in self._get_projects(): @@ -281,6 +291,8 @@ def _handle_push_event(self): self._sync_repo_versions() return + # If this is a push to an existing branch or tag, + # we need to build the version if active. version_name, version_type = self._parse_version_from_ref(data["ref"]) for project in self._get_projects(): build_versions_from_names(project, [(version_name, version_type)]) @@ -341,9 +353,60 @@ def _handle_pull_request_event(self): ) return - # We don't need to handle the other actions. + # We don't care about the other actions. return + def _handle_organization_event(self): + """ + Handle the organization event. + + Triggered when an organization is added or removed from a repository. + + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#organization + """ + data = self.request.data + action = data["action"] + + if action == "member_added": + return + + if action == "member_removed": + # Don't actually remove the user from the organization, + # it could also have been deranked to outside collaborator. + # See if GH sends a different event for that. + return + + if action == "renamed": + # Update organization. + return + + if action == "deleted": + # Delete the organization only if it's not linked to any project. + return + + # Ignore other actions: + # - member_invited: We don't do anything with invited members. + + def _handle_member_event(self): + """ + Handle the member event. + + Triggered when a user is added or removed from a repository. + + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#member + """ + data = self.request.data + action = data["action"] + + if action in ("added", "edited"): + # Sync collaborators + return + if action == "removed": + return + + # NOTE: this should never happen. + raise ValidationError(f"Unsupported action: {action}") + def _get_projects(self): remote_repository = self._get_remote_repository() if not remote_repository: From 91d23aa092e2779f24150efe376fd1cad946b5fd Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 5 Feb 2025 17:01:56 -0500 Subject: [PATCH 09/92] More hooks and fixes --- readthedocs/oauth/services/githubapp.py | 149 ++++++++++++++++++------ readthedocs/oauth/views.py | 95 ++++++++------- 2 files changed, 167 insertions(+), 77 deletions(-) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 45784aa5516..b10bc349864 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -1,11 +1,11 @@ -from functools import cached_property +from functools import cached_property, lru_cache import structlog from allauth.socialaccount.models import SocialAccount from django.conf import settings from github import Auth, GithubIntegration from github.Installation import Installation as GHInstallation -from github.NamedUser import NamedUser as GHNamedUser +from github.Organization import Organization as GHOrganization from github.Repository import Repository as GHRepository from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider @@ -14,6 +14,7 @@ GitHubAccountType, GitHubAppInstallation, RemoteOrganization, + RemoteOrganizationRelation, RemoteRepository, RemoteRepositoryRelation, ) @@ -65,7 +66,7 @@ def sync_repositories(self): def _sync_installation_repositories(self): remote_repositories = [] for repo in self.gha_client.app_installation.get_repos(): - remote_repo = self.create_or_update_repository(repo) + remote_repo = self._create_or_update_repository_from_gh(repo) if remote_repo: remote_repositories.append(remote_repo) @@ -82,16 +83,24 @@ def _sync_installation_repositories(self): def add_repositories(self, repository_ids: list[int]): for repository_id in repository_ids: repo = self.gha_client.client.get_repo(repository_id) - self.create_or_update_repository(repo) + self._create_or_update_repository_from_gh(repo) - def remove_repositories(self, repository_ids: list[int]): + def delete_repositories(self, repository_ids: list[int]): """ - Remove repositories from the given list that are not linked to a project. + Delete repositories from the given list that are not linked to a project. We don't remove repositories that are linked to a project, since a user could grant access to the repository again, and we don't want users having to manually link the project to the repository again. """ + # Extract all the organizations linked to these repositories, + # so we can remove organizations that don't have any repositories + # after removing the repositories. + remote_organizations = RemoteOrganization.objects.filter( + repositories__remote_id__in=repository_ids, + vcs_provider=self.vcs_provider_slug, + ) + RemoteRepository.objects.filter( github_app_installation=self.installation, vcs_provider=self.vcs_provider_slug, @@ -99,45 +108,60 @@ def remove_repositories(self, repository_ids: list[int]): projects=None, ).delete() - def create_or_update_repository( - self, repo: GHRepository + # Delete organizations that don't have any repositories. + remote_organizations.filter(repositories=None).delete() + + def delete_organization(self, organization_id: int): + """ + Delete an organization and all its repositories from the database only if they are not linked to a project. + """ + RemoteOrganization.objects.filter( + remote_id=str(organization_id), + vcs_provider=self.vcs_provider_slug, + repositories__projects=None, + ).delete() + + def _create_or_update_repository_from_gh( + self, gh_repo: GHRepository ) -> RemoteRepository | None: # What about a project that is public, and then becomes private? # I think we should allow creating remote repositories for these, # but block import/clone and other operations. - if not settings.ALLOW_PRIVATE_REPOS and repo.private: + if not settings.ALLOW_PRIVATE_REPOS and gh_repo.private: return target_id = self.installation.target_id target_type = self.installation.target_type # NOTE: All the repositories should be owned by the installation account. # This should never happen, unless this assumption is wrong. - if repo.owner.id != target_id or repo.owner.type != target_type: + if gh_repo.owner.id != target_id or gh_repo.owner.type != target_type: log.exception( "Repository owner does not match the installation account", - repository_id=repo.id, - repository_owner_id=repo.owner.id, + repository_id=gh_repo.id, + repository_owner_id=gh_repo.owner.id, installation_target_id=target_id, installation_target_type=target_type, ) return remote_repo, _ = RemoteRepository.objects.get_or_create( - remote_id=str(repo.id), + remote_id=str(gh_repo.id), vcs_provider=self.vcs_provider_slug, ) - remote_repo.name = repo.name - remote_repo.full_name = repo.full_name - remote_repo.description = repo.description - remote_repo.avatar_url = repo.owner.avatar_url - remote_repo.ssh_url = repo.ssh_url - remote_repo.html_url = repo.html_url - remote_repo.private = repo.private - remote_repo.default_branch = repo.default_branch + remote_repo.name = gh_repo.name + remote_repo.full_name = gh_repo.full_name + remote_repo.description = gh_repo.description + remote_repo.avatar_url = gh_repo.owner.avatar_url + remote_repo.ssh_url = gh_repo.ssh_url + remote_repo.html_url = gh_repo.html_url + remote_repo.private = gh_repo.private + remote_repo.default_branch = gh_repo.default_branch # TODO: Do we need the SSH URL for private repositories now that we can clone using a token? - remote_repo.clone_url = repo.ssh_url if repo.private else repo.clone_url + remote_repo.clone_url = ( + gh_repo.ssh_url if gh_repo.private else gh_repo.clone_url + ) # NOTE: Only one installation of our APP should give access to a repository. # This should only happen if our data is out of sync. @@ -154,31 +178,57 @@ def create_or_update_repository( remote_repo.github_app_installation = self.installation remote_repo.organization = None - if repo.owner.type == GitHubAccountType.ORGANIZATION: - remote_repo.organization = self.create_or_update_organization(repo.owner) + if gh_repo.owner.type == GitHubAccountType.ORGANIZATION: + # NOTE: The owner object doesn't have all attributes of an organization, + # so we need to fetch the organization object. + gh_organization = self._get_gh_organization(gh_repo.owner.id) + remote_repo.organization = self._create_or_update_organization_from_gh( + gh_organization + ) - self._resync_collaborators(repo, remote_repo) + self._resync_collaborators(gh_repo, remote_repo) # What about members of the organization? Do we care? # I think all of our permissions are based on the collaborators of the repository, # not the members of the organization. remote_repo.save() return remote_repo - def create_or_update_organization(self, org: GHNamedUser) -> RemoteOrganization: + # NOTE: normally, this should cache only one organization at a time, but just in case... + @lru_cache(maxsize=50) + def _get_gh_organization(self, org_id: int) -> GHOrganization: + # NOTE: cast to str, since the GitHub API expects a string, + # even if the API accepts a string or an int. + return self.gha_client.client.get_organization(str(org_id)) + + # NOTE: normally, this should cache only one organization at a time, but just in case... + @lru_cache(maxsize=50) + def _create_or_update_organization_from_gh( + self, gh_org: GHOrganization + ) -> RemoteOrganization: + """ + Create or update a remote organization from a GitHub organization object. + + We also sync the members of the organization with the database. + + This method is cached, since we want to update the organization only once per sync of an installation. + """ remote_org, _ = RemoteOrganization.objects.get_or_create( - remote_id=str(org.id), + remote_id=str(gh_org.id), vcs_provider=self.vcs_provider_slug, ) - remote_org.slug = org.login - remote_org.name = org.name + remote_org.slug = gh_org.login + remote_org.name = gh_org.name # NOTE: do we need the email of the organization? - remote_org.email = org.email - remote_org.avatar_url = org.avatar_url - remote_org.url = org.html_url + remote_org.email = gh_org.email + remote_org.avatar_url = gh_org.avatar_url + remote_org.url = gh_org.html_url remote_org.save() + self._resync_organization_members(gh_org, remote_org) return remote_org - def _resync_collaborators(self, repo: GHRepository, remote_repo: RemoteRepository): + def _resync_collaborators( + self, gh_repo: GHRepository, remote_repo: RemoteRepository + ): """ Sync collaborators of a repository with the database. @@ -186,8 +236,7 @@ def _resync_collaborators(self, repo: GHRepository, remote_repo: RemoteRepositor """ collaborators = { collaborator.id: collaborator - # Return all collaborators or just the ones with admin permission? - for collaborator in repo.get_collaborators() + for collaborator in gh_repo.get_collaborators() } remote_repo_relations_ids = [] for account in self._get_social_accounts(collaborators.keys()): @@ -196,7 +245,6 @@ def _resync_collaborators(self, repo: GHRepository, remote_repo: RemoteRepositor user=account.user, account=account, ) - remote_repo_relation.user = account.user remote_repo_relation.admin = collaborators[ int(account.uid) ].permissions.admin @@ -216,6 +264,35 @@ def _get_social_accounts(self, ids): provider=GitHubAppProvider.id, ).select_related("user") + def update_or_create_organization(self, org_id: int) -> RemoteOrganization: + gh_org = self._get_gh_organization(org_id) + return self._create_or_update_organization_from_gh(gh_org) + + def _resync_organization_members( + self, gh_org: GHOrganization, remote_org: RemoteOrganization + ): + """ + Sync members of an organization with the database. + + This method will remove members that are no longer in the list. + """ + members = {member.id: member for member in gh_org.get_members()} + remote_org_relations_ids = [] + for account in self._get_social_accounts(members.keys()): + remote_org_relation, _ = RemoteOrganizationRelation.objects.get_or_create( + remote_organization=remote_org, + user=account.user, + account=account, + ) + remote_org_relations_ids.append(remote_org_relation.pk) + + # Remove members that are no longer in the list. + RemoteOrganizationRelation.objects.filter( + remote_organization=remote_org, + ).exclude( + pk__in=remote_org_relations_ids, + ).delete() + def sync_organizations(self): if self.installation.target_type != GitHubAccountType.ORGANIZATION: return diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index b90b6317988..958bbb98462 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -123,26 +123,26 @@ def _handle_installation_event(self): """ data = self.request.data action = data["action"] - installation = data["installation"] + gh_installation = data["installation"] if action == "created": - gha_installation, created = self._get_or_create_installation() + installation, created = self._get_or_create_installation() if not created: log.info( - "Installation already exists", installation_id=installation["id"] + "Installation already exists", installation_id=gh_installation["id"] ) return if action == "deleted": - gha_installation = GitHubAppInstallation.objects.filter( - installation_id=installation["id"] + installation = GitHubAppInstallation.objects.filter( + installation_id=gh_installation["id"] ).first() - if not gha_installation: + if not installation: # If we never created the installation, we can ignore the event. # Maybe don't raise an error? - raise ValidationError(f"Installation {installation['id']} not found") - gha_installation.delete() + raise ValidationError(f"Installation {gh_installation['id']} not found") + installation.delete() return # Ignore other actions: @@ -210,7 +210,7 @@ def _handle_installation_repositories_event(self): """ data = self.request.data action = data["action"] - gha_installation, created = self._get_or_create_installation() + installation, created = self._get_or_create_installation() # If we didn't have the installation, all repositories were synced on creation. if created: @@ -218,15 +218,15 @@ def _handle_installation_repositories_event(self): if action == "added": if data["repository_selection"] == "all": - gha_installation.service.sync_repositories() + installation.service.sync_repositories() else: - gha_installation.service.add_repositories( + installation.service.add_repositories( [repo["id"] for repo in data["repositories_added"]] ) return if action == "removed": - gha_installation.service.remove_repositories( + installation.service.delete_repositories( [repo["id"] for repo in data["repositories_removed"]] ) return @@ -253,14 +253,14 @@ def _handle_repository_event(self): data = self.request.data action = data["action"] - gha_installation, created = self._get_or_create_installation() + installation, created = self._get_or_create_installation() # If the installation was just created, we already synced the repositories. if created: return if action in ("edited", "privatized", "publicized", "renamed", "trasferred"): - gha_installation.service.add_repositories([data["repository"]["id"]]) + installation.service.add_repositories([data["repository"]["id"]]) return # Ignore other actions: @@ -367,21 +367,34 @@ def _handle_organization_event(self): data = self.request.data action = data["action"] - if action == "member_added": + installation, created = self._get_or_create_installation() + + # If the installation was just created, we already synced the repositories and organization. + if created: return - if action == "member_removed": - # Don't actually remove the user from the organization, - # it could also have been deranked to outside collaborator. - # See if GH sends a different event for that. + # We need to do a full sync of the repositories if members were added or removed, + # this is since we don't know to which repositories the members have access. + # GH doesn't send a member event for this. + if action in ("member_added", "member_removed"): + installation.service.sync_repositories() return if action == "renamed": - # Update organization. + # Update organization and its members only. + # We don't need to sync the repositories. + installation.service.update_or_create_organization( + data["organization"]["id"] + ) return if action == "deleted": # Delete the organization only if it's not linked to any project. + # GH sends a repository and installation_repositories events for each repository + # when the organization is deleted. + # I didn't see that GH send the deleted action for the organization event... + # But handle it just in case. + installation.service.delete_organization(data["organization"]["id"]) return # Ignore other actions: @@ -421,8 +434,8 @@ def _get_remote_repository(self): """ data = self.request.data remote_id = data["repository"]["id"] - gha_installation, _ = self._get_or_create_installation() - return gha_installation.repositories.filter(remote_id=remote_id).first() + installation, _ = self._get_or_create_installation() + return installation.repositories.filter(remote_id=remote_id).first() def _get_or_create_installation(self, sync_repositories_on_create: bool = True): """ @@ -433,22 +446,22 @@ def _get_or_create_installation(self, sync_repositories_on_create: bool = True): """ data = self.request.data # All webhook payloads should have an installation object. - installation = data["installation"] - installation_id = installation["id"] + gh_installation = data["installation"] + installation_id = gh_installation["id"] # These fields are not always present in all payloads. - target_id = installation.get("target_id") - target_type = installation.get("target_type") + target_id = gh_installation.get("target_id") + target_type = gh_installation.get("target_type") # If they aren't present, fetch them from the API, # so we can create the installation object if needed. if not target_id or not target_type: - installation = GitHubAppClient(installation_id).app_installation - target_id = installation.target_id - target_type = installation.target_type + gh_installation = GitHubAppClient(installation_id).app_installation + target_id = gh_installation.target_id + target_type = gh_installation.target_type data = data.copy() - data["installation"] = installation.raw_data + data["installation"] = gh_installation.raw_data - gha_installation, created = GitHubAppInstallation.objects.get_or_create( + installation, created = GitHubAppInstallation.objects.get_or_create( installation_id=installation_id, defaults={ "extra_data": data, @@ -459,23 +472,23 @@ def _get_or_create_installation(self, sync_repositories_on_create: bool = True): # NOTE: An installation can't change its target_id or target_type. # This should never happen, unless this assumption is wrong. if ( - gha_installation.target_id != target_id - or gha_installation.target_type != target_type + installation.target_id != target_id + or installation.target_type != target_type ): log.exception( "Installation target_id or target_type changed", - installation_id=gha_installation.installation_id, - target_id=gha_installation.target_id, - target_type=gha_installation.target_type, + installation_id=installation.installation_id, + target_id=installation.target_id, + target_type=installation.target_type, new_target_id=target_id, new_target_type=target_type, ) - gha_installation.target_id = target_id - gha_installation.target_type = target_type - gha_installation.save() + installation.target_id = target_id + installation.target_type = target_type + installation.save() if created and sync_repositories_on_create: - gha_installation.service.sync_repositories() - return gha_installation, created + installation.service.sync_repositories() + return installation, created def _is_payload_signature_valid(self): """ From ddadee1e60d5430325e99c329cc473f0df49074b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 5 Feb 2025 17:13:42 -0500 Subject: [PATCH 10/92] Implement members hook --- readthedocs/oauth/services/githubapp.py | 2 +- readthedocs/oauth/views.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index b10bc349864..249d16f6c61 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -80,7 +80,7 @@ def _sync_installation_repositories(self): pk__in=[repo.pk for repo in remote_repositories], ).delete() - def add_repositories(self, repository_ids: list[int]): + def update_or_create_repositories(self, repository_ids: list[int]): for repository_id in repository_ids: repo = self.gha_client.client.get_repo(repository_id) self._create_or_update_repository_from_gh(repo) diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 958bbb98462..02674ce653b 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -220,7 +220,7 @@ def _handle_installation_repositories_event(self): if data["repository_selection"] == "all": installation.service.sync_repositories() else: - installation.service.add_repositories( + installation.service.update_or_create_repositories( [repo["id"] for repo in data["repositories_added"]] ) return @@ -260,7 +260,9 @@ def _handle_repository_event(self): return if action in ("edited", "privatized", "publicized", "renamed", "trasferred"): - installation.service.add_repositories([data["repository"]["id"]]) + installation.service.update_or_create_repositories( + [data["repository"]["id"]] + ) return # Ignore other actions: @@ -411,12 +413,18 @@ def _handle_member_event(self): data = self.request.data action = data["action"] - if action in ("added", "edited"): - # Sync collaborators - return - if action == "removed": + installation, created = self._get_or_create_installation() + + # If we didn't have the installation, all repositories were synced on creation. + if created: return + if action in ("added", "edited", "removed"): + # Sync collaborators + installation.service.update_or_create_repositories( + [data["repository"]["id"]] + ) + # NOTE: this should never happen. raise ValidationError(f"Unsupported action: {action}") From 2c4c0cc706f5ea359fe5ef64064f0531d9487d3e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 5 Feb 2025 17:25:07 -0500 Subject: [PATCH 11/92] Dead code --- readthedocs/oauth/services/githubapp.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 249d16f6c61..9b15229e2ba 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -59,8 +59,6 @@ def __init__(self, installation: GitHubAppInstallation): self.gha_client = GitHubAppClient(self.installation.installation_id) def sync_repositories(self): - # if self.installation.target_type != GitHubAccountType.USER: - # return return self._sync_installation_repositories() def _sync_installation_repositories(self): @@ -186,18 +184,16 @@ def _create_or_update_repository_from_gh( gh_organization ) - self._resync_collaborators(gh_repo, remote_repo) - # What about members of the organization? Do we care? - # I think all of our permissions are based on the collaborators of the repository, - # not the members of the organization. remote_repo.save() + self._resync_collaborators(gh_repo, remote_repo) return remote_repo # NOTE: normally, this should cache only one organization at a time, but just in case... @lru_cache(maxsize=50) def _get_gh_organization(self, org_id: int) -> GHOrganization: - # NOTE: cast to str, since the GitHub API expects a string, + # NOTE: cast to str, since PyGithub expects a string, # even if the API accepts a string or an int. + # TODO: send a PR upstream to fix this. return self.gha_client.client.get_organization(str(org_id)) # NOTE: normally, this should cache only one organization at a time, but just in case... @@ -292,8 +288,3 @@ def _resync_organization_members( ).exclude( pk__in=remote_org_relations_ids, ).delete() - - def sync_organizations(self): - if self.installation.target_type != GitHubAccountType.ORGANIZATION: - return - return self._sync_installation_repositories() From 3c61db63ccf5f2ceb4e19310f263f0ca47627e69 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 5 Feb 2025 17:30:09 -0500 Subject: [PATCH 12/92] Delete code --- readthedocs/oauth/views.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 02674ce653b..f406578cdaa 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -272,10 +272,6 @@ def _handle_repository_event(self): # GH will be trigger an installation_repositories event. # - archived/unarchived: We don't do anything with archived repositories. - def _sync_repo_versions(self): - for project in self._get_projects(): - trigger_sync_versions(project) - def _handle_push_event(self): """ Handle the push event. @@ -290,7 +286,8 @@ def _handle_push_event(self): created = data.get("created", False) deleted = data.get("deleted", False) if created or deleted: - self._sync_repo_versions() + for project in self._get_projects(): + trigger_sync_versions(project) return # If this is a push to an existing branch or tag, From 88b002183c252e09d54df87e1fbea65d0ff51eaa Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 6 Feb 2025 12:08:24 -0500 Subject: [PATCH 13/92] Poor man implementation --- readthedocs/api/v2/serializers.py | 1 + readthedocs/builds/tasks.py | 15 +++++++++++ readthedocs/oauth/services/githubapp.py | 36 +++++++++++++++++++++++++ readthedocs/oauth/views.py | 7 ++--- readthedocs/projects/models.py | 14 ++++++++++ 5 files changed, 70 insertions(+), 3 deletions(-) diff --git a/readthedocs/api/v2/serializers.py b/readthedocs/api/v2/serializers.py index 45e6a03c67e..cc669e55741 100644 --- a/readthedocs/api/v2/serializers.py +++ b/readthedocs/api/v2/serializers.py @@ -91,6 +91,7 @@ class Meta(ProjectSerializer.Meta): "environment_variables", "max_concurrent_builds", "readthedocs_yaml_path", + "clone_token", ) diff --git a/readthedocs/builds/tasks.py b/readthedocs/builds/tasks.py index fc076af9118..115d5a879ea 100644 --- a/readthedocs/builds/tasks.py +++ b/readthedocs/builds/tasks.py @@ -412,6 +412,21 @@ def send_build_status(build_pk, commit, status): log.debug("Sending build status.") + remote_repository = build.project.remote_repository + if remote_repository and remote_repository.github_app_installation: + # Send status report using the GitHub App API. + service = remote_repository.github_app_installation.service + try: + service.send_build_status( + build=build, + commit=commit, + status=status, + ) + return True + except Exception: + log.exception("Failed to send build status.") + return False + if provider_name in [GITHUB_BRAND, GITLAB_BRAND]: # get the service class for the project e.g: GitHubService. service_class = build.project.git_service_class() diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 9b15229e2ba..71896942c1c 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -9,6 +9,7 @@ from github.Repository import Repository as GHRepository from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider +from readthedocs.builds.constants import BUILD_STATUS_SUCCESS, SELECT_BUILD_STATUS from readthedocs.oauth.constants import GITHUB from readthedocs.oauth.models import ( GitHubAccountType, @@ -50,6 +51,17 @@ def client(self): def app_installation(self) -> GHInstallation: return self.integration_client.get_app_installation(self.installation_id) + def get_installation_token(self, permissions: dict | None = None): + """ + + https://docs.github.com/en/rest/apps/apps?apiVersion=2022-11-28#create-an-installation-access-token-for-an-app + """ + # TODO: we can pass the repository_ids to get a token with access to specific repositories. + # We should upstream this feature to PyGithub. + return self.integration_client.get_access_token( + self.installation_id, permissions=permissions + ).token + class GitHubAppService: vcs_provider_slug = GITHUB @@ -288,3 +300,27 @@ def _resync_organization_members( ).exclude( pk__in=remote_org_relations_ids, ).delete() + + def send_build_status(self, *, build, commit, status): + project = build.project + remote_repo = project.remote_repository + + if status == BUILD_STATUS_SUCCESS: + target_url = build.version.get_absolute_url() + else: + target_url = build.get_full_url() + + state = SELECT_BUILD_STATUS[status]["github"] + description = SELECT_BUILD_STATUS[status]["description"] + context = f"{settings.RTD_BUILD_STATUS_API_NAME}:{project.slug}" + + gh_repo = self.gha_client.client.get_repo(int(remote_repo.remote_id)) + gh_repo.get_commit(commit).create_status( + state=state, + target_url=target_url, + description=description, + context=context, + ) + + def get_installation_token(self, permissions: dict | None = None): + return self.gha_client.get_installation_token(permissions=permissions) diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index f406578cdaa..4081a8a9e8b 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -135,6 +135,7 @@ def _handle_installation_event(self): return if action == "deleted": + # NOTE: does the app trigger a installation_repositories event? installation = GitHubAppInstallation.objects.filter( installation_id=gh_installation["id"] ).first() @@ -327,7 +328,7 @@ def _handle_pull_request_event(self): action = data["action"] pr = data["pull_request"] - ExternalVersionData( + external_version_data = ExternalVersionData( id=str(pr["number"]), commit=pr["head"]["sha"], source_branch=pr["head"]["ref"], @@ -338,7 +339,7 @@ def _handle_pull_request_event(self): for project in self._get_projects(): external_version = get_or_create_external_version( project=project, - version_data=ExternalVersionData, + version_data=external_version_data, ) build_external_version(project, external_version) return @@ -348,7 +349,7 @@ def _handle_pull_request_event(self): for project in self._get_projects(): close_external_version( project=project, - version_data=ExternalVersionData, + version_data=external_version_data, ) return diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 0c495ea59f8..5038e1c6d68 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1015,6 +1015,7 @@ def vcs_repo( version_type=version_type, version_identifier=version_identifier, version_machine=version_machine, + token=self.clone_token, ) return repo @@ -1436,6 +1437,14 @@ def get_subproject_candidates(self, user): def organization(self): return self.organizations.first() + @property + def clone_token(self): + remote_repository = self.remote_repository + if remote_repository and remote_repository.github_app_installation: + service = remote_repository.github_app_installation.service + return f"x-access-token:{service.get_installation_token()}" + return None + class APIProject(Project): @@ -1453,12 +1462,17 @@ class APIProject(Project): """ features = [] + # This is a property in the original model, in order to + # be able to assign it a value in the constructor, we need to re-declare it + # as an attribute here. + clone_token = None class Meta: proxy = True def __init__(self, *args, **kwargs): self.features = kwargs.pop("features", []) + self.clone_token = kwargs.pop("clone_token", None) environment_variables = kwargs.pop("environment_variables", {}) ad_free = not kwargs.pop("show_advertising", True) # These fields only exist on the API return, not on the model, so we'll From ad5655e9684a9b62b7501a374928b945d205db69 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 6 Feb 2025 16:46:34 -0500 Subject: [PATCH 14/92] Stash --- readthedocs/oauth/views.py | 13 ++++++++++++- readthedocs/projects/models.py | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 4081a8a9e8b..b438565c3af 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -46,6 +46,7 @@ def post(self, request): "repository": self._handle_repository_event, "organization": self._handle_organization_event, "member": self._handle_member_event, + "github_app_authorization": self._handle_github_app_authorization_event, } if event in event_handlers: event_handlers[event]() @@ -131,7 +132,6 @@ def _handle_installation_event(self): log.info( "Installation already exists", installation_id=gh_installation["id"] ) - return if action == "deleted": @@ -380,6 +380,7 @@ def _handle_organization_event(self): installation.service.sync_repositories() return + # Hmm, installation_target should handle this instead? if action == "renamed": # Update organization and its members only. # We don't need to sync the repositories. @@ -426,6 +427,16 @@ def _handle_member_event(self): # NOTE: this should never happen. raise ValidationError(f"Unsupported action: {action}") + def _handle_github_app_authorization_event(self): + """ + Revoking the authorization of a GitHub App does not uninstall the GitHub App. + You should program your GitHub App so that when it receives this webhook, + it stops calling the API on behalf of the person who revoked the token. + + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#github_app_authorization. + """ + pass + def _get_projects(self): remote_repository = self._get_remote_repository() if not remote_repository: diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 570bce34531..a29da2a5eb6 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1439,6 +1439,9 @@ def organization(self): @property def clone_token(self): + """ + See https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation. + """ remote_repository = self.remote_repository if remote_repository and remote_repository.github_app_installation: service = remote_repository.github_app_installation.service From fa53c1c2ec904396c549dfc74b935ba766f063c1 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 6 Feb 2025 19:00:44 -0500 Subject: [PATCH 15/92] Git service: depend on the project instead of users With https://github.com/readthedocs/readthedocs.org/pull/11942, a GH app is used to interact with the GH API, a GH app is not directly tied to a user, so this assumption is not valid anymore. For most of the operations (except for syncing repos), we don't need to know who is the user that is performing the action, we just need to know the project that is being affected. --- readthedocs/builds/tasks.py | 106 ++++---------- readthedocs/oauth/models.py | 17 ++- readthedocs/oauth/services/base.py | 181 ++++++++++++++---------- readthedocs/oauth/services/bitbucket.py | 9 +- readthedocs/oauth/services/github.py | 5 +- readthedocs/oauth/services/gitlab.py | 5 +- readthedocs/oauth/tasks.py | 101 ++++++------- readthedocs/oauth/utils.py | 45 ++---- readthedocs/projects/models.py | 35 +++-- readthedocs/projects/views/mixins.py | 2 + readthedocs/projects/views/private.py | 8 +- 11 files changed, 247 insertions(+), 267 deletions(-) diff --git a/readthedocs/builds/tasks.py b/readthedocs/builds/tasks.py index fc076af9118..7e13ca62d19 100644 --- a/readthedocs/builds/tasks.py +++ b/readthedocs/builds/tasks.py @@ -30,12 +30,10 @@ ) from readthedocs.builds.models import Build, Version from readthedocs.builds.utils import memcache_lock -from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils import send_email, trigger_build from readthedocs.integrations.models import HttpExchange from readthedocs.notifications.models import Notification from readthedocs.oauth.notifications import MESSAGE_OAUTH_BUILD_STATUS_FAILURE -from readthedocs.projects.constants import GITHUB_BRAND, GITLAB_BRAND from readthedocs.projects.models import Project, WebHookEvent from readthedocs.storage import build_commands_storage from readthedocs.worker import app @@ -385,24 +383,16 @@ def sync_versions_task(project_pk, tags_data, branches_data, **kwargs): @app.task(max_retries=3, default_retry_delay=60, queue="web") def send_build_status(build_pk, commit, status): """ - Send Build Status to Git Status API for project external versions. - - It tries using these services' account in order: - - 1. user's account that imported the project - 2. each user's account from the project's maintainers + Send build status to GitHub/GitLab for a given build/commit. :param build_pk: Build primary key :param commit: commit sha of the pull/merge request :param status: build status failed, pending, or success to be sent. """ - # TODO: Send build status for Bitbucket. build = Build.objects.filter(pk=build_pk).first() if not build: return - provider_name = build.project.git_provider_name - log.bind( build_id=build.pk, project_slug=build.project.slug, @@ -412,76 +402,36 @@ def send_build_status(build_pk, commit, status): log.debug("Sending build status.") - if provider_name in [GITHUB_BRAND, GITLAB_BRAND]: - # get the service class for the project e.g: GitHubService. - service_class = build.project.git_service_class() - users = AdminPermission.admins(build.project) - - if build.project.remote_repository: - remote_repository = build.project.remote_repository - remote_repository_relations = ( - remote_repository.remote_repository_relations.filter( - account__isnull=False, - # Use ``user_in=`` instead of ``user__projects=`` here - # because User's are not related to Project's directly in - # Read the Docs for Business - user__in=AdminPermission.members(build.project), - ) - .select_related("account", "user") - .only("user", "account") - ) - - # Try using any of the users' maintainer accounts - # Try to loop through all remote repository relations for the projects users - for relation in remote_repository_relations: - service = service_class(relation.user, relation.account) - # Send status report using the API. - success = service.send_build_status( - build, - commit, - status, - ) + # Get the service class for the project e.g: GitHubService. + # We fallback to guess the service from the repo, + # in the future we should only consider projects that have a remote repository. + service_class = build.project.get_git_service_class(fallback_to_clone_url=True) + if not service_class: + log.info("Project isn't connected to a Git service.") + return False - if success: - log.debug( - "Build status report sent correctly.", - user_username=relation.user.username, - ) - return True - else: - log.warning("Project does not have a RemoteRepository.") - # Try to send build status for projects with no RemoteRepository - for user in users: - services = service_class.for_user(user) - # Try to loop through services for users all social accounts - # to send successful build status - for service in services: - success = service.send_build_status( - build, - commit, - status, - ) - if success: - log.debug( - "Build status report sent correctly using an user account.", - user_username=user.username, - ) - return True - - # NOTE: this notifications was attached to every user. - # Now, I'm attaching it to the project itself since it's a problem at project level. - Notification.objects.add( - message_id=MESSAGE_OAUTH_BUILD_STATUS_FAILURE, - attached_to=build.project, - format_values={ - "provider_name": provider_name, - "url_connect_account": reverse("socialaccount_connections"), - }, - dismissable=True, + for service in service_class.for_project(build.project): + success = service.send_build_status( + build, + commit, + status, ) + if success: + log.debug("Build status report sent correctly.") + return True + + Notification.objects.add( + message_id=MESSAGE_OAUTH_BUILD_STATUS_FAILURE, + attached_to=build.project, + format_values={ + "provider_name": service_class.provider_name, + "url_connect_account": reverse("socialaccount_connections"), + }, + dismissable=True, + ) - log.info("No social account or repository permission available.") - return False + log.info("No social account or repository permission available.") + return False @app.task(queue="web") diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 4810943d046..b236fb5737f 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -1,5 +1,5 @@ """OAuth service models.""" - +import structlog from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import User from django.core.validators import URLValidator @@ -14,6 +14,8 @@ from .constants import VCS_PROVIDER_CHOICES from .querysets import RemoteOrganizationQuerySet, RemoteRepositoryQuerySet +log = structlog.get_logger(__name__) + class RemoteOrganization(TimeStampedModel): @@ -224,6 +226,19 @@ def get_remote_repository_relation(self, user, social_account): ) return remote_repository_relation + def get_service_class(self): + from readthedocs.oauth.services import registry + + for service_cls in registry: + if service_cls.vcs_provider_slug == self.vcs_provider: + return service_cls + + # NOTE: this should never happen, but we log it just in case + log.exception( + "Service not found for the VCS provider", vcs_provider=self.vcs_provider + ) + return None + class RemoteRepositoryRelation(TimeStampedModel): remote_repository = models.ForeignKey( diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 63c5007859b..7f570acd3ab 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -1,10 +1,8 @@ """OAuth utility functions.""" - from datetime import datetime import structlog from allauth.socialaccount.models import SocialAccount -from allauth.socialaccount.providers import registry from allauth.socialaccount.providers.oauth2.views import OAuth2Adapter from django.conf import settings from django.urls import reverse @@ -14,6 +12,8 @@ from requests.exceptions import RequestException from requests_oauthlib import OAuth2Session +from readthedocs.core.permissions import AdminPermission + log = structlog.get_logger(__name__) @@ -29,19 +29,103 @@ class SyncServiceError(Exception): class Service: + """Base class for service that interacts with a VCS provider and a project.""" + + vcs_provider_slug: str + url_pattern: str + provider_name: str + default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL + default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL + + @classmethod + def for_project(self, project): + """Return an iterator of services that can be used for the project.""" + raise NotImplementedError + + @classmethod + def for_user(self, user): + """Return an iterator of services that belong to the user.""" + raise NotImplementedError + + def sync(self): + """ + Sync remote repositories and organizations. + + - Creates a new RemoteRepository/Organization per new repository + - Updates fields for existing RemoteRepository/Organization + - Deletes old RemoteRepository/Organization that are not present + for this user in the current provider + """ + raise NotImplementedError + + def setup_webhook(self, project, integration=None): + """ + Setup webhook for project. + + :param project: project to set up webhook for + :type project: Project + :param integration: Integration for the project + :type integration: Integration + :returns: boolean based on webhook set up success, and requests Response object + :rtype: (Bool, Response) + """ + raise NotImplementedError + + def update_webhook(self, project, integration): + """ + Update webhook integration. + + :param project: project to set up webhook for + :type project: Project + :param integration: Webhook integration to update + :type integration: Integration + :returns: boolean based on webhook update success, and requests Response object + :rtype: (Bool, Response) + """ + raise NotImplementedError + + def send_build_status(self, build, commit, status): + """ + Create commit status for project. + + :param build: Build to set up commit status for + :type build: Build + :param commit: commit sha of the pull/merge request + :type commit: str + :param status: build state failure, pending, or success. + :type status: str + :returns: boolean based on commit status creation was successful or not. + :rtype: Bool + """ + raise NotImplementedError + + @classmethod + def is_project_service(cls, project): + """ + Determine if this is the service the project is using. + + .. note:: + + This should be deprecated in favor of attaching the + :py:class:`RemoteRepository` to the project instance. This is a + slight improvement on the legacy check for webhooks + """ + return ( + cls.url_pattern is not None + and cls.url_pattern.search(project.repo) is not None + ) + + +class UserService(Service): + """ - Service mapping for local accounts. + Subclass of Service that interacts with a VCS provider using the user's OAuth token. :param user: User to use in token lookup and session creation :param account: :py:class:`SocialAccount` instance for user """ adapter = None - url_pattern = None - vcs_provider_slug = None - - default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL - default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL def __init__(self, user, account): self.session = None @@ -53,17 +137,20 @@ def __init__(self, user, account): social_account_id=self.account.pk, ) + @classmethod + def for_project(cls, project): + users = AdminPermission.admins(project) + for user in users: + yield from cls.for_user(user) + @classmethod def for_user(cls, user): - """Return list of instances if user has an account for the provider.""" - try: - accounts = SocialAccount.objects.filter( - user=user, - provider=cls.adapter.provider_id, - ) - return [cls(user=user, account=account) for account in accounts] - except SocialAccount.DoesNotExist: - return [] + accounts = SocialAccount.objects.filter( + user=user, + provider=cls.adapter.provider_id, + ) + for account in accounts: + yield cls(user=user, account=account) def get_adapter(self) -> type[OAuth2Adapter]: return self.adapter @@ -72,10 +159,6 @@ def get_adapter(self) -> type[OAuth2Adapter]: def provider_id(self): return self.get_adapter().provider_id - @property - def provider_name(self): - return registry.get_class(self.provider_id).name - def get_session(self): if self.session is None: self.create_session() @@ -300,60 +383,8 @@ def get_provider_data(self, project, integration): """ raise NotImplementedError - def setup_webhook(self, project, integration=None): - """ - Setup webhook for project. - - :param project: project to set up webhook for - :type project: Project - :param integration: Integration for the project - :type integration: Integration - :returns: boolean based on webhook set up success, and requests Response object - :rtype: (Bool, Response) - """ - raise NotImplementedError - - def update_webhook(self, project, integration): - """ - Update webhook integration. - - :param project: project to set up webhook for - :type project: Project - :param integration: Webhook integration to update - :type integration: Integration - :returns: boolean based on webhook update success, and requests Response object - :rtype: (Bool, Response) - """ + def sync_repositories(self): raise NotImplementedError - def send_build_status(self, build, commit, status): - """ - Create commit status for project. - - :param build: Build to set up commit status for - :type build: Build - :param commit: commit sha of the pull/merge request - :type commit: str - :param status: build state failure, pending, or success. - :type status: str - :returns: boolean based on commit status creation was successful or not. - :rtype: Bool - """ + def sync_organizations(self): raise NotImplementedError - - @classmethod - def is_project_service(cls, project): - """ - Determine if this is the service the project is using. - - .. note:: - - This should be deprecated in favor of attaching the - :py:class:`RemoteRepository` to the project instance. This is a - slight improvement on the legacy check for webhooks - """ - # TODO Replace this check by keying project to remote repos - return ( - cls.url_pattern is not None - and cls.url_pattern.search(project.repo) is not None - ) diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 7ffab66f3e8..992a7db3996 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -15,12 +15,12 @@ from ..constants import BITBUCKET from ..models import RemoteOrganization, RemoteRepository, RemoteRepositoryRelation -from .base import Service, SyncServiceError +from .base import SyncServiceError, UserService log = structlog.get_logger(__name__) -class BitbucketService(Service): +class BitbucketService(UserService): """Provider service for Bitbucket.""" @@ -29,6 +29,7 @@ class BitbucketService(Service): url_pattern = re.compile(r"bitbucket.org") https_url_pattern = re.compile(r"^https:\/\/[^@]+@bitbucket.org/") vcs_provider_slug = BITBUCKET + provider_name = "Bitbucket" def sync_repositories(self): """Sync repositories from Bitbucket API.""" @@ -393,3 +394,7 @@ def update_webhook(self, project, integration): log.exception("Bitbucket webhook update failed for project.") return (False, resp) + + def send_build_status(self, build, commit, status): + """Send build status is not supported/implemented for Bitbucket.""" + return True diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index fb95492a81d..4dacfb5a2ce 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -15,12 +15,12 @@ from ..constants import GITHUB from ..models import RemoteOrganization, RemoteRepository -from .base import Service, SyncServiceError +from .base import SyncServiceError, UserService log = structlog.get_logger(__name__) -class GitHubService(Service): +class GitHubService(UserService): """Provider service for GitHub.""" @@ -28,6 +28,7 @@ class GitHubService(Service): # TODO replace this with a less naive check url_pattern = re.compile(r"github\.com") vcs_provider_slug = GITHUB + provider_name = "GitHub" def sync_repositories(self): """Sync repositories from GitHub API.""" diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index ee809f92efb..4e65012c566 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -16,12 +16,12 @@ from ..constants import GITLAB from ..models import RemoteOrganization, RemoteRepository -from .base import Service, SyncServiceError +from .base import SyncServiceError, UserService log = structlog.get_logger(__name__) -class GitLabService(Service): +class GitLabService(UserService): """ Provider service for GitLab. @@ -31,6 +31,7 @@ class GitLabService(Service): - https://docs.gitlab.com/ce/api/oauth2.html """ + provider_name = "GitLab" adapter = GitLabOAuth2Adapter # Just use the network location to determine if it's a GitLab project # because private repos have another base url, eg. git@gitlab.example.com diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index c98ba45ce80..59125bbdfa9 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -3,7 +3,6 @@ import datetime import structlog -from allauth.socialaccount.providers import registry as allauth_registry from django.contrib.auth.models import User from django.db.models.functions import ExtractIsoWeekDay from django.urls import reverse @@ -153,8 +152,9 @@ def sync_active_users_remote_repositories(): log.exception("There was a problem re-syncing RemoteRepository.") +# TODO: remove user_pk from the signature on the next release. @app.task(queue="web") -def attach_webhook(project_pk, user_pk, integration=None): +def attach_webhook(project_pk, user_pk=None, integration=None, **kwargs): """ Add post-commit hook on project import. @@ -162,86 +162,67 @@ def attach_webhook(project_pk, user_pk, integration=None): all accounts until we set up a webhook. This should remain around for legacy connections -- that is, projects that do not have a remote repository them and were not set up with a VCS provider. + + :param project_pk: Project primary key + :param integration: Integration instance. If used, this function should + be called directly, not as a task. """ project = Project.objects.filter(pk=project_pk).first() - user = User.objects.filter(pk=user_pk).first() - - if not project or not user: + if not project: return False if integration: - service = SERVICE_MAP.get(integration.integration_type) - - if not service: - log.warning("There are no registered services in the application.") - Notification.objects.add( - message_id=MESSAGE_OAUTH_WEBHOOK_INVALID, - attached_to=project, - dismissable=True, - format_values={ - "url_integrations": reverse( - "projects_integrations", - args=[project.slug], - ), - }, - ) - return None + service_class = SERVICE_MAP.get(integration.integration_type) else: - for service_cls in registry: - if service_cls.is_project_service(project): - service = service_cls - break - else: - log.warning("There are no registered services in the application.") - Notification.objects.add( - message_id=MESSAGE_OAUTH_WEBHOOK_INVALID, - attached_to=project, - dismissable=True, - format_values={ - "url_integrations": reverse( - "projects_integrations", - args=[project.slug], - ), - }, - ) - return None - - provider_class = allauth_registry.get_class(service.adapter.provider_id) - - user_accounts = service.for_user(user) - for account in user_accounts: - success, __ = account.setup_webhook(project, integration=integration) - if success: - # NOTE: do we want to communicate that we connect the webhook here? - # messages.add_message(request, "Webhook successfully added.") + # Get the service class for the project e.g: GitHubService. + # We fallback to guess the service from the repo, + # in the future we should only consider projects that have a remote repository. + service_class = project.get_git_service_class(fallback_to_clone_url=True) - project.has_valid_webhook = True - project.save() - return True - - # No valid account found - if user_accounts: + if not service_class: Notification.objects.add( - message_id=MESSAGE_OAUTH_WEBHOOK_NO_PERMISSIONS, - dismissable=True, + message_id=MESSAGE_OAUTH_WEBHOOK_INVALID, attached_to=project, + dismissable=True, format_values={ - "provider_name": provider_class.name, - "url_docs_webhook": "https://docs.readthedocs.io/page/webhooks.html", + "url_integrations": reverse( + "projects_integrations", + args=[project.slug], + ), }, ) - else: + return False + + services = list(service_class.for_project(project)) + if not services: Notification.objects.add( message_id=MESSAGE_OAUTH_WEBHOOK_NO_ACCOUNT, dismissable=True, attached_to=project, format_values={ - "provider_name": provider_class.name, + "provider_name": service_class.vcs_provider_name, "url_connect_account": reverse( "projects_integrations", args=[project.slug], ), }, ) + return False + for service in service_class.for_project(project): + success, _ = service.setup_webhook(project, integration=integration) + if success: + project.has_valid_webhook = True + project.save() + return True + + Notification.objects.add( + message_id=MESSAGE_OAUTH_WEBHOOK_NO_PERMISSIONS, + dismissable=True, + attached_to=project, + format_values={ + "provider_name": service_class.vcs_provider_name, + "url_docs_webhook": "https://docs.readthedocs.io/page/webhooks.html", + }, + ) return False diff --git a/readthedocs/oauth/utils.py b/readthedocs/oauth/utils.py index 7d527e1a066..e71ca87b056 100644 --- a/readthedocs/oauth/utils.py +++ b/readthedocs/oauth/utils.py @@ -21,43 +21,26 @@ def update_webhook(project, integration, request=None): # FIXME: this method supports ``request=None`` on its definition. # However, it does not work when passing ``request=None`` as # it uses that object without checking if it's ``None`` or not. - service_cls = SERVICE_MAP.get(integration.integration_type) - if service_cls is None: + service_class = SERVICE_MAP.get(integration.integration_type) + if service_class is None: return None # TODO: remove after integrations without a secret are removed. if not integration.secret: integration.save() - updated = False - if project.remote_repository: - remote_repository_relations = ( - project.remote_repository.remote_repository_relations.filter( - account__isnull=False, user=request.user - ).select_related("account") - ) - - for relation in remote_repository_relations: - service = service_cls(request.user, relation.account) - updated, __ = service.update_webhook(project, integration) - - if updated: - break - else: - # The project was imported manually and doesn't have a RemoteRepository - # attached. We do brute force over all the accounts registered for this - # service - service_accounts = service_cls.for_user(request.user) - for service in service_accounts: - updated, __ = service.update_webhook(project, integration) - if updated: - break - - if updated: - messages.success(request, _("Webhook activated")) - project.has_valid_webhook = True - project.save() - return True + # The project was imported manually and doesn't have a RemoteRepository + # attached. We do brute force over all the accounts registered for this + # service + service_class = project.get_git_service_class() or service_class + + for service in service_class.for_project(project): + updated, __ = service.update_webhook(project, integration) + if updated: + messages.success(request, _("Webhook activated")) + project.has_valid_webhook = True + project.save() + return True messages.error( request, diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index a2ea85c862d..d16b4872ced 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -7,7 +7,6 @@ from urllib.parse import urlparse import structlog -from allauth.socialaccount.providers import registry as allauth_registry from django.conf import settings from django.contrib.auth.models import User from django.contrib.contenttypes.fields import GenericRelation @@ -1025,28 +1024,36 @@ def vcs_class(self): """ return backend_cls.get(self.repo_type) - def git_service_class(self): - """Get the service class for project. e.g: GitHubService, GitLabService.""" + def _guess_service_class(self): from readthedocs.oauth.services import registry for service_cls in registry: if service_cls.is_project_service(self): - service = service_cls - break - else: - log.warning("There are no registered services in the application.") - service = None + return service_cls + return None - return service + def get_git_service_class(self, fallback_to_clone_url=False): + """ + Get the service class for project. e.g: GitHubService, GitLabService. + + :param fallback_to_clone_url: If the project doesn't have a remote repository, + we try to guess the service class based on the clone URL. + """ + service_cls = None + if self.has_feature(Feature.DONT_SYNC_WITH_REMOTE_REPO): + return self._guess_service_class() + service_cls = ( + self.remote_repository and self.remote_repository.get_service_class() + ) + if not service_cls and fallback_to_clone_url: + return self._guess_service_class() + return service_cls @property def git_provider_name(self): """Get the provider name for project. e.g: GitHub, GitLab, Bitbucket.""" - service = self.git_service_class() - if service: - provider_class = allauth_registry.get_class(service.adapter.provider_id) - return provider_class.name - return None + service_class = self.get_git_service_class() + return service_class.provider_name if service_class else None def find(self, filename, version): """ diff --git a/readthedocs/projects/views/mixins.py b/readthedocs/projects/views/mixins.py index 66c0fa92623..1a112299fab 100644 --- a/readthedocs/projects/views/mixins.py +++ b/readthedocs/projects/views/mixins.py @@ -140,6 +140,8 @@ def trigger_initial_build(self, project, user): from readthedocs.oauth.tasks import attach_webhook task_promise = chain( + # TODO: Remove user_pk on the next release, + # it's used just to keep backward compatibility with the old task signature. attach_webhook.si(project.pk, user.pk), update_docs, ) diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index 56812ec4d3a..aeacb38c81c 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -999,8 +999,10 @@ def form_valid(self, form): if self.object.has_sync: attach_webhook( project_pk=self.get_project().pk, - user_pk=self.request.user.pk, integration=self.object, + # TODO: Remove user_pk on the next release, + # it's used just to keep backward compatibility with the old task signature. + user_pk=None, ) return HttpResponseRedirect(self.get_success_url()) @@ -1056,7 +1058,9 @@ def post(self, request, *args, **kwargs): # the per-integration sync instead. attach_webhook( project_pk=self.get_project().pk, - user_pk=request.user.pk, + # TODO: Remove user_pk on the next release, + # it's used just to keep backward compatibility with the old task signature. + user_pk=None, ) return HttpResponseRedirect(self.get_success_url()) From f031b4b3b453d41b5959260220d8c650c8cfe38d Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 6 Feb 2025 19:10:47 -0500 Subject: [PATCH 16/92] Fix name --- readthedocs/oauth/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index 59125bbdfa9..c157da99bf6 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -200,7 +200,7 @@ def attach_webhook(project_pk, user_pk=None, integration=None, **kwargs): dismissable=True, attached_to=project, format_values={ - "provider_name": service_class.vcs_provider_name, + "provider_name": service_class.provider_name, "url_connect_account": reverse( "projects_integrations", args=[project.slug], @@ -221,7 +221,7 @@ def attach_webhook(project_pk, user_pk=None, integration=None, **kwargs): dismissable=True, attached_to=project, format_values={ - "provider_name": service_class.vcs_provider_name, + "provider_name": service_class.provider_name, "url_docs_webhook": "https://docs.readthedocs.io/page/webhooks.html", }, ) From f7a3917b3d447af17bc322860dadcabce4415350 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Feb 2025 11:24:29 -0500 Subject: [PATCH 17/92] Fix tests --- readthedocs/projects/models.py | 2 +- readthedocs/rtd_tests/tests/test_celery.py | 7 ++++--- readthedocs/rtd_tests/tests/test_oauth.py | 4 ++-- readthedocs/rtd_tests/tests/test_oauth_sync.py | 2 +- readthedocs/rtd_tests/tests/test_project.py | 10 ++++++++-- readthedocs/rtd_tests/tests/test_project_views.py | 2 +- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index d16b4872ced..7b4a33d0cbd 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1052,7 +1052,7 @@ def get_git_service_class(self, fallback_to_clone_url=False): @property def git_provider_name(self): """Get the provider name for project. e.g: GitHub, GitLab, Bitbucket.""" - service_class = self.get_git_service_class() + service_class = self.get_git_service_class(fallback_to_clone_url=True) return service_class.provider_name if service_class else None def find(self, filename, version): diff --git a/readthedocs/rtd_tests/tests/test_celery.py b/readthedocs/rtd_tests/tests/test_celery.py index b64e91233fe..cd0bcf7f366 100644 --- a/readthedocs/rtd_tests/tests/test_celery.py +++ b/readthedocs/rtd_tests/tests/test_celery.py @@ -10,6 +10,7 @@ from readthedocs.builds.constants import BUILD_STATUS_SUCCESS, EXTERNAL, LATEST from readthedocs.builds.models import Build, Version from readthedocs.notifications.models import Notification +from readthedocs.oauth.constants import GITHUB, GITLAB from readthedocs.oauth.models import RemoteRepository, RemoteRepositoryRelation from readthedocs.oauth.notifications import MESSAGE_OAUTH_BUILD_STATUS_FAILURE from readthedocs.projects.models import Project @@ -85,8 +86,8 @@ def test_send_build_status_with_remote_repo_github(self, send_build_status): self.project.repo = "https://github.com/test/test/" self.project.save() - social_account = get(SocialAccount, user=self.eric, provider="gitlab") - remote_repo = get(RemoteRepository) + social_account = get(SocialAccount, user=self.eric, provider="github") + remote_repo = get(RemoteRepository, vcs_provider=GITHUB) remote_repo.projects.add(self.project) get( RemoteRepositoryRelation, @@ -159,7 +160,7 @@ def test_send_build_status_with_remote_repo_gitlab(self, send_build_status): self.project.save() social_account = get(SocialAccount, user=self.eric, provider="gitlab") - remote_repo = get(RemoteRepository) + remote_repo = get(RemoteRepository, vcs_provider=GITLAB) remote_repo.projects.add(self.project) get( RemoteRepositoryRelation, diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index 9e84886e0b5..ba206f12a7a 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -254,7 +254,7 @@ def test_make_organization(self): def test_import_with_no_token(self): """User without a GitHub SocialToken does not return a service.""" - services = GitHubService.for_user(get(User)) + services = list(GitHubService.for_user(get(User))) self.assertEqual(services, []) def test_multiple_users_same_repo(self): @@ -776,7 +776,7 @@ def test_make_organization(self): def test_import_with_no_token(self): """User without a Bitbucket SocialToken does not return a service.""" - services = BitbucketService.for_user(get(User)) + services = list(BitbucketService.for_user(get(User))) self.assertEqual(services, []) @mock.patch("readthedocs.oauth.services.bitbucket.log") diff --git a/readthedocs/rtd_tests/tests/test_oauth_sync.py b/readthedocs/rtd_tests/tests/test_oauth_sync.py index 43bc308de5c..5773f6e6357 100644 --- a/readthedocs/rtd_tests/tests/test_oauth_sync.py +++ b/readthedocs/rtd_tests/tests/test_oauth_sync.py @@ -70,7 +70,7 @@ def setUp(self): SocialToken, account=self.socialaccount, ) - self.service = GitHubService.for_user(self.user)[0] + self.service = list(GitHubService.for_user(self.user))[0] @requests_mock.Mocker(kw="mock_request") def test_sync_delete_stale(self, mock_request): diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 7a1e238f302..befa60bccb1 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -239,7 +239,10 @@ def test_git_provider_name_github(self): def test_git_service_class_github(self): self.pip.repo = "https://github.com/pypa/pip" self.pip.save() - self.assertEqual(self.pip.git_service_class(), GitHubService) + self.assertEqual(self.pip.get_git_service_class(), None) + self.assertEqual( + self.pip.get_git_service_class(fallback_to_clone_url=True), GitHubService + ) def test_git_provider_name_gitlab(self): self.pip.repo = "https://gitlab.com/pypa/pip" @@ -249,7 +252,10 @@ def test_git_provider_name_gitlab(self): def test_git_service_class_gitlab(self): self.pip.repo = "https://gitlab.com/pypa/pip" self.pip.save() - self.assertEqual(self.pip.git_service_class(), GitLabService) + self.assertEqual(self.pip.get_git_service_class(), None) + self.assertEqual( + self.pip.get_git_service_class(fallback_to_clone_url=True), GitLabService + ) @mock.patch("readthedocs.projects.forms.trigger_build", mock.MagicMock()) diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index e760c64a690..2f78690af71 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -455,8 +455,8 @@ def test_integration_create(self, attach_webhook): self.assertEqual(response.status_code, 302) attach_webhook.assert_called_once_with( project_pk=self.project.pk, - user_pk=self.user.pk, integration=integration.first(), + user_pk=None, ) @mock.patch("readthedocs.projects.views.private.attach_webhook") From 083cfa012a4e3e0218a0f7eca80ff2c33e0c582e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Feb 2025 12:08:44 -0500 Subject: [PATCH 18/92] Small updates --- readthedocs/oauth/services/base.py | 7 ++++--- readthedocs/oauth/tasks.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 7f570acd3ab..8596303e3f0 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -2,6 +2,7 @@ from datetime import datetime import structlog +import re from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.providers.oauth2.views import OAuth2Adapter from django.conf import settings @@ -32,7 +33,7 @@ class Service: """Base class for service that interacts with a VCS provider and a project.""" vcs_provider_slug: str - url_pattern: str + url_pattern: re.Pattern | None provider_name: str default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL @@ -53,8 +54,8 @@ def sync(self): - Creates a new RemoteRepository/Organization per new repository - Updates fields for existing RemoteRepository/Organization - - Deletes old RemoteRepository/Organization that are not present - for this user in the current provider + - Deletes old RemoteRepository/Organization that are no longer present + in this provider. """ raise NotImplementedError diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index c157da99bf6..13a6185c1f1 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -209,7 +209,7 @@ def attach_webhook(project_pk, user_pk=None, integration=None, **kwargs): ) return False - for service in service_class.for_project(project): + for service in services: success, _ = service.setup_webhook(project, integration=integration) if success: project.has_valid_webhook = True From 8e6caa3e6157a0230c0989784e8a421cacd7f7bd Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Feb 2025 12:18:49 -0500 Subject: [PATCH 19/92] Format --- readthedocs/oauth/services/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 8596303e3f0..941a7f277b5 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -1,8 +1,8 @@ """OAuth utility functions.""" +import re from datetime import datetime import structlog -import re from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.providers.oauth2.views import OAuth2Adapter from django.conf import settings From 6d686f3a5042273db22459bf73808709a4a65eae Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Feb 2025 13:06:55 -0500 Subject: [PATCH 20/92] Format --- readthedocs/oauth/models.py | 1 + readthedocs/oauth/views.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 81ecc7f745a..25cde9cf0c3 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -1,5 +1,6 @@ """OAuth service models.""" from functools import cached_property + from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import User from django.core.validators import URLValidator diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index b438565c3af..9f615e5a700 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -435,7 +435,6 @@ def _handle_github_app_authorization_event(self): See https://docs.github.com/en/webhooks/webhook-events-and-payloads#github_app_authorization. """ - pass def _get_projects(self): remote_repository = self._get_remote_repository() From d771092e29c6347659d8c29f7b6c71dde8d9725b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Feb 2025 14:28:43 -0500 Subject: [PATCH 21/92] Refactor --- readthedocs/oauth/services/base.py | 17 ++- readthedocs/oauth/services/bitbucket.py | 2 +- readthedocs/oauth/services/github.py | 2 +- readthedocs/oauth/services/githubapp.py | 136 +++++++++++++++--------- readthedocs/oauth/services/gitlab.py | 2 +- readthedocs/oauth/views.py | 8 +- readthedocs/projects/models.py | 12 +-- 7 files changed, 117 insertions(+), 62 deletions(-) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 941a7f277b5..8c32ddff1b1 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -33,8 +33,8 @@ class Service: """Base class for service that interacts with a VCS provider and a project.""" vcs_provider_slug: str - url_pattern: re.Pattern | None provider_name: str + url_pattern: re.Pattern | None = None default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL @@ -85,7 +85,7 @@ def update_webhook(self, project, integration): """ raise NotImplementedError - def send_build_status(self, build, commit, status): + def send_build_status(self, *, build, commit, status): """ Create commit status for project. @@ -100,6 +100,15 @@ def send_build_status(self, build, commit, status): """ raise NotImplementedError + def get_clone_token(self, project): + """ + Get a clone token for the project. + + :param project: project to get clone token for + :type project: Project + """ + raise NotImplementedError + @classmethod def is_project_service(cls, project): """ @@ -389,3 +398,7 @@ def sync_repositories(self): def sync_organizations(self): raise NotImplementedError + + def get_clone_token(self, project): + """User services make use of SSH keys only for cloning.""" + return None diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 992a7db3996..be71adfa117 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -395,6 +395,6 @@ def update_webhook(self, project, integration): return (False, resp) - def send_build_status(self, build, commit, status): + def send_build_status(self, *, build, commit, status): """Send build status is not supported/implemented for Bitbucket.""" return True diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 4dacfb5a2ce..0e8cf81d33f 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -408,7 +408,7 @@ def update_webhook(self, project, integration): return (False, resp) - def send_build_status(self, build, commit, status): + def send_build_status(self, *, build, commit, status): """ Create GitHub commit status for project. diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 71896942c1c..abf9b6f62d3 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -3,7 +3,7 @@ import structlog from allauth.socialaccount.models import SocialAccount from django.conf import settings -from github import Auth, GithubIntegration +from github import Auth, Github, GithubIntegration from github.Installation import Installation as GHInstallation from github.Organization import Organization as GHOrganization from github.Repository import Repository as GHRepository @@ -19,63 +19,84 @@ RemoteRepository, RemoteRepositoryRelation, ) +from readthedocs.oauth.services.base import Service log = structlog.get_logger(__name__) -class GitHubAppClient: - def __init__(self, installation_id: int): - self.installation_id = installation_id +# TODO: cache this? +def get_gh_app_client() -> GithubIntegration: + """Return a client authenticated as the GitHub App to interact with the API""" + app_auth = Auth.AppAuth( + app_id=settings.GITHUB_APP_CLIENT_ID, + private_key=settings.GITHUB_APP_PRIVATE_KEY, + # 10 minutes is the maximum allowed by GitHub. + # PyGithub will handle the token expiration and renew it automatically. + jwt_expiry=60 * 10, + ) + return GithubIntegration(auth=app_auth) - def _get_auth(self): - app_auth = Auth.AppAuth( - app_id=settings.GITHUB_APP_CLIENT_ID, - private_key=settings.GITHUB_APP_PRIVATE_KEY, - # 10 minutes is the maximum allowed by GitHub. - # PyGithub will handle the token expiration and renew it automatically. - jwt_expiry=60 * 10, - ) - return app_auth - @cached_property - def integration_client(self): - """Return a client authenticated as the GitHub App to interact with the installation API""" - return GithubIntegration(auth=self._get_auth()) +class GitHubAppService(Service): + vcs_provider_slug = GITHUB + provider_name = "GitHub" - @cached_property - def client(self): - """Return a client authenticated as the GitHub App to interact with most of the GH API""" - return self.integration_client.get_github_for_installation(self.installation_id) + def __init__(self, installation: GitHubAppInstallation): + self.installation = installation + self.gha_client = get_gh_app_client() @cached_property def app_installation(self) -> GHInstallation: - return self.integration_client.get_app_installation(self.installation_id) - - def get_installation_token(self, permissions: dict | None = None): - """ - - https://docs.github.com/en/rest/apps/apps?apiVersion=2022-11-28#create-an-installation-access-token-for-an-app - """ - # TODO: we can pass the repository_ids to get a token with access to specific repositories. - # We should upstream this feature to PyGithub. - return self.integration_client.get_access_token( - self.installation_id, permissions=permissions - ).token - + return self.gha_client.get_app_installation( + self.installation.installation_id, + ) -class GitHubAppService: - vcs_provider_slug = GITHUB + @cached_property + def installation_client(self) -> Github: + """Return a client authenticated as the GitHub installation to interact with the GH API.""" + return self.gha_client.get_github_for_installation( + self.installation.installation_id + ) - def __init__(self, installation: GitHubAppInstallation): - self.installation = installation - self.gha_client = GitHubAppClient(self.installation.installation_id) + @classmethod + def for_project(cls, project): + if ( + not project.remote_repository + or not project.remote_repository.github_app_installation + ): + return None - def sync_repositories(self): - return self._sync_installation_repositories() + yield cls(project.remote_repository.github_app_installation) - def _sync_installation_repositories(self): + @classmethod + def for_user(cls, user): + social_accounts = SocialAccount.objects.filter( + user=user, + provider=GitHubAppProvider.id, + ) + for account in social_accounts: + account_id = account.uid + account_login = account.extra_data.get("login") + + # GH doens't have an API to get the user based on the account_id, + # so we need to make sure we are getting the correct user. + # We don't want to mix up the accounts of different users in case the account was renamed. + gh_installation = get_gh_app_client().get_user_installation(account_login) + if gh_installation.target_id != account_id: + continue + + # TODO: get or create the installation object. + installation = GitHubAppInstallation.objects.filter( + installation_id=gh_installation.id + ).first() + if installation: + yield cls(installation) + + # TODO: what about organizations installations? + + def sync(self): remote_repositories = [] - for repo in self.gha_client.app_installation.get_repos(): + for repo in self.app_installation.get_repos(): remote_repo = self._create_or_update_repository_from_gh(repo) if remote_repo: remote_repositories.append(remote_repo) @@ -92,7 +113,7 @@ def _sync_installation_repositories(self): def update_or_create_repositories(self, repository_ids: list[int]): for repository_id in repository_ids: - repo = self.gha_client.client.get_repo(repository_id) + repo = self.installation_client.get_repo(repository_id) self._create_or_update_repository_from_gh(repo) def delete_repositories(self, repository_ids: list[int]): @@ -206,7 +227,7 @@ def _get_gh_organization(self, org_id: int) -> GHOrganization: # NOTE: cast to str, since PyGithub expects a string, # even if the API accepts a string or an int. # TODO: send a PR upstream to fix this. - return self.gha_client.client.get_organization(str(org_id)) + return self.installation_client.get_organization(str(org_id)) # NOTE: normally, this should cache only one organization at a time, but just in case... @lru_cache(maxsize=50) @@ -314,7 +335,7 @@ def send_build_status(self, *, build, commit, status): description = SELECT_BUILD_STATUS[status]["description"] context = f"{settings.RTD_BUILD_STATUS_API_NAME}:{project.slug}" - gh_repo = self.gha_client.client.get_repo(int(remote_repo.remote_id)) + gh_repo = self.installation_client.get_repo(int(remote_repo.remote_id)) gh_repo.get_commit(commit).create_status( state=state, target_url=target_url, @@ -322,5 +343,24 @@ def send_build_status(self, *, build, commit, status): context=context, ) - def get_installation_token(self, permissions: dict | None = None): - return self.gha_client.get_installation_token(permissions=permissions) + def get_clone_token(self, project): + """ + See https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation. + + https://docs.github.com/en/rest/apps/apps?apiVersion=2022-11-28#create-an-installation-access-token-for-an-app + """ + # TODO: we can pass the repository_ids to get a token with access to specific repositories. + # We should upstream this feature to PyGithub. + # We can also pass a specific permissions object to get a token with specific permissions. + access_token = self.gha_client.get_access_token( + self.installation.installation_id + ) + return f"x-access-token:{access_token.token}" + + def setup_webhook(self, project, integration=None): + """When using a GitHub App, we don't need to set up a webhook.""" + return True + + def update_webhook(self, project, integration=None): + """When using a GitHub App, we don't need to set up a webhook.""" + return True diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 4e65012c566..7a986e0f291 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -497,7 +497,7 @@ def update_webhook(self, project, integration): return (False, resp) - def send_build_status(self, build, commit, status): + def send_build_status(self, *, build, commit, status): """ Create GitLab commit status for project. diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 9f615e5a700..5fe0bab3948 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -21,7 +21,7 @@ trigger_sync_versions, ) from readthedocs.oauth.models import GitHubAppInstallation -from readthedocs.oauth.services.githubapp import GitHubAppClient +from readthedocs.oauth.services.githubapp import get_gh_app_client from readthedocs.projects.models import Project log = structlog.get_logger(__name__) @@ -30,6 +30,10 @@ class GitHubAppWebhookView(APIView): authentication_classes = [] + def __init__(self, **kwargs): + self.gha_client = get_gh_app_client() + super().__init__(**kwargs) + def post(self, request): if not self._is_payload_signature_valid(): raise ValidationError("Invalid webhook signature") @@ -471,7 +475,7 @@ def _get_or_create_installation(self, sync_repositories_on_create: bool = True): # If they aren't present, fetch them from the API, # so we can create the installation object if needed. if not target_id or not target_type: - gh_installation = GitHubAppClient(installation_id).app_installation + gh_installation = self.gha_client.get_app_installation(installation_id) target_id = gh_installation.target_id target_type = gh_installation.target_type data = data.copy() diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index fbfa0411520..696cc2aa60b 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1446,13 +1446,11 @@ def organization(self): @property def clone_token(self): - """ - See https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation. - """ - remote_repository = self.remote_repository - if remote_repository and remote_repository.github_app_installation: - service = remote_repository.github_app_installation.service - return f"x-access-token:{service.get_installation_token()}" + service_class = self.get_git_service_class() + for service in service_class.for_project(self): + token = service.get_clone_token(self) + if token: + return token return None From 0d0511c15f1bd67915a69dc4774293b21e5c825d Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Feb 2025 15:39:55 -0500 Subject: [PATCH 22/92] Fix import --- readthedocs/oauth/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 25cde9cf0c3..e6c71916928 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -1,6 +1,7 @@ """OAuth service models.""" from functools import cached_property +import structlog from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import User from django.core.validators import URLValidator @@ -280,6 +281,10 @@ def get_remote_repository_relation(self, user, social_account): def get_service_class(self): from readthedocs.oauth.services import registry + from readthedocs.oauth.services.githubapp import GitHubAppService + + if self.github_app_installation: + return GitHubAppService for service_cls in registry: if service_cls.vcs_provider_slug == self.vcs_provider: From 4308bbf64f59e6ba26165538f81afcccfd498d6c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Feb 2025 15:43:13 -0500 Subject: [PATCH 23/92] Check for None --- readthedocs/projects/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 696cc2aa60b..07e66b47e85 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1447,6 +1447,8 @@ def organization(self): @property def clone_token(self): service_class = self.get_git_service_class() + if not service_class: + return None for service in service_class.for_project(self): token = service.get_clone_token(self) if token: From 7065003e2cbe88d59775261d5cb07a0aeeeffd7d Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Feb 2025 18:05:08 -0500 Subject: [PATCH 24/92] refactor --- readthedocs/oauth/clients.py | 86 +++++++++++++++++++++++ readthedocs/oauth/services/base.py | 91 +++---------------------- readthedocs/oauth/services/bitbucket.py | 9 +-- readthedocs/oauth/services/github.py | 14 ++-- readthedocs/oauth/services/githubapp.py | 48 +++++-------- readthedocs/oauth/services/gitlab.py | 14 ++-- 6 files changed, 125 insertions(+), 137 deletions(-) create mode 100644 readthedocs/oauth/clients.py diff --git a/readthedocs/oauth/clients.py b/readthedocs/oauth/clients.py new file mode 100644 index 00000000000..a1607cad1f1 --- /dev/null +++ b/readthedocs/oauth/clients.py @@ -0,0 +1,86 @@ +from datetime import datetime + +import structlog +from django.conf import settings +from django.utils import timezone +from github import Auth, GithubIntegration +from requests_oauthlib import OAuth2Session + +log = structlog.get_logger(__name__) + + +def _get_token_updater(token): + """ + Update token given data from OAuth response. + + Expect the following response into the closure:: + + { + u'token_type': u'bearer', + u'scopes': u'webhook repository team account', + u'refresh_token': u'...', + u'access_token': u'...', + u'expires_in': 3600, + u'expires_at': 1449218652.558185 + } + """ + + def _updater(data): + token.token = data["access_token"] + token.token_secret = data.get("refresh_token", "") + token.expires_at = timezone.make_aware( + datetime.fromtimestamp(data["expires_at"]), + ) + token.save() + log.info("Updated token.", token_id=token.pk) + + return _updater + + +def get_oauth2_client(account): + """Get an OAuth2 client for the given social account.""" + token = account.socialtoken_set.first() + if token is None: + return None + + token_config = { + "access_token": token.token, + "token_type": "bearer", + } + if token.expires_at is not None: + token_expires = (token.expires_at - timezone.now()).total_seconds() + token_config.update( + { + "refresh_token": token.token_secret, + "expires_in": token_expires, + } + ) + + provider = account.get_provider() + social_app = provider.app + oauth2_adapter = provider.get_oauth2_adapter(request=provider.request) + + session = OAuth2Session( + client_id=social_app.client_id, + token=token_config, + auto_refresh_kwargs={ + "client_id": social_app.client_id, + "client_secret": social_app.secret, + }, + auto_refresh_url=oauth2_adapter.access_token_url, + token_updater=_get_token_updater(token), + ) + return session + + +# TODO: cache this? +def get_gh_app_client() -> GithubIntegration: + """Return a client authenticated as the GitHub App to interact with the API""" + app_auth = Auth.AppAuth( + app_id=settings.GITHUB_APP_CLIENT_ID, + private_key=settings.GITHUB_APP_PRIVATE_KEY, + # 10 minutes is the maximum allowed by GitHub. + # PyGithub will handle the token expiration and renew it automatically. + jwt_expiry=60 * 10, + ) + return GithubIntegration(auth=app_auth) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 8c32ddff1b1..6bd3ba5fa70 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -1,25 +1,24 @@ """OAuth utility functions.""" + import re -from datetime import datetime +from functools import cached_property import structlog from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.providers.oauth2.views import OAuth2Adapter from django.conf import settings from django.urls import reverse -from django.utils import timezone from django.utils.translation import gettext_lazy as _ from oauthlib.oauth2.rfc6749.errors import InvalidClientIdError from requests.exceptions import RequestException -from requests_oauthlib import OAuth2Session +from readthedocs.allauth.clients import get_oauth2_client from readthedocs.core.permissions import AdminPermission log = structlog.get_logger(__name__) class SyncServiceError(Exception): - """Error raised when a service failed to sync.""" INVALID_OR_REVOKED_ACCESS_TOKEN = _( @@ -29,7 +28,6 @@ class SyncServiceError(Exception): class Service: - """Base class for service that interacts with a VCS provider and a project.""" vcs_provider_slug: str @@ -101,12 +99,7 @@ def send_build_status(self, *, build, commit, status): raise NotImplementedError def get_clone_token(self, project): - """ - Get a clone token for the project. - - :param project: project to get clone token for - :type project: Project - """ + """Get a token used for cloning the repository.""" raise NotImplementedError @classmethod @@ -127,7 +120,6 @@ def is_project_service(cls, project): class UserService(Service): - """ Subclass of Service that interacts with a VCS provider using the user's OAuth token. @@ -138,7 +130,6 @@ class UserService(Service): adapter = None def __init__(self, user, account): - self.session = None self.user = user self.account = account log.bind( @@ -169,18 +160,9 @@ def get_adapter(self) -> type[OAuth2Adapter]: def provider_id(self): return self.get_adapter().provider_id - def get_session(self): - if self.session is None: - self.create_session() - return self.session - - def get_access_token_url(self): - # ``access_token_url`` is a property in some adapters, - # so we need to instantiate it to get the actual value. - # pylint doesn't recognize that get_adapter returns a class. - # pylint: disable=not-callable - adapter = self.get_adapter()(request=None) - return adapter.access_token_url + @cached_property + def session(self): + return get_oauth2_client(self.account) def create_session(self): """ @@ -190,63 +172,6 @@ def create_session(self): attributes. If there is an ``expires_at``, treat the session as an auto renewing token. Some providers expire tokens after as little as 2 hours. """ - token = self.account.socialtoken_set.first() - if token is None: - return None - - token_config = { - "access_token": token.token, - "token_type": "bearer", - } - if token.expires_at is not None: - token_expires = (token.expires_at - timezone.now()).total_seconds() - token_config.update( - { - "refresh_token": token.token_secret, - "expires_in": token_expires, - } - ) - - social_app = self.account.get_provider().app - self.session = OAuth2Session( - client_id=social_app.client_id, - token=token_config, - auto_refresh_kwargs={ - "client_id": social_app.client_id, - "client_secret": social_app.secret, - }, - auto_refresh_url=self.get_access_token_url(), - token_updater=self.token_updater(token), - ) - - return self.session or None - - def token_updater(self, token): - """ - Update token given data from OAuth response. - - Expect the following response into the closure:: - - { - u'token_type': u'bearer', - u'scopes': u'webhook repository team account', - u'refresh_token': u'...', - u'access_token': u'...', - u'expires_in': 3600, - u'expires_at': 1449218652.558185 - } - """ - - def _updater(data): - token.token = data["access_token"] - token.token_secret = data.get("refresh_token", "") - token.expires_at = timezone.make_aware( - datetime.fromtimestamp(data["expires_at"]), - ) - token.save() - log.info("Updated token.", token_id=token.pk) - - return _updater def paginate(self, url, **kwargs): """ @@ -259,7 +184,7 @@ def paginate(self, url, **kwargs): """ resp = None try: - resp = self.get_session().get(url, params=kwargs) + resp = self.session.get(url, params=kwargs) # TODO: this check of the status_code would be better in the # ``create_session`` method since it could be used from outside, but diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index be71adfa117..e6d8a7255f8 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -235,7 +235,6 @@ def get_provider_data(self, project, integration): if integration.provider_data: return integration.provider_data - session = self.get_session() owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) url = f"https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/hooks" @@ -247,7 +246,7 @@ def get_provider_data(self, project, integration): url=url, ) try: - resp = session.get(url) + resp = self.session.get(url) if resp.status_code == 200: recv_data = resp.json() @@ -284,7 +283,6 @@ def setup_webhook(self, project, integration=None): :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ - session = self.get_session() owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) url = f"https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/hooks" if not integration: @@ -302,7 +300,7 @@ def setup_webhook(self, project, integration=None): ) try: - resp = session.post( + resp = self.session.post( url, data=data, headers={"content-type": "application/json"}, @@ -355,13 +353,12 @@ def update_webhook(self, project, integration): if not provider_data: return self.setup_webhook(project, integration) - session = self.get_session() data = self.get_webhook_data(project, integration) resp = None try: # Expect to throw KeyError here if provider_data is invalid url = provider_data["links"]["self"]["href"] - resp = session.put( + resp = self.session.put( url, data=data, headers={"content-type": "application/json"}, diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 0e8cf81d33f..7248da1b6c4 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -56,7 +56,7 @@ def sync_organizations(self): try: orgs = self.paginate("https://api.github.com/user/orgs", per_page=100) for org in orgs: - org_details = self.get_session().get(org["url"]).json() + org_details = self.session.get(org["url"]).json() remote_organization = self.create_organization( org_details, create_user_relationship=True, @@ -240,7 +240,6 @@ def get_provider_data(self, project, integration): if integration.provider_data: return integration.provider_data - session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) url = f"https://api.github.com/repos/{owner}/{repo}/hooks" log.bind( @@ -252,7 +251,7 @@ def get_provider_data(self, project, integration): rtd_webhook_url = self.get_webhook_url(project, integration) try: - resp = session.get(url) + resp = self.session.get(url) if resp.status_code == 200: recv_data = resp.json() @@ -287,7 +286,6 @@ def setup_webhook(self, project, integration=None): :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ - session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) if not integration: @@ -305,7 +303,7 @@ def setup_webhook(self, project, integration=None): ) resp = None try: - resp = session.post( + resp = self.session.post( url, data=data, headers={"content-type": "application/json"}, @@ -352,7 +350,6 @@ def update_webhook(self, project, integration): :returns: boolean based on webhook update success, and requests Response object :rtype: (Bool, Response) """ - session = self.get_session() data = self.get_webhook_data(project, integration) resp = None @@ -369,7 +366,7 @@ def update_webhook(self, project, integration): try: url = provider_data.get("url") - resp = session.patch( + resp = self.session.patch( url, data=data, headers={"content-type": "application/json"}, @@ -421,7 +418,6 @@ def send_build_status(self, *, build, commit, status): :returns: boolean based on commit status creation was successful or not. :rtype: Bool """ - session = self.get_session() project = build.project owner, repo = build_utils.get_github_username_repo(url=project.repo) @@ -456,7 +452,7 @@ def send_build_status(self, *, build, commit, status): ) resp = None try: - resp = session.post( + resp = self.session.post( statuses_url, data=json.dumps(data), headers={"content-type": "application/json"}, diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index abf9b6f62d3..b4f63b06999 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -3,13 +3,15 @@ import structlog from allauth.socialaccount.models import SocialAccount from django.conf import settings -from github import Auth, Github, GithubIntegration +from github import Github from github.Installation import Installation as GHInstallation from github.Organization import Organization as GHOrganization from github.Repository import Repository as GHRepository from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider +from readthedocs.allauth.providers.githubapp.views import GitHubAppOAuth2Adapter from readthedocs.builds.constants import BUILD_STATUS_SUCCESS, SELECT_BUILD_STATUS +from readthedocs.oauth.clients import get_gh_app_client, get_oauth2_client from readthedocs.oauth.constants import GITHUB from readthedocs.oauth.models import ( GitHubAccountType, @@ -24,22 +26,10 @@ log = structlog.get_logger(__name__) -# TODO: cache this? -def get_gh_app_client() -> GithubIntegration: - """Return a client authenticated as the GitHub App to interact with the API""" - app_auth = Auth.AppAuth( - app_id=settings.GITHUB_APP_CLIENT_ID, - private_key=settings.GITHUB_APP_PRIVATE_KEY, - # 10 minutes is the maximum allowed by GitHub. - # PyGithub will handle the token expiration and renew it automatically. - jwt_expiry=60 * 10, - ) - return GithubIntegration(auth=app_auth) - - class GitHubAppService(Service): vcs_provider_slug = GITHUB provider_name = "GitHub" + adapter = GitHubAppOAuth2Adapter def __init__(self, installation: GitHubAppInstallation): self.installation = installation @@ -70,29 +60,27 @@ def for_project(cls, project): @classmethod def for_user(cls, user): + """ + https://docs.github.com/en/rest/apps/installations?apiVersion=2022-11-28#list-app-installations-accessible-to-the-user-access-token instead. + """ social_accounts = SocialAccount.objects.filter( user=user, provider=GitHubAppProvider.id, ) for account in social_accounts: - account_id = account.uid - account_login = account.extra_data.get("login") - - # GH doens't have an API to get the user based on the account_id, - # so we need to make sure we are getting the correct user. - # We don't want to mix up the accounts of different users in case the account was renamed. - gh_installation = get_gh_app_client().get_user_installation(account_login) - if gh_installation.target_id != account_id: - continue + oauth2_client = get_oauth2_client(account) + resp = oauth2_client.get("https://api.github.com/app/installations") - # TODO: get or create the installation object. - installation = GitHubAppInstallation.objects.filter( - installation_id=gh_installation.id - ).first() - if installation: - yield cls(installation) + if resp.status_code != 200: + continue - # TODO: what about organizations installations? + for gh_installation in resp.json()["installations"]: + # TODO: get or create the installation object. + installation = GitHubAppInstallation.objects.filter( + installation_id=gh_installation["id"], + ).first() + if installation: + yield cls(installation) def sync(self): remote_repositories = [] diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 7a986e0f291..df5b0cb9e36 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -130,7 +130,7 @@ def sync_organizations(self): # admin permission fields for GitLab projects. # So, fetch every single project data from the API # which contains the admin permission fields. - resp = self.get_session().get( + resp = self.session.get( "{url}/api/v4/projects/{id}".format( url=self.adapter.provider_default_url, id=repo["id"] ) @@ -326,7 +326,6 @@ def get_provider_data(self, project, integration): if repo_id is None: return None - session = self.get_session() log.bind( project_slug=project.slug, integration_id=integration.pk, @@ -335,7 +334,7 @@ def get_provider_data(self, project, integration): rtd_webhook_url = self.get_webhook_url(project, integration) try: - resp = session.get( + resp = self.session.get( "{url}/api/v4/projects/{repo_id}/hooks".format( url=self.adapter.provider_default_url, repo_id=repo_id, @@ -395,9 +394,8 @@ def setup_webhook(self, project, integration=None): url=url, ) data = self.get_webhook_data(repo_id, project, integration) - session = self.get_session() try: - resp = session.post( + resp = self.session.post( url, data=data, headers={"content-type": "application/json"}, @@ -447,7 +445,6 @@ def update_webhook(self, project, integration): return self.setup_webhook(project, integration) resp = None - session = self.get_session() repo_id = self._get_repo_id(project) if repo_id is None: @@ -461,7 +458,7 @@ def update_webhook(self, project, integration): ) try: hook_id = provider_data.get("id") - resp = session.put( + resp = self.session.put( "{url}/api/v4/projects/{repo_id}/hooks/{hook_id}".format( url=self.adapter.provider_default_url, repo_id=repo_id, @@ -511,7 +508,6 @@ def send_build_status(self, *, build, commit, status): :rtype: Bool """ resp = None - session = self.get_session() project = build.project repo_id = self._get_repo_id(project) @@ -547,7 +543,7 @@ def send_build_status(self, *, build, commit, status): url=url, ) try: - resp = session.post( + resp = self.session.post( url, data=json.dumps(data), headers={"content-type": "application/json"}, From b2a8a2bde9e1250ff3b3c3f326767596fdd5dd21 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Feb 2025 18:08:56 -0500 Subject: [PATCH 25/92] Fix import --- readthedocs/oauth/services/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 6bd3ba5fa70..c84ede01c03 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -12,8 +12,8 @@ from oauthlib.oauth2.rfc6749.errors import InvalidClientIdError from requests.exceptions import RequestException -from readthedocs.allauth.clients import get_oauth2_client from readthedocs.core.permissions import AdminPermission +from readthedocs.oauth.clients import get_oauth2_client log = structlog.get_logger(__name__) From b0d9dccc34a691f4680bd527a39b1750b372f79c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Feb 2025 11:17:47 -0500 Subject: [PATCH 26/92] Full feature! --- readthedocs/oauth/clients.py | 1 - readthedocs/oauth/models.py | 33 +++++++++++++++++++++++ readthedocs/oauth/services/__init__.py | 9 +++++-- readthedocs/oauth/services/githubapp.py | 14 +++++----- readthedocs/oauth/views.py | 36 +++++++------------------ 5 files changed, 57 insertions(+), 36 deletions(-) diff --git a/readthedocs/oauth/clients.py b/readthedocs/oauth/clients.py index a1607cad1f1..d3ad8164456 100644 --- a/readthedocs/oauth/clients.py +++ b/readthedocs/oauth/clients.py @@ -73,7 +73,6 @@ def get_oauth2_client(account): return session -# TODO: cache this? def get_gh_app_client() -> GithubIntegration: """Return a client authenticated as the GitHub App to interact with the API""" app_auth = Auth.AppAuth( diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index e6c71916928..5995fba61dd 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -1,4 +1,5 @@ """OAuth service models.""" + from functools import cached_property import structlog @@ -19,6 +20,38 @@ log = structlog.get_logger(__name__) +class GitHubAppInstallationManager(models.Manager): + def get_or_create_installation( + self, *, installation_id, target_id, target_type, extra_data=None + ): + installation, created = self.get_or_create( + installation_id=installation_id, + defaults={ + "target_id": target_id, + "target_type": target_type, + "extra_data": extra_data or {}, + }, + ) + # NOTE: An installation can't change its target_id or target_type. + # This should never happen, unless this assumption is wrong. + if ( + installation.target_id != target_id + or installation.target_type != target_type + ): + log.exception( + "Installation target_id or target_type changed", + installation_id=installation.installation_id, + target_id=installation.target_id, + target_type=installation.target_type, + new_target_id=target_id, + new_target_type=target_type, + ) + installation.target_id = target_id + installation.target_type = target_type + installation.save() + return installation, created + + class GitHubAccountType(models.TextChoices): USER = "User", _("User") ORGANIZATION = "Organization", _("Organization") diff --git a/readthedocs/oauth/services/__init__.py b/readthedocs/oauth/services/__init__.py index bce163683f4..104034000cb 100644 --- a/readthedocs/oauth/services/__init__.py +++ b/readthedocs/oauth/services/__init__.py @@ -1,7 +1,7 @@ """Conditional classes for OAuth services.""" from readthedocs.core.utils.extend import SettingsOverrideObject -from readthedocs.oauth.services import bitbucket, github, gitlab +from readthedocs.oauth.services import bitbucket, github, githubapp, gitlab class GitHubService(SettingsOverrideObject): @@ -19,4 +19,9 @@ class GitLabService(SettingsOverrideObject): _override_setting = "OAUTH_GITLAB_SERVICE" -registry = [GitHubService, BitbucketService, GitLabService] +class GitHubAppService(SettingsOverrideObject): + _default_class = githubapp.GitHubAppService + _override_setting = "OAUTH_GITHUB_APP_SERVICE" + + +registry = [GitHubService, BitbucketService, GitLabService, GitHubAppService] diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index b4f63b06999..5b4721965bf 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -9,7 +9,6 @@ from github.Repository import Repository as GHRepository from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider -from readthedocs.allauth.providers.githubapp.views import GitHubAppOAuth2Adapter from readthedocs.builds.constants import BUILD_STATUS_SUCCESS, SELECT_BUILD_STATUS from readthedocs.oauth.clients import get_gh_app_client, get_oauth2_client from readthedocs.oauth.constants import GITHUB @@ -29,7 +28,6 @@ class GitHubAppService(Service): vcs_provider_slug = GITHUB provider_name = "GitHub" - adapter = GitHubAppOAuth2Adapter def __init__(self, installation: GitHubAppInstallation): self.installation = installation @@ -69,15 +67,17 @@ def for_user(cls, user): ) for account in social_accounts: oauth2_client = get_oauth2_client(account) - resp = oauth2_client.get("https://api.github.com/app/installations") + resp = oauth2_client.get("https://api.github.com/user/installations") if resp.status_code != 200: continue for gh_installation in resp.json()["installations"]: - # TODO: get or create the installation object. - installation = GitHubAppInstallation.objects.filter( + installation = GitHubAppInstallation.objects.get_or_create_installation( installation_id=gh_installation["id"], + target_id=gh_installation["target_id"], + target_type=gh_installation["target_type"], + extra_data={"installation": gh_installation}, ).first() if installation: yield cls(installation) @@ -146,8 +146,8 @@ def _create_or_update_repository_from_gh( # What about a project that is public, and then becomes private? # I think we should allow creating remote repositories for these, # but block import/clone and other operations. - if not settings.ALLOW_PRIVATE_REPOS and gh_repo.private: - return + # if not settings.ALLOW_PRIVATE_REPOS and gh_repo.private: + # return target_id = self.installation.target_id target_type = self.installation.target_type diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 5fe0bab3948..17af6bdfb3e 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -223,7 +223,7 @@ def _handle_installation_repositories_event(self): if action == "added": if data["repository_selection"] == "all": - installation.service.sync_repositories() + installation.service.sync() else: installation.service.update_or_create_repositories( [repo["id"] for repo in data["repositories_added"]] @@ -381,7 +381,7 @@ def _handle_organization_event(self): # this is since we don't know to which repositories the members have access. # GH doesn't send a member event for this. if action in ("member_added", "member_removed"): - installation.service.sync_repositories() + installation.service.sync() return # Hmm, installation_target should handle this instead? @@ -481,33 +481,17 @@ def _get_or_create_installation(self, sync_repositories_on_create: bool = True): data = data.copy() data["installation"] = gh_installation.raw_data - installation, created = GitHubAppInstallation.objects.get_or_create( + ( + installation, + created, + ) = GitHubAppInstallation.objects.get_or_create_installation( installation_id=installation_id, - defaults={ - "extra_data": data, - "target_id": target_id, - "target_type": target_type, - }, + target_id=target_id, + target_type=target_type, + extra_data=data, ) - # NOTE: An installation can't change its target_id or target_type. - # This should never happen, unless this assumption is wrong. - if ( - installation.target_id != target_id - or installation.target_type != target_type - ): - log.exception( - "Installation target_id or target_type changed", - installation_id=installation.installation_id, - target_id=installation.target_id, - target_type=installation.target_type, - new_target_id=target_id, - new_target_type=target_type, - ) - installation.target_id = target_id - installation.target_type = target_type - installation.save() if created and sync_repositories_on_create: - installation.service.sync_repositories() + installation.service.sync() return installation, created def _is_payload_signature_valid(self): From 54e7ec6dce19a3f7cd0a94cdc0e0004118217098 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Feb 2025 12:09:14 -0500 Subject: [PATCH 27/92] More refactor --- readthedocs/api/v2/views/model_views.py | 4 ++-- readthedocs/builds/tasks.py | 2 +- readthedocs/oauth/models.py | 2 ++ readthedocs/oauth/services/base.py | 7 ++++--- readthedocs/oauth/services/bitbucket.py | 9 ++++----- readthedocs/oauth/services/github.py | 7 +++---- readthedocs/oauth/services/githubapp.py | 12 +++++++----- readthedocs/oauth/services/gitlab.py | 6 +++--- readthedocs/oauth/tasks.py | 6 +++--- readthedocs/projects/models.py | 2 +- readthedocs/projects/views/private.py | 2 +- 11 files changed, 31 insertions(+), 28 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index b730a42bac8..7317a630999 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -420,7 +420,7 @@ def get_queryset(self): self.model.objects.api_v2(self.request.user) .filter( remote_organization_relations__account__provider__in=[ - service.adapter.provider_id for service in registry + service.allauth_provider.id for service in registry ] ) .distinct() @@ -466,7 +466,7 @@ def get_queryset(self): query = query.filter( remote_repository_relations__account__provider__in=[ - service.adapter.provider_id for service in registry + service.allauth_provider.id for service in registry ], ).distinct() diff --git a/readthedocs/builds/tasks.py b/readthedocs/builds/tasks.py index 7e13ca62d19..a9a7092703c 100644 --- a/readthedocs/builds/tasks.py +++ b/readthedocs/builds/tasks.py @@ -424,7 +424,7 @@ def send_build_status(build_pk, commit, status): message_id=MESSAGE_OAUTH_BUILD_STATUS_FAILURE, attached_to=build.project, format_values={ - "provider_name": service_class.provider_name, + "provider_name": service_class.allauth_provider.name, "url_connect_account": reverse("socialaccount_connections"), }, dismissable=True, diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 5995fba61dd..3e4c855ba05 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -80,6 +80,8 @@ class GitHubAppInstallation(TimeStampedModel): default=dict, ) + objects = GitHubAppInstallationManager() + class Meta(TimeStampedModel.Meta): pass diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index c84ede01c03..f6482ecef7f 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -5,6 +5,7 @@ import structlog from allauth.socialaccount.models import SocialAccount +from allauth.socialaccount.providers.oauth2.provider import OAuth2Provider from allauth.socialaccount.providers.oauth2.views import OAuth2Adapter from django.conf import settings from django.urls import reverse @@ -31,7 +32,7 @@ class Service: """Base class for service that interacts with a VCS provider and a project.""" vcs_provider_slug: str - provider_name: str + allauth_provider = type[OAuth2Provider] url_pattern: re.Pattern | None = None default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL @@ -196,7 +197,7 @@ def paginate(self, url, **kwargs): # needs to reconnect his account raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( - provider=self.provider_name + provider=self.allauth_provider.name ) ) @@ -210,7 +211,7 @@ def paginate(self, url, **kwargs): log.warning("access_token or refresh_token failed.", url=url) raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( - provider=self.provider_name + provider=self.allauth_provider.name ) ) # Catch exceptions with request or deserializing JSON diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index e6d8a7255f8..c4609e93096 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -4,8 +4,8 @@ import re import structlog -from allauth.socialaccount.providers.bitbucket_oauth2.views import ( - BitbucketOAuth2Adapter, +from allauth.socialaccount.providers.bitbucket_oauth2.provider import ( + BitbucketOAuth2Provider, ) from django.conf import settings from requests.exceptions import RequestException @@ -24,12 +24,11 @@ class BitbucketService(UserService): """Provider service for Bitbucket.""" - adapter = BitbucketOAuth2Adapter + allauth_provider = BitbucketOAuth2Provider + vcs_provider_slug = BITBUCKET # TODO replace this with a less naive check url_pattern = re.compile(r"bitbucket.org") https_url_pattern = re.compile(r"^https:\/\/[^@]+@bitbucket.org/") - vcs_provider_slug = BITBUCKET - provider_name = "Bitbucket" def sync_repositories(self): """Sync repositories from Bitbucket API.""" diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 7248da1b6c4..c4dd5edfe45 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -4,7 +4,7 @@ import re import structlog -from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter +from allauth.socialaccount.providers.github.provider import GitHubProvider from django.conf import settings from oauthlib.oauth2.rfc6749.errors import InvalidGrantError, TokenExpiredError from requests.exceptions import RequestException @@ -24,11 +24,10 @@ class GitHubService(UserService): """Provider service for GitHub.""" - adapter = GitHubOAuth2Adapter + vcs_provider_slug = GITHUB + allauth_provider = GitHubProvider # TODO replace this with a less naive check url_pattern = re.compile(r"github\.com") - vcs_provider_slug = GITHUB - provider_name = "GitHub" def sync_repositories(self): """Sync repositories from GitHub API.""" diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 5b4721965bf..c921aadbe06 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -27,7 +27,7 @@ class GitHubAppService(Service): vcs_provider_slug = GITHUB - provider_name = "GitHub" + allauth_provider = GitHubAppProvider def __init__(self, installation: GitHubAppInstallation): self.installation = installation @@ -63,7 +63,7 @@ def for_user(cls, user): """ social_accounts = SocialAccount.objects.filter( user=user, - provider=GitHubAppProvider.id, + provider=cls.allauth_provider.id, ) for account in social_accounts: oauth2_client = get_oauth2_client(account) @@ -171,7 +171,9 @@ def _create_or_update_repository_from_gh( remote_repo.name = gh_repo.name remote_repo.full_name = gh_repo.full_name remote_repo.description = gh_repo.description - remote_repo.avatar_url = gh_repo.owner.avatar_url + remote_repo.avatar_url = ( + gh_repo.owner.avatar_url or self.default_user_avatar_url + ) remote_repo.ssh_url = gh_repo.ssh_url remote_repo.html_url = gh_repo.html_url remote_repo.private = gh_repo.private @@ -237,7 +239,7 @@ def _create_or_update_organization_from_gh( remote_org.name = gh_org.name # NOTE: do we need the email of the organization? remote_org.email = gh_org.email - remote_org.avatar_url = gh_org.avatar_url + remote_org.avatar_url = gh_org.avatar_url or self.default_org_avatar_url remote_org.url = gh_org.html_url remote_org.save() self._resync_organization_members(gh_org, remote_org) @@ -278,7 +280,7 @@ def _resync_collaborators( def _get_social_accounts(self, ids): return SocialAccount.objects.filter( uid__in=ids, - provider=GitHubAppProvider.id, + provider=self.allauth_provider.id, ).select_related("user") def update_or_create_organization(self, org_id: int) -> RemoteOrganization: diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index df5b0cb9e36..7dc9c323ad7 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -5,6 +5,7 @@ from urllib.parse import quote_plus, urlparse import structlog +from allauth.socialaccount.providers.gitlab.provider import GitLabProvider from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter from django.conf import settings from oauthlib.oauth2.rfc6749.errors import InvalidGrantError, TokenExpiredError @@ -31,12 +32,11 @@ class GitLabService(UserService): - https://docs.gitlab.com/ce/api/oauth2.html """ - provider_name = "GitLab" - adapter = GitLabOAuth2Adapter + allauth_provider = GitLabProvider # Just use the network location to determine if it's a GitLab project # because private repos have another base url, eg. git@gitlab.example.com url_pattern = re.compile( - re.escape(urlparse(adapter.provider_default_url).netloc), + re.escape(urlparse(GitLabOAuth2Adapter.provider_default_url).netloc), ) PERMISSION_NO_ACCESS = 0 diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index 13a6185c1f1..88b8409029a 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -49,7 +49,7 @@ def sync_remote_repositories(user_id): try: service.sync() except SyncServiceError: - failed_services.add(service.provider_name) + failed_services.add(service.allauth_provider.name) if failed_services: raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( @@ -200,7 +200,7 @@ def attach_webhook(project_pk, user_pk=None, integration=None, **kwargs): dismissable=True, attached_to=project, format_values={ - "provider_name": service_class.provider_name, + "provider_name": service_class.allauth_provider.name, "url_connect_account": reverse( "projects_integrations", args=[project.slug], @@ -221,7 +221,7 @@ def attach_webhook(project_pk, user_pk=None, integration=None, **kwargs): dismissable=True, attached_to=project, format_values={ - "provider_name": service_class.provider_name, + "provider_name": service_class.allauth_provider.name, "url_docs_webhook": "https://docs.readthedocs.io/page/webhooks.html", }, ) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 07e66b47e85..5275c19f6f2 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1055,7 +1055,7 @@ def get_git_service_class(self, fallback_to_clone_url=False): def git_provider_name(self): """Get the provider name for project. e.g: GitHub, GitLab, Bitbucket.""" service_class = self.get_git_service_class(fallback_to_clone_url=True) - return service_class.provider_name if service_class else None + return service_class.allauth_provider.name if service_class else None def find(self, filename, version): """ diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index aeacb38c81c..73c918049da 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -489,7 +489,7 @@ def get(self, request, *args, **kwargs): deprecated_accounts = SocialAccount.objects.filter( user=self.request.user ).exclude( - provider__in=[service.adapter.provider_id for service in registry], + provider__in=[service.allauth_provider_id for service in registry], ) # yapf: disable for account in deprecated_accounts: provider_account = account.get_provider_account() From 4d5a2290fa94deb3247787b9ee38bc7f22d52586 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Feb 2025 12:50:44 -0500 Subject: [PATCH 28/92] More fixes and updates --- readthedocs/builds/tasks.py | 6 +++--- readthedocs/oauth/views.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/readthedocs/builds/tasks.py b/readthedocs/builds/tasks.py index a9a7092703c..898cfbe9354 100644 --- a/readthedocs/builds/tasks.py +++ b/readthedocs/builds/tasks.py @@ -412,9 +412,9 @@ def send_build_status(build_pk, commit, status): for service in service_class.for_project(build.project): success = service.send_build_status( - build, - commit, - status, + build=build, + commit=commit, + status=status, ) if success: log.debug("Build status report sent correctly.") diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 17af6bdfb3e..1a3ee917800 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -43,7 +43,6 @@ def post(self, request): event_handlers = { "installation": self._handle_installation_event, "installation_repositories": self._handle_installation_repositories_event, - # Hmm, don't think we need this one. "installation_target": self._handle_installation_target_event, "push": self._handle_push_event, "pull_request": self._handle_pull_request_event, @@ -245,7 +244,18 @@ def _handle_installation_target_event(self): Triggered when the target of an installation changes, like when the user or organization changes its username/slug. + + Looks like this is only triggered when a username is changed, + when an organization is renamed, it doesn't trigger this event + (maybe a bug?). """ + installation, created = self._get_or_create_installation() + + # If we didn't have the installation, all repositories were synced on creation. + if created: + return + + installation.service.sync() def _handle_repository_event(self): """ @@ -385,6 +395,7 @@ def _handle_organization_event(self): return # Hmm, installation_target should handle this instead? + # But I wasn't able to trigger neitehr of those events when renaming an organization. if action == "renamed": # Update organization and its members only. # We don't need to sync the repositories. From 12aab7957b09476ca9440d90d07ef93b38be5e0e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Feb 2025 14:32:39 -0500 Subject: [PATCH 29/92] Docstrings and updates --- .../allauth/providers/githubapp/provider.py | 6 ++ .../allauth/providers/githubapp/urls.py | 2 + .../allauth/providers/githubapp/views.py | 2 + readthedocs/core/adapters.py | 16 +++- readthedocs/oauth/admin.py | 9 +- readthedocs/oauth/clients.py | 2 +- readthedocs/oauth/models.py | 17 ++-- readthedocs/oauth/services/__init__.py | 2 + readthedocs/oauth/services/githubapp.py | 65 ++++++++++--- readthedocs/oauth/views.py | 92 +++++++++++++------ readthedocs/projects/models.py | 7 ++ 11 files changed, 166 insertions(+), 54 deletions(-) diff --git a/readthedocs/allauth/providers/githubapp/provider.py b/readthedocs/allauth/providers/githubapp/provider.py index f785635ceb7..c9360018347 100644 --- a/readthedocs/allauth/providers/githubapp/provider.py +++ b/readthedocs/allauth/providers/githubapp/provider.py @@ -4,6 +4,12 @@ class GitHubAppProvider(GitHubProvider): + """ + Provider for GitHub App. + + We subclass the GitHubProvider to so we have two separate providers for GitHub and GitHub App. + """ + id = "githubapp" name = "GitHub App" oauth2_adapter_class = GitHubAppOAuth2Adapter diff --git a/readthedocs/allauth/providers/githubapp/urls.py b/readthedocs/allauth/providers/githubapp/urls.py index 49d8a5f9514..525e82eacc7 100644 --- a/readthedocs/allauth/providers/githubapp/urls.py +++ b/readthedocs/allauth/providers/githubapp/urls.py @@ -1,3 +1,5 @@ +"""Copied from allauth.socialaccount.providers.github.urls.""" + from allauth.socialaccount.providers.oauth2.urls import default_urlpatterns from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider diff --git a/readthedocs/allauth/providers/githubapp/views.py b/readthedocs/allauth/providers/githubapp/views.py index b0165884cae..d40eede0ef4 100644 --- a/readthedocs/allauth/providers/githubapp/views.py +++ b/readthedocs/allauth/providers/githubapp/views.py @@ -1,3 +1,5 @@ +"""Copied from allauth.socialaccount.providers.github.views.""" + from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter from allauth.socialaccount.providers.oauth2.views import ( OAuth2CallbackView, diff --git a/readthedocs/core/adapters.py b/readthedocs/core/adapters.py index 0e2b6d27b20..ed037504903 100644 --- a/readthedocs/core/adapters.py +++ b/readthedocs/core/adapters.py @@ -58,12 +58,18 @@ def save_user(self, request, user, form, commit=True): class SocialAccountAdapter(DefaultSocialAccountAdapter): def pre_social_login(self, request, sociallogin): """ - Remove all email addresses except the primary one. + Additional logic to apply before social login. - We don't want to populate all email addresses from the social account, - it also makes it easy to mark only the primary email address as verified - for providers that don't return information about email verification - even if the email is verified (like GitLab). + - Remove all email addresses except the primary one. + + We don't want to populate all email addresses from the social account, + it also makes it easy to mark only the primary email address as verified + for providers that don't return information about email verification + even if the email is verified (like GitLab). + + - Connect a GitHub App (new integration) account to an existing GitHub account (old integration) + if it belongs to the same user. This avoids creating a new account when the user + signs up with the new integration. """ sociallogin.email_addresses = [ email for email in sociallogin.email_addresses if email.primary diff --git a/readthedocs/oauth/admin.py b/readthedocs/oauth/admin.py index 7c3da78a4d7..80865f598ed 100644 --- a/readthedocs/oauth/admin.py +++ b/readthedocs/oauth/admin.py @@ -10,7 +10,14 @@ RemoteRepositoryRelation, ) -admin.site.register(GitHubAppInstallation) + +@admin.register(GitHubAppInstallation) +class GitHubAppInstallationAdmin(admin.ModelAdmin): + list_display = ( + "installation_id", + "target_type", + "target_id", + ) @admin.register(RemoteRepository) diff --git a/readthedocs/oauth/clients.py b/readthedocs/oauth/clients.py index d3ad8164456..b7e38c11c86 100644 --- a/readthedocs/oauth/clients.py +++ b/readthedocs/oauth/clients.py @@ -74,7 +74,7 @@ def get_oauth2_client(account): def get_gh_app_client() -> GithubIntegration: - """Return a client authenticated as the GitHub App to interact with the API""" + """Return a client authenticated as the GitHub App to interact with the API.""" app_auth = Auth.AppAuth( app_id=settings.GITHUB_APP_CLIENT_ID, private_key=settings.GITHUB_APP_PRIVATE_KEY, diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 3e4c855ba05..860e6120793 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -24,6 +24,12 @@ class GitHubAppInstallationManager(models.Manager): def get_or_create_installation( self, *, installation_id, target_id, target_type, extra_data=None ): + """ + Get or create a GitHub app installation. + + Only the installation_id is unique, the target_id and target_type could change, + but this should never happen. + """ installation, created = self.get_or_create( installation_id=installation_id, defaults={ @@ -83,7 +89,7 @@ class GitHubAppInstallation(TimeStampedModel): objects = GitHubAppInstallationManager() class Meta(TimeStampedModel.Meta): - pass + verbose_name = _("GitHub app installation") @cached_property def service(self): @@ -256,11 +262,10 @@ class RemoteRepository(TimeStampedModel): related_name="repositories", null=True, blank=True, - # Delete the repository if the installation is deleted? - # or keep the repository and just remove the installation? - # I think we should keep the repository, but only if it's linked to a project, - # since a user could re-install the app, they shouldn't need to - # manually link each project to the repository again. + # When an installation is deleted, we don't delete the repository + # if it's linked to a project. This is in case the user re-installs the app, + # they shouldn't need to manually link each project to the repository again. + # NOTE: I also see how this may be unexpected behavior in some cases. on_delete=models.SET_NULL, ) diff --git a/readthedocs/oauth/services/__init__.py b/readthedocs/oauth/services/__init__.py index 104034000cb..43cc2d92733 100644 --- a/readthedocs/oauth/services/__init__.py +++ b/readthedocs/oauth/services/__init__.py @@ -24,4 +24,6 @@ class GitHubAppService(SettingsOverrideObject): _override_setting = "OAUTH_GITHUB_APP_SERVICE" +# NOTE: GitHubAppService should be listed after GitHubService, +# since they share the same vcs_provider_slug. registry = [GitHubService, BitbucketService, GitLabService, GitHubAppService] diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index c921aadbe06..31d67454b38 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -35,6 +35,11 @@ def __init__(self, installation: GitHubAppInstallation): @cached_property def app_installation(self) -> GHInstallation: + """ + Return the installation object from the GitHub API. + + Usefull to interact with installation related endpoints. + """ return self.gha_client.get_app_installation( self.installation.installation_id, ) @@ -48,6 +53,13 @@ def installation_client(self) -> Github: @classmethod def for_project(cls, project): + """ + Return a GitHubAppService for the installation linked to the project. + + Since this service only works for projects that have a remote repository, + and are linked to a GitHub App installation, + this returns only one service or None. + """ if ( not project.remote_repository or not project.remote_repository.github_app_installation @@ -59,7 +71,21 @@ def for_project(cls, project): @classmethod def for_user(cls, user): """ - https://docs.github.com/en/rest/apps/installations?apiVersion=2022-11-28#list-app-installations-accessible-to-the-user-access-token instead. + Return a GitHubAppService for each installation accessible to the user. + + In order to get the installations accessible to the user, we need to use + the GitHub API authenticated as the user, making use of the user's access token + (not the installation token). + + See: + + - https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-with-a-github-app-on-behalf-of-a-user + - https://docs.github.com/en/rest/apps/installations?apiVersion=2022-11-28#list-app-installations-accessible-to-the-user-access-token + + .. note:: + + If the installation wasn't in our database, we create it + (but we don't sync the repositories, since the caller should be responsible for that). """ social_accounts = SocialAccount.objects.filter( user=user, @@ -83,6 +109,15 @@ def for_user(cls, user): yield cls(installation) def sync(self): + """ + Sync all repositories and organizations that are accessible to the installation. + + Repositories that are no longer accessible to the installation are removed from the database + only if they are not linked to a project. This is in case the user wants to grant access to the repository again. + + If a remote organization doesn't have any repositories after removing the repositories, + we remove the organization from the database. + """ remote_repositories = [] for repo in self.app_installation.get_repos(): remote_repo = self._create_or_update_repository_from_gh(repo) @@ -100,6 +135,7 @@ def sync(self): ).delete() def update_or_create_repositories(self, repository_ids: list[int]): + """Update or create repositories from the given list of repository IDs.""" for repository_id in repository_ids: repo = self.installation_client.get_repo(repository_id) self._create_or_update_repository_from_gh(repo) @@ -111,6 +147,8 @@ def delete_repositories(self, repository_ids: list[int]): We don't remove repositories that are linked to a project, since a user could grant access to the repository again, and we don't want users having to manually link the project to the repository again. + + We also remove organizations that don't have any repositories after removing the repositories. """ # Extract all the organizations linked to these repositories, # so we can remove organizations that don't have any repositories @@ -131,9 +169,7 @@ def delete_repositories(self, repository_ids: list[int]): remote_organizations.filter(repositories=None).delete() def delete_organization(self, organization_id: int): - """ - Delete an organization and all its repositories from the database only if they are not linked to a project. - """ + """Delete an organization and all its repositories from the database only if they are not linked to a project.""" RemoteOrganization.objects.filter( remote_id=str(organization_id), vcs_provider=self.vcs_provider_slug, @@ -143,12 +179,12 @@ def delete_organization(self, organization_id: int): def _create_or_update_repository_from_gh( self, gh_repo: GHRepository ) -> RemoteRepository | None: - # What about a project that is public, and then becomes private? - # I think we should allow creating remote repositories for these, - # but block import/clone and other operations. - # if not settings.ALLOW_PRIVATE_REPOS and gh_repo.private: - # return + """ + Create or update a remote repository from a GitHub repository object. + We also sync the collaborators of the repository with the database, + and create or update the organization of the repository. + """ target_id = self.installation.target_id target_type = self.installation.target_type # NOTE: All the repositories should be owned by the installation account. @@ -214,6 +250,7 @@ def _create_or_update_repository_from_gh( # NOTE: normally, this should cache only one organization at a time, but just in case... @lru_cache(maxsize=50) def _get_gh_organization(self, org_id: int) -> GHOrganization: + """Get a GitHub organization object given its numeric ID.""" # NOTE: cast to str, since PyGithub expects a string, # even if the API accepts a string or an int. # TODO: send a PR upstream to fix this. @@ -335,13 +372,17 @@ def send_build_status(self, *, build, commit, status): def get_clone_token(self, project): """ - See https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation. + Return a token for HTTP Git clone access to the repository. + + See: - https://docs.github.com/en/rest/apps/apps?apiVersion=2022-11-28#create-an-installation-access-token-for-an-app + - https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation + - https://docs.github.com/en/rest/apps/apps?apiVersion=2022-11-28#create-an-installation-access-token-for-an-app """ # TODO: we can pass the repository_ids to get a token with access to specific repositories. # We should upstream this feature to PyGithub. - # We can also pass a specific permissions object to get a token with specific permissions. + # We can also pass a specific permissions object to get a token with specific permissions + # if we want to scope this token even more. access_token = self.gha_client.get_access_token( self.installation.installation_id ) diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 1a3ee917800..9c46ee5c08a 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -38,8 +38,15 @@ def post(self, request): if not self._is_payload_signature_valid(): raise ValidationError("Invalid webhook signature") - event = self.request.headers.get(GITHUB_EVENT_HEADER) + # Most of the events have an installation object and action. + installation_id = request.data.get("installation", {}).get("id", "unknown") + action = request.data.get("action", "unknown") + log.bind( + installation_id=installation_id, + action=action, + ) + event = self.request.headers.get(GITHUB_EVENT_HEADER) event_handlers = { "installation": self._handle_installation_event, "installation_repositories": self._handle_installation_repositories_event, @@ -61,6 +68,9 @@ def _handle_installation_event(self): Handle the installation event. Triggered when a user installs or uninstalls the GitHub App under an account (user or organization). + We create the installation object and sync the repositories, or delete the installation accordingly. + + Payload example: .. code-block:: json @@ -128,25 +138,25 @@ def _handle_installation_event(self): data = self.request.data action = data["action"] gh_installation = data["installation"] + installation_id = gh_installation["id"] if action == "created": - installation, created = self._get_or_create_installation() + _, created = self._get_or_create_installation() if not created: - log.info( - "Installation already exists", installation_id=gh_installation["id"] - ) + log.info("Installation already exists") return if action == "deleted": # NOTE: does the app trigger a installation_repositories event? - installation = GitHubAppInstallation.objects.filter( - installation_id=gh_installation["id"] - ).first() - if not installation: - # If we never created the installation, we can ignore the event. - # Maybe don't raise an error? - raise ValidationError(f"Installation {gh_installation['id']} not found") - installation.delete() + # TODO: what to do with the repositories? + # Maybe delete the ones that are on linked to any projects? + count, _ = GitHubAppInstallation.objects.filter( + installation_id=installation_id + ).delete() + if count > 0: + log.info("Installation deleted") + else: + log.info("Installation not found") return # Ignore other actions: @@ -160,6 +170,14 @@ def _handle_installation_repositories_event(self): Triggered when a repository is added or removed from an installation. + If the installation had access to a repository, and the repository is deleted, + this event will be triggered. + + When a repository is deleted, we delete its remote repository object, + but only if it's not linked to any project. + + Payload example: + .. code-block:: json { "action": "added", @@ -244,10 +262,14 @@ def _handle_installation_target_event(self): Triggered when the target of an installation changes, like when the user or organization changes its username/slug. - - Looks like this is only triggered when a username is changed, - when an organization is renamed, it doesn't trigger this event - (maybe a bug?). + + .. note:: + + Looks like this is only triggered when a username is changed, + when an organization is renamed, it doesn't trigger this event + (maybe a bug?). + + When this happens, we re-sync all the repositories, so they use the new name. """ installation, created = self._get_or_create_installation() @@ -291,9 +313,11 @@ def _handle_push_event(self): """ Handle the push event. - Triggered when a commit is pushed, when a commit tag is pushed, - when a branch is deleted, when a tag is deleted, - or when a repository is created from a template. + Triggered when a commit is pushed (including a new branch or tag is created), + when a branch or tag is deleted, or when a repository is created from a template. + + If a new branch or tag is created, we trigger a sync of the versions, + if the version already exists, we build it if it's active. See https://docs.github.com/en/webhooks/webhook-events-and-payloads#push. """ @@ -317,7 +341,7 @@ def _parse_version_from_ref(self, ref: str): The ref can be a branch or a tag. - :param ref: The ref to parse. + :param ref: The ref to parse (e.g. refs/heads/main, refs/tags/v1.0.0). :returns: A tuple with the version name and type. """ heads_prefix = "refs/heads/" @@ -374,7 +398,8 @@ def _handle_organization_event(self): """ Handle the organization event. - Triggered when an organization is added or removed from a repository. + Triggered when an member is added or removed from an organization, + or when the organization is renamed or deleted. See https://docs.github.com/en/webhooks/webhook-events-and-payloads#organization """ @@ -394,14 +419,12 @@ def _handle_organization_event(self): installation.service.sync() return - # Hmm, installation_target should handle this instead? - # But I wasn't able to trigger neitehr of those events when renaming an organization. + # NOTE: installation_target should handle this instead? + # But I wasn't able to trigger neither of those events when renaming an organization. + # Maybe a bug? + # If the organization is renamed, we need to sync the repositories, so they use the new name. if action == "renamed": - # Update organization and its members only. - # We don't need to sync the repositories. - installation.service.update_or_create_organization( - data["organization"]["id"] - ) + installation.service.sync() return if action == "deleted": @@ -452,6 +475,13 @@ def _handle_github_app_authorization_event(self): """ def _get_projects(self): + """ + Get all projects linked to the repository that triggered the event. + + .. note:: + + This should only be used for events that have a repository object. + """ remote_repository = self._get_remote_repository() if not remote_repository: return Project.objects.none() @@ -462,6 +492,10 @@ def _get_remote_repository(self): Get the remote repository from the request data. If the repository doesn't exist, return None. + + .. note:: + + This should only be used for events that have a repository object. """ data = self.request.data remote_id = data["repository"]["id"] diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 5275c19f6f2..ef2598da054 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1446,6 +1446,13 @@ def organization(self): @property def clone_token(self): + """ + Return a token for HTTP Git clone access to the repository. + + .. note:: + + Only repositories granted acces by a GitHub app installation will return a token. + """ service_class = self.get_git_service_class() if not service_class: return None From c2ffe7df20ae70740355fea952b0321cf8ba95c1 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Feb 2025 14:39:19 -0500 Subject: [PATCH 30/92] Note on GH user access tokens --- readthedocs/oauth/services/githubapp.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 31d67454b38..c6b75895c5e 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -80,12 +80,19 @@ def for_user(cls, user): See: - https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-with-a-github-app-on-behalf-of-a-user + - https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-a-user-access-token-for-a-github-app - https://docs.github.com/en/rest/apps/installations?apiVersion=2022-11-28#list-app-installations-accessible-to-the-user-access-token .. note:: If the installation wasn't in our database, we create it (but we don't sync the repositories, since the caller should be responsible for that). + + .. note:: + + User access tokens expire after 8 hours, but our OAuth2 client should handle refreshing the token. + But, the refresh token expires after 6 months, in order to refresh that token, + the user needs to sign in using GitHub again (just a normal sing-in, not a re-authorization or sign-up). """ social_accounts = SocialAccount.objects.filter( user=user, From 56b024f609e14fd9fe43408da97e2bdbc96a3fac Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Feb 2025 14:53:55 -0500 Subject: [PATCH 31/92] Remove code --- readthedocs/oauth/services/base.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index f6482ecef7f..87722cc551f 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -6,7 +6,6 @@ import structlog from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.providers.oauth2.provider import OAuth2Provider -from allauth.socialaccount.providers.oauth2.views import OAuth2Adapter from django.conf import settings from django.urls import reverse from django.utils.translation import gettext_lazy as _ @@ -128,14 +127,12 @@ class UserService(Service): :param account: :py:class:`SocialAccount` instance for user """ - adapter = None - def __init__(self, user, account): self.user = user self.account = account log.bind( user_username=self.user.username, - social_provider=self.provider_id, + social_provider=self.allauth_provider.id, social_account_id=self.account.pk, ) @@ -149,18 +146,11 @@ def for_project(cls, project): def for_user(cls, user): accounts = SocialAccount.objects.filter( user=user, - provider=cls.adapter.provider_id, + provider=cls.allauth_provider.id, ) for account in accounts: yield cls(user=user, account=account) - def get_adapter(self) -> type[OAuth2Adapter]: - return self.adapter - - @property - def provider_id(self): - return self.get_adapter().provider_id - @cached_property def session(self): return get_oauth2_client(self.account) From c09186b8c481bc734a8c6d4050795f76f6f7abb7 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Feb 2025 15:25:53 -0500 Subject: [PATCH 32/92] Error handling and logging --- readthedocs/oauth/models.py | 82 +++++++++++- readthedocs/oauth/services/githubapp.py | 165 ++++++++++++++---------- readthedocs/oauth/views.py | 34 +++-- 3 files changed, 199 insertions(+), 82 deletions(-) diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 860e6120793..1f447a8f4db 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -14,7 +14,7 @@ from readthedocs.projects.constants import REPO_CHOICES from readthedocs.projects.models import Project -from .constants import VCS_PROVIDER_CHOICES +from .constants import GITHUB, VCS_PROVIDER_CHOICES from .querysets import RemoteOrganizationQuerySet, RemoteRepositoryQuerySet log = structlog.get_logger(__name__) @@ -98,6 +98,86 @@ def service(self): return GitHubAppService(self) + def delete(self, *args, **kwargs): + """Override delete method to remove orphaned linked repositories.""" + self.delete_orphaned_repositories() + return super().delete(*args, **kwargs) + + def delete_orphaned_repositories(self, repository_ids: list[int] | None = None): + """ + Delete orphaned repositories linked to this installation. + + When an installation is deleted, we delete all its remote repositories + that are not linked to a project. This is in case the user re-installs the app, + they shouldn't need to manually link each project to the repository again. + + We also remove organizations that don't have any repositories after removing the repositories. + + :param repository_ids: List of repository ids (remote ID) to delete. + If None, all repositories will be considered for deletion. + """ + if repository_ids is not None and not repository_ids: + log.info("No repositories to delete") + return + + remote_organizations = RemoteOrganization.objects.filter( + repositories__github_app_installation=self, + vcs_provider=GITHUB, + ) + remote_repositories = self.repositories.filter( + projects=None, + vcs_provider=GITHUB, + ) + if repository_ids: + remote_organizations = remote_organizations.filter( + repositories__remote_id__in=repository_ids + ) + remote_repositories = remote_repositories.filter( + remote_id__in=repository_ids + ) + + # Fetch all IDs before deleting the repositories, so we can filter the organizations later. + remote_organizations_ids = list( + remote_organizations.values_list("id", flat=True) + ) + + count, deleted = remote_repositories.delete() + log.info( + "Deleted repositories without projects", + count=count, + deleted=deleted, + installation_id=self.installation_id, + ) + # TODO: we should probably remove the user relation as well, + # since we want to keep the remote repository linked to the project, + # but not to the user. + + count, deleted = RemoteOrganization.objects.filter( + id__in=remote_organizations_ids, + repositories=None, + ).delete() + log.info( + "Deleted orphaned organizations", + count=count, + deleted=deleted, + installation_id=self.installation_id, + ) + + def delete_orphaned_organization(self, organization_id: int): + """Delete an organization and all its repositories from the database only if they are not linked to a project.""" + count, deleted = RemoteOrganization.objects.filter( + remote_id=str(organization_id), + vcs_provider=GITHUB, + repositories__projects=None, + ).delete() + log.info( + "Deleted orphaned organization", + count=count, + deleted=deleted, + organization_id=organization_id, + installation_id=self.installation_id, + ) + class RemoteOrganization(TimeStampedModel): """ diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index c6b75895c5e..1ce7d36d379 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -3,7 +3,7 @@ import structlog from allauth.socialaccount.models import SocialAccount from django.conf import settings -from github import Github +from github import Github, GithubException from github.Installation import Installation as GHInstallation from github.Organization import Organization as GHOrganization from github.Repository import Repository as GHRepository @@ -20,7 +20,7 @@ RemoteRepository, RemoteRepositoryRelation, ) -from readthedocs.oauth.services.base import Service +from readthedocs.oauth.services.base import Service, SyncServiceError log = structlog.get_logger(__name__) @@ -39,6 +39,8 @@ def app_installation(self) -> GHInstallation: Return the installation object from the GitHub API. Usefull to interact with installation related endpoints. + + If the installation is no longer accessible, this will raise a GithubException. """ return self.gha_client.get_app_installation( self.installation.installation_id, @@ -101,8 +103,14 @@ def for_user(cls, user): for account in social_accounts: oauth2_client = get_oauth2_client(account) resp = oauth2_client.get("https://api.github.com/user/installations") - if resp.status_code != 200: + log.info( + "Failed to fetch installations from GitHub", + user=user, + account_id=account.uid, + status_code=resp.status_code, + response=resp.json(), + ) continue for gh_installation in resp.json()["installations"]: @@ -126,63 +134,44 @@ def sync(self): we remove the organization from the database. """ remote_repositories = [] - for repo in self.app_installation.get_repos(): - remote_repo = self._create_or_update_repository_from_gh(repo) - if remote_repo: - remote_repositories.append(remote_repo) - - # Remove repositories that are no longer in the list, - # and that are not linked to a project. - RemoteRepository.objects.filter( - github_app_installation=self.installation, - vcs_provider=self.vcs_provider_slug, - projects=None, - ).exclude( + try: + for repo in self.app_installation.get_repos(): + remote_repo = self._create_or_update_repository_from_gh(repo) + if remote_repo: + remote_repositories.append(remote_repo) + except GithubException: + # TODO: if we lost access to the installations, + # we should remove the installation from the database, + # and clean up the repositories, organizations, and relations. + log.info( + "Failed to sync repositories for installation", + installation_id=self.installation.installation_id, + exc_info=True, + ) + raise SyncServiceError() + + repos_to_delete = self.installation.repositories.exclude( pk__in=[repo.pk for repo in remote_repositories], - ).delete() + ).values_list("remote_id", flat=True) + self.installation.delete_orphaned_repositories(repos_to_delete) def update_or_create_repositories(self, repository_ids: list[int]): """Update or create repositories from the given list of repository IDs.""" for repository_id in repository_ids: - repo = self.installation_client.get_repo(repository_id) + try: + repo = self.installation_client.get_repo(repository_id) + except GithubException: + log.info( + "Failed to fetch repository from GitHub", + repository_id=repository_id, + exc_info=True, + ) + # TODO: if we lost access to the repository, + # we should remove the repository from the database, + # and clean up the collaborators and relations. + continue self._create_or_update_repository_from_gh(repo) - def delete_repositories(self, repository_ids: list[int]): - """ - Delete repositories from the given list that are not linked to a project. - - We don't remove repositories that are linked to a project, since a user could - grant access to the repository again, and we don't want users having to manually - link the project to the repository again. - - We also remove organizations that don't have any repositories after removing the repositories. - """ - # Extract all the organizations linked to these repositories, - # so we can remove organizations that don't have any repositories - # after removing the repositories. - remote_organizations = RemoteOrganization.objects.filter( - repositories__remote_id__in=repository_ids, - vcs_provider=self.vcs_provider_slug, - ) - - RemoteRepository.objects.filter( - github_app_installation=self.installation, - vcs_provider=self.vcs_provider_slug, - remote_id__in=repository_ids, - projects=None, - ).delete() - - # Delete organizations that don't have any repositories. - remote_organizations.filter(repositories=None).delete() - - def delete_organization(self, organization_id: int): - """Delete an organization and all its repositories from the database only if they are not linked to a project.""" - RemoteOrganization.objects.filter( - remote_id=str(organization_id), - vcs_provider=self.vcs_provider_slug, - repositories__projects=None, - ).delete() - def _create_or_update_repository_from_gh( self, gh_repo: GHRepository ) -> RemoteRepository | None: @@ -322,14 +311,26 @@ def _resync_collaborators( ).delete() def _get_social_accounts(self, ids): + """Get social accounts given a list of GitHub user IDs.""" return SocialAccount.objects.filter( uid__in=ids, provider=self.allauth_provider.id, ).select_related("user") - def update_or_create_organization(self, org_id: int) -> RemoteOrganization: - gh_org = self._get_gh_organization(org_id) - return self._create_or_update_organization_from_gh(gh_org) + def update_or_create_organization(self, org_id: int) -> RemoteOrganization | None: + try: + gh_org = self._get_gh_organization(org_id) + return self._create_or_update_organization_from_gh(gh_org) + except GithubException: + log.info( + "Failed to fetch organization from GitHub", + organization_id=org_id, + exc_info=True, + ) + # TODO: if we lost access to the organization, + # we should remove the organization from the database, + # and clean up the members and relations. + return None def _resync_organization_members( self, gh_org: GHOrganization, remote_org: RemoteOrganization @@ -357,6 +358,11 @@ def _resync_organization_members( ).delete() def send_build_status(self, *, build, commit, status): + """ + Create a commit status on GitHub for the given build. + + See https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#create-a-commit-status. + """ project = build.project remote_repo = project.remote_repository @@ -369,13 +375,29 @@ def send_build_status(self, *, build, commit, status): description = SELECT_BUILD_STATUS[status]["description"] context = f"{settings.RTD_BUILD_STATUS_API_NAME}:{project.slug}" - gh_repo = self.installation_client.get_repo(int(remote_repo.remote_id)) - gh_repo.get_commit(commit).create_status( - state=state, - target_url=target_url, - description=description, - context=context, - ) + try: + # NOTE: we use the lazy option to avoid fetching the repository object, + # since we only need the object to interact with the commit status API. + gh_repo = self.installation_client.get_repo( + int(remote_repo.remote_id), lazy=True + ) + gh_repo.get_commit(commit).create_status( + state=state, + target_url=target_url, + description=description, + context=context, + ) + return True + except GithubException: + log.info( + "Failed to send build status to GitHub", + project=project.slug, + build=build.pk, + commit=commit, + status=status, + exc_info=True, + ) + return False def get_clone_token(self, project): """ @@ -390,10 +412,19 @@ def get_clone_token(self, project): # We should upstream this feature to PyGithub. # We can also pass a specific permissions object to get a token with specific permissions # if we want to scope this token even more. - access_token = self.gha_client.get_access_token( - self.installation.installation_id - ) - return f"x-access-token:{access_token.token}" + try: + access_token = self.gha_client.get_access_token( + self.installation.installation_id + ) + return f"x-access-token:{access_token.token}" + except GithubException: + log.info( + "Failed to get clone token for project", + installation_id=self.installation.installation_id, + project=project.slug, + exc_info=True, + ) + return None def setup_webhook(self, project, integration=None): """When using a GitHub App, we don't need to set up a webhook.""" diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 9c46ee5c08a..2da6aa85723 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -36,16 +36,12 @@ def __init__(self, **kwargs): def post(self, request): if not self._is_payload_signature_valid(): + log.debug("Invalid webhook signature") raise ValidationError("Invalid webhook signature") # Most of the events have an installation object and action. installation_id = request.data.get("installation", {}).get("id", "unknown") action = request.data.get("action", "unknown") - log.bind( - installation_id=installation_id, - action=action, - ) - event = self.request.headers.get(GITHUB_EVENT_HEADER) event_handlers = { "installation": self._handle_installation_event, @@ -58,9 +54,17 @@ def post(self, request): "member": self._handle_member_event, "github_app_authorization": self._handle_github_app_authorization_event, } + log.bind( + installation_id=installation_id, + action=action, + event=event, + ) if event in event_handlers: + log.debug("Handling event") event_handlers[event]() return Response(status=200) + + log.debug("Unsupported event") raise ValidationError(f"Unsupported event: {event}") def _handle_installation_event(self): @@ -147,13 +151,13 @@ def _handle_installation_event(self): return if action == "deleted": - # NOTE: does the app trigger a installation_repositories event? - # TODO: what to do with the repositories? - # Maybe delete the ones that are on linked to any projects? - count, _ = GitHubAppInstallation.objects.filter( + # NOTE: When an installation is deleted, this doesn't trigger an installation_repositories event. + # So we need to call the delete method explicitly here, so we delete orphan remote repositories. + installation = GitHubAppInstallation.objects.filter( installation_id=installation_id - ).delete() - if count > 0: + ).first() + if installation: + installation.delete() log.info("Installation deleted") else: log.info("Installation not found") @@ -248,7 +252,7 @@ def _handle_installation_repositories_event(self): return if action == "removed": - installation.service.delete_repositories( + installation.delete_orphaned_repositories( [repo["id"] for repo in data["repositories_removed"]] ) return @@ -431,9 +435,9 @@ def _handle_organization_event(self): # Delete the organization only if it's not linked to any project. # GH sends a repository and installation_repositories events for each repository # when the organization is deleted. - # I didn't see that GH send the deleted action for the organization event... + # I didn't see GH send the deleted action for the organization event... # But handle it just in case. - installation.service.delete_organization(data["organization"]["id"]) + installation.delete_orphaned_organization(data["organization"]["id"]) return # Ignore other actions: @@ -473,6 +477,7 @@ def _handle_github_app_authorization_event(self): See https://docs.github.com/en/webhooks/webhook-events-and-payloads#github_app_authorization. """ + # TODO: what to do here? def _get_projects(self): """ @@ -520,6 +525,7 @@ def _get_or_create_installation(self, sync_repositories_on_create: bool = True): # If they aren't present, fetch them from the API, # so we can create the installation object if needed. if not target_id or not target_type: + log.debug("Incomplete installation object, fetching from the API") gh_installation = self.gha_client.get_app_installation(installation_id) target_id = gh_installation.target_id target_type = gh_installation.target_type From 0b2c686988fbe106bcbdb70357539aadd17948d2 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Feb 2025 16:00:29 -0500 Subject: [PATCH 33/92] Validate when trying to connect to an existing account --- readthedocs/core/adapters.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/readthedocs/core/adapters.py b/readthedocs/core/adapters.py index ed037504903..b9620e99e93 100644 --- a/readthedocs/core/adapters.py +++ b/readthedocs/core/adapters.py @@ -1,10 +1,16 @@ """Allauth overrides.""" + import structlog from allauth.account.adapter import DefaultAccountAdapter +from allauth.account.adapter import get_adapter as get_account_adapter +from allauth.exceptions import ImmediateHttpResponse from allauth.socialaccount.adapter import DefaultSocialAccountAdapter from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.providers.github.provider import GitHubProvider +from django.contrib import messages +from django.http import HttpResponseRedirect +from django.urls import reverse from django.utils.encoding import force_str from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider @@ -82,4 +88,22 @@ def pre_social_login(self, request, sociallogin): uid=sociallogin.account.uid, ).first() if social_account: + # If the user is logged in, and the GH OAuth account belongs to + # a different user, we should not connect the accounts, + # this is the same as trying to connect an existing GH account to another user. + if ( + request.user.is_authenticated + and request.user != social_account.user + ): + message_template = ( + "socialaccount/messages/account_connected_other.txt" + ) + get_account_adapter(request).add_message( + request=request, + level=messages.ERROR, + message_template=message_template, + ) + url = reverse("socialaccount_connections") + raise ImmediateHttpResponse(HttpResponseRedirect(url)) + sociallogin.connect(request, social_account.user) From a7f1cb7bd661c78c3b2d6fa60c44c6870cd7d6fb Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Feb 2025 16:11:08 -0500 Subject: [PATCH 34/92] Less indentation --- readthedocs/core/adapters.py | 43 +++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/readthedocs/core/adapters.py b/readthedocs/core/adapters.py index b9620e99e93..3397986a7af 100644 --- a/readthedocs/core/adapters.py +++ b/readthedocs/core/adapters.py @@ -87,23 +87,26 @@ def pre_social_login(self, request, sociallogin): provider=GitHubProvider.id, uid=sociallogin.account.uid, ).first() - if social_account: - # If the user is logged in, and the GH OAuth account belongs to - # a different user, we should not connect the accounts, - # this is the same as trying to connect an existing GH account to another user. - if ( - request.user.is_authenticated - and request.user != social_account.user - ): - message_template = ( - "socialaccount/messages/account_connected_other.txt" - ) - get_account_adapter(request).add_message( - request=request, - level=messages.ERROR, - message_template=message_template, - ) - url = reverse("socialaccount_connections") - raise ImmediateHttpResponse(HttpResponseRedirect(url)) - - sociallogin.connect(request, social_account.user) + # No existing GitHub account found, nothing to do. + if not social_account: + return + + # If the user is logged in, and the GH OAuth account belongs to + # a different user, we should not connect the accounts, + # this is the same as trying to connect an existing GH account to another user. + if ( + request.user.is_authenticated + and request.user != social_account.user + ): + message_template = ( + "socialaccount/messages/account_connected_other.txt" + ) + get_account_adapter(request).add_message( + request=request, + level=messages.ERROR, + message_template=message_template, + ) + url = reverse("socialaccount_connections") + raise ImmediateHttpResponse(HttpResponseRedirect(url)) + + sociallogin.connect(request, social_account.user) From 50637b554a64e9a97927f8a03cf163a0e9feea22 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Feb 2025 17:06:39 -0500 Subject: [PATCH 35/92] And both apps lived happy forever after. --- readthedocs/oauth/services/base.py | 25 ++++++++++++++----------- readthedocs/oauth/services/github.py | 6 ++++++ readthedocs/oauth/services/githubapp.py | 5 ++++- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 87722cc551f..299ab3e6178 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -155,15 +155,6 @@ def for_user(cls, user): def session(self): return get_oauth2_client(self.account) - def create_session(self): - """ - Create OAuth session for user. - - This configures the OAuth session based on the :py:class:`SocialToken` - attributes. If there is an ``expires_at``, treat the session as an auto - renewing token. Some providers expire tokens after as little as 2 hours. - """ - def paginate(self, url, **kwargs): """ Recursively combine results from service's pagination. @@ -248,7 +239,13 @@ def sync(self): remote_repository__remote_id__in=repository_remote_ids, remote_repository__vcs_provider=self.vcs_provider_slug, ) - .filter(account=self.account) + .filter( + account=self.account, + # Skip repositories that are managed by a GH app installation. + # NOTE: this is leaking the GH app logic into the parent class, + # but this works for now. + remote_repository__github_app_installation=None, + ) .delete() ) @@ -261,7 +258,13 @@ def sync(self): remote_organization__remote_id__in=organization_remote_ids, remote_organization__vcs_provider=self.vcs_provider_slug, ) - .filter(account=self.account) + .filter( + account=self.account, + # Skip organization that have repositories managed by a GH app installation. + # NOTE: this is leaking the GH app logic into the parent class, + # but this works for now. + remote_organization__remote_repositories__github_app_installation=None, + ) .delete() ) diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index c4dd5edfe45..3f82e727d86 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -103,6 +103,12 @@ def create_repository(self, fields, privacy=None): vcs_provider=self.vcs_provider_slug, ) + if repo.github_app_installation: + log.info( + "Repository is managed by a GitHub App installation, skipping.", + ) + return + # TODO: For debugging: https://github.com/readthedocs/readthedocs.org/pull/9449. if created: _old_remote_repository = RemoteRepository.objects.filter( diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 1ce7d36d379..ecef3bc52bc 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -31,7 +31,10 @@ class GitHubAppService(Service): def __init__(self, installation: GitHubAppInstallation): self.installation = installation - self.gha_client = get_gh_app_client() + + @cached_property + def gha_client(self): + return get_gh_app_client() @cached_property def app_installation(self) -> GHInstallation: From 7fc1466a2b09c6b7d0dd97ec21cc7569a5c97250 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Feb 2025 17:19:24 -0500 Subject: [PATCH 36/92] Updates from review --- readthedocs/builds/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/builds/tasks.py b/readthedocs/builds/tasks.py index 7e13ca62d19..1a085b823e1 100644 --- a/readthedocs/builds/tasks.py +++ b/readthedocs/builds/tasks.py @@ -407,7 +407,7 @@ def send_build_status(build_pk, commit, status): # in the future we should only consider projects that have a remote repository. service_class = build.project.get_git_service_class(fallback_to_clone_url=True) if not service_class: - log.info("Project isn't connected to a Git service.") + log.info("Project isn't connected to a Git service, not sending build status.") return False for service in service_class.for_project(build.project): @@ -430,7 +430,7 @@ def send_build_status(build_pk, commit, status): dismissable=True, ) - log.info("No social account or repository permission available.") + log.info("No social account or repository permission available, no build status sent.") return False From 27ad8c6a21dba388e6c7feb61206ea329f9a65de Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Feb 2025 17:28:59 -0500 Subject: [PATCH 37/92] Skip incompatible integrations --- readthedocs/oauth/utils.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/readthedocs/oauth/utils.py b/readthedocs/oauth/utils.py index e71ca87b056..544376ac7af 100644 --- a/readthedocs/oauth/utils.py +++ b/readthedocs/oauth/utils.py @@ -29,10 +29,19 @@ def update_webhook(project, integration, request=None): if not integration.secret: integration.save() - # The project was imported manually and doesn't have a RemoteRepository - # attached. We do brute force over all the accounts registered for this - # service - service_class = project.get_git_service_class() or service_class + # If the integration's service class is different from the project's + # git service class, we skip the update, as the webhook is not valid + # (we can't create a GitHub webhook for a GitLab project, for example). + if service_class != project.get_git_service_class(fallback_to_clone_url=True): + messages.error( + request, + _( + "This integration type is not compatible with the project's Git provider." + ), + ) + project.has_valid_webhook = False + project.save() + return False for service in service_class.for_project(project): updated, __ = service.update_webhook(project, integration) From 6ea1655d9c392fd25ae2abd7e67b505196335dbf Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Feb 2025 17:43:05 -0500 Subject: [PATCH 38/92] Format --- readthedocs/core/adapters.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/readthedocs/core/adapters.py b/readthedocs/core/adapters.py index 3397986a7af..cc07e701ca8 100644 --- a/readthedocs/core/adapters.py +++ b/readthedocs/core/adapters.py @@ -94,13 +94,8 @@ def pre_social_login(self, request, sociallogin): # If the user is logged in, and the GH OAuth account belongs to # a different user, we should not connect the accounts, # this is the same as trying to connect an existing GH account to another user. - if ( - request.user.is_authenticated - and request.user != social_account.user - ): - message_template = ( - "socialaccount/messages/account_connected_other.txt" - ) + if request.user.is_authenticated and request.user != social_account.user: + message_template = "socialaccount/messages/account_connected_other.txt" get_account_adapter(request).add_message( request=request, level=messages.ERROR, From 5daf4da3eac8e7219c23e1348695e19b0163f7e8 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Feb 2025 23:05:26 -0500 Subject: [PATCH 39/92] Check for attribute instead --- readthedocs/builds/tasks.py | 8 +++++++- readthedocs/oauth/services/base.py | 1 + readthedocs/oauth/services/bitbucket.py | 4 ---- readthedocs/oauth/services/github.py | 1 + readthedocs/oauth/services/gitlab.py | 1 + 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/readthedocs/builds/tasks.py b/readthedocs/builds/tasks.py index 1a085b823e1..211bcdf436c 100644 --- a/readthedocs/builds/tasks.py +++ b/readthedocs/builds/tasks.py @@ -410,6 +410,10 @@ def send_build_status(build_pk, commit, status): log.info("Project isn't connected to a Git service, not sending build status.") return False + if not service_class.supports_build_status: + log.info("Git service doesn't support build status.") + return False + for service in service_class.for_project(build.project): success = service.send_build_status( build, @@ -430,7 +434,9 @@ def send_build_status(build_pk, commit, status): dismissable=True, ) - log.info("No social account or repository permission available, no build status sent.") + log.info( + "No social account or repository permission available, no build status sent." + ) return False diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 941a7f277b5..b2c30a78f62 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -37,6 +37,7 @@ class Service: provider_name: str default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL + supports_build_status = False @classmethod def for_project(self, project): diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 992a7db3996..193c11ba03c 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -394,7 +394,3 @@ def update_webhook(self, project, integration): log.exception("Bitbucket webhook update failed for project.") return (False, resp) - - def send_build_status(self, build, commit, status): - """Send build status is not supported/implemented for Bitbucket.""" - return True diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 4dacfb5a2ce..e0f22457bd7 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -29,6 +29,7 @@ class GitHubService(UserService): url_pattern = re.compile(r"github\.com") vcs_provider_slug = GITHUB provider_name = "GitHub" + supports_build_status = True def sync_repositories(self): """Sync repositories from GitHub API.""" diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 4e65012c566..9050bcfabbb 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -33,6 +33,7 @@ class GitLabService(UserService): provider_name = "GitLab" adapter = GitLabOAuth2Adapter + supports_build_status = True # Just use the network location to determine if it's a GitLab project # because private repos have another base url, eg. git@gitlab.example.com url_pattern = re.compile( From 8e73ab47730df3eaab37b223d09c1a21e5083fef Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 13 Feb 2025 12:05:52 -0500 Subject: [PATCH 40/92] Fix querysets --- readthedocs/oauth/services/base.py | 25 +++++++++++++++++++++++-- readthedocs/oauth/services/github.py | 22 +++++++++++++++++++--- readthedocs/oauth/services/githubapp.py | 10 ++++++---- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 299ab3e6178..f84d4a58864 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -14,6 +14,11 @@ from readthedocs.core.permissions import AdminPermission from readthedocs.oauth.clients import get_oauth2_client +from readthedocs.oauth.models import ( + GitHubAccountType, + GitHubAppInstallation, + RemoteOrganization, +) log = structlog.get_logger(__name__) @@ -241,6 +246,7 @@ def sync(self): ) .filter( account=self.account, + remote_repository__vcs_provider=self.vcs_provider_slug, # Skip repositories that are managed by a GH app installation. # NOTE: this is leaking the GH app logic into the parent class, # but this works for now. @@ -253,6 +259,17 @@ def sync(self): organization_remote_ids = [ o.remote_id for o in remote_organizations if o is not None ] + + account_organizations = RemoteOrganization.objects.filter( + remote_organization_relations__account=self.account, + vcs_provider=self.vcs_provider_slug, + ) + + organizations_ids_managed_by_gh_app = GitHubAppInstallation.objects.filter( + target_id__in=account_organizations.values_list("remote_id", flat=True), + target_type=GitHubAccountType.ORGANIZATION, + ).values_list("target_id", flat=True) + ( self.user.remote_organization_relations.exclude( remote_organization__remote_id__in=organization_remote_ids, @@ -260,10 +277,14 @@ def sync(self): ) .filter( account=self.account, - # Skip organization that have repositories managed by a GH app installation. + remote_organization__vcs_provider=self.vcs_provider_slug, + ) + .exclude( + # Skip organization that are managed by a GH app installation. # NOTE: this is leaking the GH app logic into the parent class, # but this works for now. - remote_organization__remote_repositories__github_app_installation=None, + remote_organization__remote_id__in=organizations_ids_managed_by_gh_app, + remote_organization__vcs_provider=self.vcs_provider_slug, ) .delete() ) diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 3f82e727d86..40bd0739c27 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -14,14 +14,18 @@ from readthedocs.integrations.models import Integration from ..constants import GITHUB -from ..models import RemoteOrganization, RemoteRepository +from ..models import ( + GitHubAccountType, + GitHubAppInstallation, + RemoteOrganization, + RemoteRepository, +) from .base import SyncServiceError, UserService log = structlog.get_logger(__name__) class GitHubService(UserService): - """Provider service for GitHub.""" vcs_provider_slug = GITHUB @@ -60,6 +64,9 @@ def sync_organizations(self): org_details, create_user_relationship=True, ) + if not remote_organization: + continue + remote_organizations.append(remote_organization) org_url = org["url"] @@ -188,8 +195,17 @@ def create_organization(self, fields, create_user_relationship=False): will be created/updated. :rtype: RemoteOrganization """ + remote_id = fields["id"] + if GitHubAppInstallation.objects.filter( + target_id=remote_id, target_type=GitHubAccountType.ORGANIZATION + ).exists(): + log.info( + "Organization is managed by a GitHub App installation, skipping.", + ) + return + organization, _ = RemoteOrganization.objects.get_or_create( - remote_id=str(fields["id"]), + remote_id=str(remote_id), vcs_provider=self.vcs_provider_slug, ) if create_user_relationship: diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index ecef3bc52bc..1151737f62b 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -250,10 +250,12 @@ def _create_or_update_repository_from_gh( @lru_cache(maxsize=50) def _get_gh_organization(self, org_id: int) -> GHOrganization: """Get a GitHub organization object given its numeric ID.""" - # NOTE: cast to str, since PyGithub expects a string, - # even if the API accepts a string or an int. - # TODO: send a PR upstream to fix this. - return self.installation_client.get_organization(str(org_id)) + # NOTE: getting an organization by its numeric ID is not supported by PyGithub yet, + # see https://github.com/PyGithub/PyGithub/pull/3192. + # return self.installation_client.get_organization(org_id) + requester = self.installation_client.requester + headers, data = requester.requestJsonAndCheck("GET", f"/organizations/{org_id}") + return GHOrganization(requester, headers, data, completed=True) # NOTE: normally, this should cache only one organization at a time, but just in case... @lru_cache(maxsize=50) From 12892987dbc01abe9366c785086fad379307df0a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 13 Feb 2025 13:34:08 -0500 Subject: [PATCH 41/92] Git service: attach each service to a allauth provider --- readthedocs/api/v2/views/model_views.py | 4 +- readthedocs/builds/tasks.py | 2 +- readthedocs/oauth/services/base.py | 111 ++------------ readthedocs/oauth/services/bitbucket.py | 29 ++-- readthedocs/oauth/services/github.py | 34 ++--- readthedocs/oauth/services/gitlab.py | 40 +++-- readthedocs/oauth/tasks.py | 6 +- readthedocs/projects/models.py | 2 +- readthedocs/projects/views/private.py | 2 +- readthedocs/rtd_tests/tests/test_oauth.py | 174 ++++++++++------------ 10 files changed, 148 insertions(+), 256 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index b730a42bac8..7317a630999 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -420,7 +420,7 @@ def get_queryset(self): self.model.objects.api_v2(self.request.user) .filter( remote_organization_relations__account__provider__in=[ - service.adapter.provider_id for service in registry + service.allauth_provider.id for service in registry ] ) .distinct() @@ -466,7 +466,7 @@ def get_queryset(self): query = query.filter( remote_repository_relations__account__provider__in=[ - service.adapter.provider_id for service in registry + service.allauth_provider.id for service in registry ], ).distinct() diff --git a/readthedocs/builds/tasks.py b/readthedocs/builds/tasks.py index 211bcdf436c..464df1da544 100644 --- a/readthedocs/builds/tasks.py +++ b/readthedocs/builds/tasks.py @@ -428,7 +428,7 @@ def send_build_status(build_pk, commit, status): message_id=MESSAGE_OAUTH_BUILD_STATUS_FAILURE, attached_to=build.project, format_values={ - "provider_name": service_class.provider_name, + "provider_name": service_class.allauth_provider.name, "url_connect_account": reverse("socialaccount_connections"), }, dismissable=True, diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index b2c30a78f62..b8a4a42a27e 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -1,19 +1,18 @@ """OAuth utility functions.""" import re -from datetime import datetime +from functools import cached_property import structlog from allauth.socialaccount.models import SocialAccount -from allauth.socialaccount.providers.oauth2.views import OAuth2Adapter +from allauth.socialaccount.providers.oauth2.provider import OAuth2Provider from django.conf import settings from django.urls import reverse -from django.utils import timezone from django.utils.translation import gettext_lazy as _ from oauthlib.oauth2.rfc6749.errors import InvalidClientIdError from requests.exceptions import RequestException -from requests_oauthlib import OAuth2Session from readthedocs.core.permissions import AdminPermission +from readthedocs.oauth.clients import get_oauth2_client log = structlog.get_logger(__name__) @@ -33,8 +32,9 @@ class Service: """Base class for service that interacts with a VCS provider and a project.""" vcs_provider_slug: str + allauth_provider = type[OAuth2Provider] + url_pattern: re.Pattern | None - provider_name: str default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL supports_build_status = False @@ -127,15 +127,12 @@ class UserService(Service): :param account: :py:class:`SocialAccount` instance for user """ - adapter = None - def __init__(self, user, account): - self.session = None self.user = user self.account = account log.bind( user_username=self.user.username, - social_provider=self.provider_id, + social_provider=self.allauth_provider.id, social_account_id=self.account.pk, ) @@ -149,96 +146,14 @@ def for_project(cls, project): def for_user(cls, user): accounts = SocialAccount.objects.filter( user=user, - provider=cls.adapter.provider_id, + provider=cls.allauth_provider.id, ) for account in accounts: yield cls(user=user, account=account) - def get_adapter(self) -> type[OAuth2Adapter]: - return self.adapter - - @property - def provider_id(self): - return self.get_adapter().provider_id - - def get_session(self): - if self.session is None: - self.create_session() - return self.session - - def get_access_token_url(self): - # ``access_token_url`` is a property in some adapters, - # so we need to instantiate it to get the actual value. - # pylint doesn't recognize that get_adapter returns a class. - # pylint: disable=not-callable - adapter = self.get_adapter()(request=None) - return adapter.access_token_url - - def create_session(self): - """ - Create OAuth session for user. - - This configures the OAuth session based on the :py:class:`SocialToken` - attributes. If there is an ``expires_at``, treat the session as an auto - renewing token. Some providers expire tokens after as little as 2 hours. - """ - token = self.account.socialtoken_set.first() - if token is None: - return None - - token_config = { - "access_token": token.token, - "token_type": "bearer", - } - if token.expires_at is not None: - token_expires = (token.expires_at - timezone.now()).total_seconds() - token_config.update( - { - "refresh_token": token.token_secret, - "expires_in": token_expires, - } - ) - - social_app = self.account.get_provider().app - self.session = OAuth2Session( - client_id=social_app.client_id, - token=token_config, - auto_refresh_kwargs={ - "client_id": social_app.client_id, - "client_secret": social_app.secret, - }, - auto_refresh_url=self.get_access_token_url(), - token_updater=self.token_updater(token), - ) - - return self.session or None - - def token_updater(self, token): - """ - Update token given data from OAuth response. - - Expect the following response into the closure:: - - { - u'token_type': u'bearer', - u'scopes': u'webhook repository team account', - u'refresh_token': u'...', - u'access_token': u'...', - u'expires_in': 3600, - u'expires_at': 1449218652.558185 - } - """ - - def _updater(data): - token.token = data["access_token"] - token.token_secret = data.get("refresh_token", "") - token.expires_at = timezone.make_aware( - datetime.fromtimestamp(data["expires_at"]), - ) - token.save() - log.info("Updated token.", token_id=token.pk) - - return _updater + @cached_property + def session(self): + return get_oauth2_client(self.account) def paginate(self, url, **kwargs): """ @@ -251,7 +166,7 @@ def paginate(self, url, **kwargs): """ resp = None try: - resp = self.get_session().get(url, params=kwargs) + resp = self.session.get(url, params=kwargs) # TODO: this check of the status_code would be better in the # ``create_session`` method since it could be used from outside, but @@ -263,7 +178,7 @@ def paginate(self, url, **kwargs): # needs to reconnect his account raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( - provider=self.provider_name + provider=self.allauth_provider.name ) ) @@ -277,7 +192,7 @@ def paginate(self, url, **kwargs): log.warning("access_token or refresh_token failed.", url=url) raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( - provider=self.provider_name + provider=self.allauth_provider.name ) ) # Catch exceptions with request or deserializing JSON diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 193c11ba03c..60f36ae4ac8 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -4,8 +4,8 @@ import re import structlog -from allauth.socialaccount.providers.bitbucket_oauth2.views import ( - BitbucketOAuth2Adapter, +from allauth.socialaccount.providers.bitbucket_oauth2.provider import ( + BitbucketOAuth2Provider, ) from django.conf import settings from requests.exceptions import RequestException @@ -24,12 +24,12 @@ class BitbucketService(UserService): """Provider service for Bitbucket.""" - adapter = BitbucketOAuth2Adapter + vcs_provider_slug = BITBUCKET + allauth_provider = BitbucketOAuth2Provider + base_api_url = "https://api.bitbucket.org/2.0" # TODO replace this with a less naive check url_pattern = re.compile(r"bitbucket.org") https_url_pattern = re.compile(r"^https:\/\/[^@]+@bitbucket.org/") - vcs_provider_slug = BITBUCKET - provider_name = "Bitbucket" def sync_repositories(self): """Sync repositories from Bitbucket API.""" @@ -49,7 +49,7 @@ def sync_repositories(self): log.warning("Error syncing Bitbucket repositories") raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( - provider=self.vcs_provider_slug + provider=self.allauth_provider.name ) ) @@ -82,7 +82,7 @@ def sync_organizations(self): try: workspaces = self.paginate( - "https://api.bitbucket.org/2.0/workspaces/", + f"{self.base_api_url}/workspaces/", role="member", ) for workspace in workspaces: @@ -102,7 +102,7 @@ def sync_organizations(self): log.warning("Error syncing Bitbucket organizations") raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( - provider=self.vcs_provider_slug + provider=self.allauth_provider.name ) ) @@ -235,9 +235,8 @@ def get_provider_data(self, project, integration): if integration.provider_data: return integration.provider_data - session = self.get_session() owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) - url = f"https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/hooks" + url = f"{self.base_api_url}/repositories/{owner}/{repo}/hooks" rtd_webhook_url = self.get_webhook_url(project, integration) @@ -247,7 +246,7 @@ def get_provider_data(self, project, integration): url=url, ) try: - resp = session.get(url) + resp = self.session.get(url) if resp.status_code == 200: recv_data = resp.json() @@ -284,9 +283,8 @@ def setup_webhook(self, project, integration=None): :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ - session = self.get_session() owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) - url = f"https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/hooks" + url = f"{self.base_api_url}/repositories/{owner}/{repo}/hooks" if not integration: integration, _ = Integration.objects.get_or_create( project=project, @@ -302,7 +300,7 @@ def setup_webhook(self, project, integration=None): ) try: - resp = session.post( + resp = self.session.post( url, data=data, headers={"content-type": "application/json"}, @@ -355,13 +353,12 @@ def update_webhook(self, project, integration): if not provider_data: return self.setup_webhook(project, integration) - session = self.get_session() data = self.get_webhook_data(project, integration) resp = None try: # Expect to throw KeyError here if provider_data is invalid url = provider_data["links"]["self"]["href"] - resp = session.put( + resp = self.session.put( url, data=data, headers={"content-type": "application/json"}, diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index e0f22457bd7..e60c305c7f2 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -4,7 +4,7 @@ import re import structlog -from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter +from allauth.socialaccount.providers.github.provider import GitHubProvider from django.conf import settings from oauthlib.oauth2.rfc6749.errors import InvalidGrantError, TokenExpiredError from requests.exceptions import RequestException @@ -24,11 +24,11 @@ class GitHubService(UserService): """Provider service for GitHub.""" - adapter = GitHubOAuth2Adapter + vcs_provider_slug = GITHUB + allauth_provider = GitHubProvider + base_api_url = "https://api.github.com" # TODO replace this with a less naive check url_pattern = re.compile(r"github\.com") - vcs_provider_slug = GITHUB - provider_name = "GitHub" supports_build_status = True def sync_repositories(self): @@ -36,7 +36,7 @@ def sync_repositories(self): remote_repositories = [] try: - repos = self.paginate("https://api.github.com/user/repos", per_page=100) + repos = self.paginate(f"{self.base_api_url}/user/repos", per_page=100) for repo in repos: remote_repository = self.create_repository(repo) remote_repositories.append(remote_repository) @@ -55,9 +55,9 @@ def sync_organizations(self): remote_repositories = [] try: - orgs = self.paginate("https://api.github.com/user/orgs", per_page=100) + orgs = self.paginate(f"{self.base_api_url}/user/orgs", per_page=100) for org in orgs: - org_details = self.get_session().get(org["url"]).json() + org_details = self.session.get(org["url"]).json() remote_organization = self.create_organization( org_details, create_user_relationship=True, @@ -77,7 +77,7 @@ def sync_organizations(self): log.warning("Error syncing GitHub organizations") raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( - provider=self.vcs_provider_slug + provider=self.allauth_provider.name ) ) @@ -241,9 +241,8 @@ def get_provider_data(self, project, integration): if integration.provider_data: return integration.provider_data - session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) - url = f"https://api.github.com/repos/{owner}/{repo}/hooks" + url = f"{self.base_api_url}/repos/{owner}/{repo}/hooks" log.bind( url=url, project_slug=project.slug, @@ -253,7 +252,7 @@ def get_provider_data(self, project, integration): rtd_webhook_url = self.get_webhook_url(project, integration) try: - resp = session.get(url) + resp = self.session.get(url) if resp.status_code == 200: recv_data = resp.json() @@ -288,7 +287,6 @@ def setup_webhook(self, project, integration=None): :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ - session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) if not integration: @@ -298,7 +296,7 @@ def setup_webhook(self, project, integration=None): ) data = self.get_webhook_data(project, integration) - url = f"https://api.github.com/repos/{owner}/{repo}/hooks" + url = f"{self.base_api_url}/repos/{owner}/{repo}/hooks" log.bind( url=url, project_slug=project.slug, @@ -306,7 +304,7 @@ def setup_webhook(self, project, integration=None): ) resp = None try: - resp = session.post( + resp = self.session.post( url, data=data, headers={"content-type": "application/json"}, @@ -353,7 +351,6 @@ def update_webhook(self, project, integration): :returns: boolean based on webhook update success, and requests Response object :rtype: (Bool, Response) """ - session = self.get_session() data = self.get_webhook_data(project, integration) resp = None @@ -370,7 +367,7 @@ def update_webhook(self, project, integration): try: url = provider_data.get("url") - resp = session.patch( + resp = self.session.patch( url, data=data, headers={"content-type": "application/json"}, @@ -422,14 +419,13 @@ def send_build_status(self, build, commit, status): :returns: boolean based on commit status creation was successful or not. :rtype: Bool """ - session = self.get_session() project = build.project owner, repo = build_utils.get_github_username_repo(url=project.repo) # select the correct status and description. github_build_status = SELECT_BUILD_STATUS[status]["github"] description = SELECT_BUILD_STATUS[status]["description"] - statuses_url = f"https://api.github.com/repos/{owner}/{repo}/statuses/{commit}" + statuses_url = f"{self.base_api_url}/repos/{owner}/{repo}/statuses/{commit}" if status == BUILD_STATUS_SUCCESS: # Link to the documentation for this version @@ -457,7 +453,7 @@ def send_build_status(self, build, commit, status): ) resp = None try: - resp = session.post( + resp = self.session.post( statuses_url, data=json.dumps(data), headers={"content-type": "application/json"}, diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 9050bcfabbb..bb2dc21813d 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -5,7 +5,7 @@ from urllib.parse import quote_plus, urlparse import structlog -from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter +from allauth.socialaccount.providers.gitlab.provider import GitLabProvider from django.conf import settings from oauthlib.oauth2.rfc6749.errors import InvalidGrantError, TokenExpiredError from requests.exceptions import RequestException @@ -31,13 +31,13 @@ class GitLabService(UserService): - https://docs.gitlab.com/ce/api/oauth2.html """ - provider_name = "GitLab" - adapter = GitLabOAuth2Adapter + allauth_provider = GitLabProvider + base_api_url = "https://gitlab.com" supports_build_status = True # Just use the network location to determine if it's a GitLab project # because private repos have another base url, eg. git@gitlab.example.com url_pattern = re.compile( - re.escape(urlparse(adapter.provider_default_url).netloc), + re.escape(urlparse(base_api_url).netloc), ) PERMISSION_NO_ACCESS = 0 @@ -75,7 +75,7 @@ def sync_repositories(self): remote_repositories = [] try: repos = self.paginate( - "{url}/api/v4/projects".format(url=self.adapter.provider_default_url), + f"{self.base_api_url}/api/v4/projects", per_page=100, archived=False, order_by="path", @@ -102,7 +102,7 @@ def sync_organizations(self): try: orgs = self.paginate( - "{url}/api/v4/groups".format(url=self.adapter.provider_default_url), + f"{self.base_api_url}/api/v4/groups", per_page=100, all_available=False, order_by="path", @@ -112,7 +112,7 @@ def sync_organizations(self): remote_organization = self.create_organization(org) org_repos = self.paginate( "{url}/api/v4/groups/{id}/projects".format( - url=self.adapter.provider_default_url, + url=self.base_api_url, id=org["id"], ), per_page=100, @@ -131,9 +131,9 @@ def sync_organizations(self): # admin permission fields for GitLab projects. # So, fetch every single project data from the API # which contains the admin permission fields. - resp = self.get_session().get( + resp = self.session.get( "{url}/api/v4/projects/{id}".format( - url=self.adapter.provider_default_url, id=repo["id"] + url=self.base_api_url, id=repo["id"] ) ) @@ -272,7 +272,7 @@ def create_organization(self, fields): organization.name = fields.get("name") organization.slug = fields.get("path") organization.url = "{url}/{path}".format( - url=self.adapter.provider_default_url, + url=self.base_api_url, path=fields.get("path"), ) organization.avatar_url = fields.get("avatar_url") @@ -327,7 +327,6 @@ def get_provider_data(self, project, integration): if repo_id is None: return None - session = self.get_session() log.bind( project_slug=project.slug, integration_id=integration.pk, @@ -336,9 +335,9 @@ def get_provider_data(self, project, integration): rtd_webhook_url = self.get_webhook_url(project, integration) try: - resp = session.get( + resp = self.session.get( "{url}/api/v4/projects/{repo_id}/hooks".format( - url=self.adapter.provider_default_url, + url=self.base_api_url, repo_id=repo_id, ), ) @@ -385,7 +384,7 @@ def setup_webhook(self, project, integration=None): ) repo_id = self._get_repo_id(project) - url = f"{self.adapter.provider_default_url}/api/v4/projects/{repo_id}/hooks" + url = f"{self.base_api_url}/api/v4/projects/{repo_id}/hooks" if repo_id is None: return (False, resp) @@ -396,9 +395,8 @@ def setup_webhook(self, project, integration=None): url=url, ) data = self.get_webhook_data(repo_id, project, integration) - session = self.get_session() try: - resp = session.post( + resp = self.session.post( url, data=data, headers={"content-type": "application/json"}, @@ -448,7 +446,6 @@ def update_webhook(self, project, integration): return self.setup_webhook(project, integration) resp = None - session = self.get_session() repo_id = self._get_repo_id(project) if repo_id is None: @@ -462,9 +459,9 @@ def update_webhook(self, project, integration): ) try: hook_id = provider_data.get("id") - resp = session.put( + resp = self.session.put( "{url}/api/v4/projects/{repo_id}/hooks/{hook_id}".format( - url=self.adapter.provider_default_url, + url=self.base_api_url, repo_id=repo_id, hook_id=hook_id, ), @@ -512,7 +509,6 @@ def send_build_status(self, build, commit, status): :rtype: Bool """ resp = None - session = self.get_session() project = build.project repo_id = self._get_repo_id(project) @@ -539,7 +535,7 @@ def send_build_status(self, build, commit, status): "description": description, "context": context, } - url = f"{self.adapter.provider_default_url}/api/v4/projects/{repo_id}/statuses/{commit}" + url = f"{self.base_api_url}/api/v4/projects/{repo_id}/statuses/{commit}" log.bind( project_slug=project.slug, @@ -548,7 +544,7 @@ def send_build_status(self, build, commit, status): url=url, ) try: - resp = session.post( + resp = self.session.post( url, data=json.dumps(data), headers={"content-type": "application/json"}, diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index 13a6185c1f1..cff086804d4 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -49,7 +49,7 @@ def sync_remote_repositories(user_id): try: service.sync() except SyncServiceError: - failed_services.add(service.provider_name) + failed_services.add(service_cls.allauth_provider.name) if failed_services: raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( @@ -200,7 +200,7 @@ def attach_webhook(project_pk, user_pk=None, integration=None, **kwargs): dismissable=True, attached_to=project, format_values={ - "provider_name": service_class.provider_name, + "provider_name": service_class.allauth_provider.name, "url_connect_account": reverse( "projects_integrations", args=[project.slug], @@ -221,7 +221,7 @@ def attach_webhook(project_pk, user_pk=None, integration=None, **kwargs): dismissable=True, attached_to=project, format_values={ - "provider_name": service_class.provider_name, + "provider_name": service_class.allauth_provider.name, "url_docs_webhook": "https://docs.readthedocs.io/page/webhooks.html", }, ) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 7b4a33d0cbd..091c72bcea3 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1053,7 +1053,7 @@ def get_git_service_class(self, fallback_to_clone_url=False): def git_provider_name(self): """Get the provider name for project. e.g: GitHub, GitLab, Bitbucket.""" service_class = self.get_git_service_class(fallback_to_clone_url=True) - return service_class.provider_name if service_class else None + return service_class.allauth_provider.name if service_class else None def find(self, filename, version): """ diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index aeacb38c81c..ecbbbf1863e 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -489,7 +489,7 @@ def get(self, request, *args, **kwargs): deprecated_accounts = SocialAccount.objects.filter( user=self.request.user ).exclude( - provider__in=[service.adapter.provider_id for service in registry], + provider__in=[service.allauth_provider.id for service in registry], ) # yapf: disable for account in deprecated_accounts: provider_account = account.get_provider_account() diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index ba206f12a7a..02ee702535c 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -301,9 +301,9 @@ def test_multiple_users_same_repo(self): self.assertEqual(github_project_2, github_project_6) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_send_build_status_successful(self, session, mock_logger): - session().post.return_value.status_code = 201 + session.post.return_value.status_code = 201 success = self.service.send_build_status( self.external_build, self.external_build.commit, BUILD_STATUS_SUCCESS ) @@ -315,9 +315,9 @@ def test_send_build_status_successful(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_send_build_status_404_error(self, session, mock_logger): - session().post.return_value.status_code = 404 + session.post.return_value.status_code = 404 success = self.service.send_build_status( self.external_build, self.external_build.commit, BUILD_STATUS_SUCCESS ) @@ -329,9 +329,9 @@ def test_send_build_status_404_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_send_build_status_value_error(self, session, mock_logger): - session().post.side_effect = ValueError + session.post.side_effect = ValueError success = self.service.send_build_status( self.external_build, self.external_build.commit, BUILD_STATUS_SUCCESS ) @@ -357,10 +357,10 @@ def test_create_public_repo_when_private_projects_are_enabled(self): self.assertEqual(repo.remote_id, str(self.repo_with_org_response_data["id"])) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_setup_webhook_successful(self, session, mock_logger): - session().post.return_value.status_code = 201 - session().post.return_value.json.return_value = {} + session.post.return_value.status_code = 201 + session.post.return_value.json.return_value = {} success, _ = self.service.setup_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -373,9 +373,9 @@ def test_setup_webhook_successful(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_setup_webhook_404_error(self, session, mock_logger): - session().post.return_value.status_code = 404 + session.post.return_value.status_code = 404 success, _ = self.service.setup_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -387,9 +387,9 @@ def test_setup_webhook_404_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_setup_webhook_value_error(self, session, mock_logger): - session().post.side_effect = ValueError + session.post.side_effect = ValueError self.service.setup_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -405,10 +405,10 @@ def test_setup_webhook_value_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_update_webhook_successful(self, session, mock_logger): - session().patch.return_value.status_code = 201 - session().patch.return_value.json.return_value = {} + session.patch.return_value.status_code = 201 + session.patch.return_value.json.return_value = {} success, _ = self.service.update_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -423,29 +423,29 @@ def test_update_webhook_successful(self, session, mock_logger): "GitHub webhook update successful for project.", ) - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") @mock.patch("readthedocs.oauth.services.github.GitHubService.setup_webhook") def test_update_webhook_404_error(self, setup_webhook, session): - session().patch.return_value.status_code = 404 + session.patch.return_value.status_code = 404 self.service.update_webhook(self.project, self.integration) setup_webhook.assert_called_once_with(self.project, self.integration) - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") @mock.patch("readthedocs.oauth.services.github.GitHubService.setup_webhook") def test_update_webhook_no_provider_data(self, setup_webhook, session): self.integration.provider_data = {} self.integration.save() - session().patch.side_effect = AttributeError + session.patch.side_effect = AttributeError self.service.update_webhook(self.project, self.integration) setup_webhook.assert_called_once_with(self.project, self.integration) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_update_webhook_value_error(self, session, mock_logger): - session().patch.side_effect = ValueError + session.patch.side_effect = ValueError self.service.update_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -456,7 +456,7 @@ def test_update_webhook_value_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_get_provider_data_successful(self, session, mock_logger): self.integration.provider_data = {} self.integration.save() @@ -474,8 +474,8 @@ def test_get_provider_data_successful(self, session, mock_logger): ) webhook_data[0]["config"]["url"] = rtd_webhook_url - session().get.return_value.status_code = 200 - session().get.return_value.json.return_value = webhook_data + session.get.return_value.status_code = 200 + session.get.return_value.json.return_value = webhook_data self.service.get_provider_data(self.project, self.integration) @@ -492,12 +492,12 @@ def test_get_provider_data_successful(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_get_provider_data_404_error(self, session, mock_logger): self.integration.provider_data = {} self.integration.save() - session().get.return_value.status_code = 404 + session.get.return_value.status_code = 404 self.service.get_provider_data(self.project, self.integration) @@ -510,12 +510,12 @@ def test_get_provider_data_404_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.github.log") - @mock.patch("readthedocs.oauth.services.github.GitHubService.get_session") + @mock.patch("readthedocs.oauth.services.github.GitHubService.session") def test_get_provider_data_attribute_error(self, session, mock_logger): self.integration.provider_data = {} self.integration.save() - session().get.side_effect = AttributeError + session.get.side_effect = AttributeError self.service.get_provider_data(self.project, self.integration) @@ -531,10 +531,6 @@ def test_get_provider_data_attribute_error(self, session, mock_logger): "GitHub webhook Listing failed for project.", ) - def test_get_access_token_url(self): - url = self.service.get_access_token_url() - self.assertEqual(url, "https://github.com/login/oauth/access_token") - class BitbucketOAuthTests(TestCase): fixtures = ["eric", "test_data"] @@ -780,10 +776,10 @@ def test_import_with_no_token(self): self.assertEqual(services, []) @mock.patch("readthedocs.oauth.services.bitbucket.log") - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") def test_setup_webhook_successful(self, session, mock_logger): - session().post.return_value.status_code = 201 - session().post.return_value.json.return_value = {} + session.post.return_value.status_code = 201 + session.post.return_value.json.return_value = {} success, _ = self.service.setup_webhook(self.project, self.integration) self.assertTrue(success) @@ -797,9 +793,9 @@ def test_setup_webhook_successful(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.bitbucket.log") - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") def test_setup_webhook_404_error(self, session, mock_logger): - session().post.return_value.status_code = 404 + session.post.return_value.status_code = 404 success, _ = self.service.setup_webhook(self.project, self.integration) self.assertFalse(success) @@ -813,9 +809,9 @@ def test_setup_webhook_404_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.bitbucket.log") - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") def test_setup_webhook_value_error(self, session, mock_logger): - session().post.side_effect = ValueError + session.post.side_effect = ValueError self.service.setup_webhook(self.project, self.integration) mock_logger.bind.assert_called_with( @@ -828,10 +824,10 @@ def test_setup_webhook_value_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.bitbucket.log") - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") def test_update_webhook_successful(self, session, mock_logger): - session().put.return_value.status_code = 200 - session().put.return_value.json.return_value = {} + session.put.return_value.status_code = 200 + session.put.return_value.json.return_value = {} success, _ = self.service.update_webhook(self.project, self.integration) self.assertTrue(success) @@ -841,29 +837,29 @@ def test_update_webhook_successful(self, session, mock_logger): "Bitbucket webhook update successful for project.", ) - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.setup_webhook") def test_update_webhook_404_error(self, setup_webhook, session): - session().put.return_value.status_code = 404 + session.put.return_value.status_code = 404 self.service.update_webhook(self.project, self.integration) setup_webhook.assert_called_once_with(self.project, self.integration) - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.setup_webhook") def test_update_webhook_no_provider_data(self, setup_webhook, session): self.integration.provider_data = {} self.integration.save() - session().put.side_effect = AttributeError + session.put.side_effect = AttributeError self.service.update_webhook(self.project, self.integration) setup_webhook.assert_called_once_with(self.project, self.integration) @mock.patch("readthedocs.oauth.services.bitbucket.log") - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") def test_update_webhook_value_error(self, session, mock_logger): - session().put.side_effect = ValueError + session.put.side_effect = ValueError self.service.update_webhook(self.project, self.integration) mock_logger.bind.assert_called_with(project_slug=self.project.slug) @@ -872,7 +868,7 @@ def test_update_webhook_value_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.bitbucket.log") - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") def test_get_provider_data_successful(self, session, mock_logger): self.integration.provider_data = {} self.integration.save() @@ -890,8 +886,8 @@ def test_get_provider_data_successful(self, session, mock_logger): ) webhook_data["values"][0]["url"] = rtd_webhook_url - session().get.return_value.status_code = 200 - session().get.return_value.json.return_value = webhook_data + session.get.return_value.status_code = 200 + session.get.return_value.json.return_value = webhook_data self.service.get_provider_data(self.project, self.integration) @@ -908,12 +904,12 @@ def test_get_provider_data_successful(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.bitbucket.log") - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") def test_get_provider_data_404_error(self, session, mock_logger): self.integration.provider_data = {} self.integration.save() - session().get.return_value.status_code = 404 + session.get.return_value.status_code = 404 self.service.get_provider_data(self.project, self.integration) @@ -930,12 +926,12 @@ def test_get_provider_data_404_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.bitbucket.log") - @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.get_session") + @mock.patch("readthedocs.oauth.services.bitbucket.BitbucketService.session") def test_get_provider_data_attribute_error(self, session, mock_logger): self.integration.provider_data = {} self.integration.save() - session().get.side_effect = AttributeError + session.get.side_effect = AttributeError self.service.get_provider_data(self.project, self.integration) @@ -951,10 +947,6 @@ def test_get_provider_data_attribute_error(self, session, mock_logger): "Bitbucket webhook Listing failed for project.", ) - def test_get_access_token_url(self): - url = self.service.get_access_token_url() - self.assertEqual(url, "https://bitbucket.org/site/oauth2/access_token") - class GitLabOAuthTests(TestCase): fixtures = ["eric", "test_data"] @@ -1164,10 +1156,10 @@ def test_make_private_project(self): self.assertIsNotNone(repo) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") @mock.patch("readthedocs.oauth.services.gitlab.GitLabService._get_repo_id") def test_send_build_status_successful(self, repo_id, session, mock_logger): - session().post.return_value.status_code = 201 + session.post.return_value.status_code = 201 repo_id().return_value = "9999" success = self.service.send_build_status( @@ -1181,10 +1173,10 @@ def test_send_build_status_successful(self, repo_id, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") @mock.patch("readthedocs.oauth.services.gitlab.GitLabService._get_repo_id") def test_send_build_status_404_error(self, repo_id, session, mock_logger): - session().post.return_value.status_code = 404 + session.post.return_value.status_code = 404 repo_id.return_value = "9999" success = self.service.send_build_status( @@ -1198,10 +1190,10 @@ def test_send_build_status_404_error(self, repo_id, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") @mock.patch("readthedocs.oauth.services.gitlab.GitLabService._get_repo_id") def test_send_build_status_value_error(self, repo_id, session, mock_logger): - session().post.side_effect = ValueError + session.post.side_effect = ValueError repo_id().return_value = "9999" success = self.service.send_build_status( @@ -1221,10 +1213,10 @@ def test_send_build_status_value_error(self, repo_id, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") def test_setup_webhook_successful(self, session, mock_logger): - session().post.return_value.status_code = 201 - session().post.return_value.json.return_value = {} + session.post.return_value.status_code = 201 + session.post.return_value.json.return_value = {} success, _ = self.service.setup_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -1239,9 +1231,9 @@ def test_setup_webhook_successful(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") def test_setup_webhook_404_error(self, session, mock_logger): - session().post.return_value.status_code = 404 + session.post.return_value.status_code = 404 success, _ = self.service.setup_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -1254,9 +1246,9 @@ def test_setup_webhook_404_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") def test_setup_webhook_value_error(self, session, mock_logger): - session().post.side_effect = ValueError + session.post.side_effect = ValueError self.service.setup_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -1272,12 +1264,12 @@ def test_setup_webhook_value_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") @mock.patch("readthedocs.oauth.services.gitlab.GitLabService._get_repo_id") def test_update_webhook_successful(self, repo_id, session, mock_logger): repo_id.return_value = "9999" - session().put.return_value.status_code = 200 - session().put.return_value.json.return_value = {} + session.put.return_value.status_code = 200 + session.put.return_value.json.return_value = {} success, _ = self.service.update_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -1292,17 +1284,17 @@ def test_update_webhook_successful(self, repo_id, session, mock_logger): "GitLab webhook update successful for project.", ) - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.setup_webhook") @mock.patch("readthedocs.oauth.services.gitlab.GitLabService._get_repo_id") def test_update_webhook_404_error(self, repo_id, setup_webhook, session): repo_id.return_value = "9999" - session().put.return_value.status_code = 404 + session.put.return_value.status_code = 404 self.service.update_webhook(self.project, self.integration) setup_webhook.assert_called_once_with(self.project, self.integration) - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.setup_webhook") @mock.patch("readthedocs.oauth.services.gitlab.GitLabService._get_repo_id") def test_update_webhook_no_provider_data(self, repo_id, setup_webhook, session): @@ -1310,17 +1302,17 @@ def test_update_webhook_no_provider_data(self, repo_id, setup_webhook, session): self.integration.save() repo_id.return_value = "9999" - session().put.side_effect = AttributeError + session.put.side_effect = AttributeError self.service.update_webhook(self.project, self.integration) setup_webhook.assert_called_once_with(self.project, self.integration) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") @mock.patch("readthedocs.oauth.services.gitlab.GitLabService._get_repo_id") def test_update_webhook_value_error(self, repo_id, session, mock_logger): repo_id.return_value = "9999" - session().put.side_effect = ValueError + session.put.side_effect = ValueError self.service.update_webhook(self.project, self.integration) self.integration.refresh_from_db() @@ -1336,7 +1328,7 @@ def test_update_webhook_value_error(self, repo_id, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") def test_get_provider_data_successful(self, session, mock_logger): self.integration.provider_data = {} self.integration.save() @@ -1354,8 +1346,8 @@ def test_get_provider_data_successful(self, session, mock_logger): ) webhook_data[0]["url"] = rtd_webhook_url - session().get.return_value.status_code = 200 - session().get.return_value.json.return_value = webhook_data + session.get.return_value.status_code = 200 + session.get.return_value.json.return_value = webhook_data self.service.get_provider_data(self.project, self.integration) @@ -1371,12 +1363,12 @@ def test_get_provider_data_successful(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") def test_get_provider_data_404_error(self, session, mock_logger): self.integration.provider_data = {} self.integration.save() - session().get.return_value.status_code = 404 + session.get.return_value.status_code = 404 self.service.get_provider_data(self.project, self.integration) @@ -1392,12 +1384,12 @@ def test_get_provider_data_404_error(self, session, mock_logger): ) @mock.patch("readthedocs.oauth.services.gitlab.log") - @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.get_session") + @mock.patch("readthedocs.oauth.services.gitlab.GitLabService.session") def test_get_provider_data_attribute_error(self, session, mock_logger): self.integration.provider_data = {} self.integration.save() - session().get.side_effect = AttributeError + session.get.side_effect = AttributeError self.service.get_provider_data(self.project, self.integration) @@ -1411,7 +1403,3 @@ def test_get_provider_data_attribute_error(self, session, mock_logger): mock_logger.exception.assert_called_with( "GitLab webhook Listing failed for project.", ) - - def test_get_access_token_url(self): - url = self.service.get_access_token_url() - self.assertEqual(url, "https://gitlab.com/oauth/token") From 531960894011f36b7c1323b1bf2c4583848fddeb Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 13 Feb 2025 13:39:50 -0500 Subject: [PATCH 42/92] Missed this file --- readthedocs/oauth/clients.py | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 readthedocs/oauth/clients.py diff --git a/readthedocs/oauth/clients.py b/readthedocs/oauth/clients.py new file mode 100644 index 00000000000..9cc91ed9c59 --- /dev/null +++ b/readthedocs/oauth/clients.py @@ -0,0 +1,71 @@ +from datetime import datetime + +import structlog +from django.utils import timezone +from requests_oauthlib import OAuth2Session + +log = structlog.get_logger(__name__) + + +def _get_token_updater(token): + """ + Update token given data from OAuth response. + + Expect the following response into the closure:: + + { + u'token_type': u'bearer', + u'scopes': u'webhook repository team account', + u'refresh_token': u'...', + u'access_token': u'...', + u'expires_in': 3600, + u'expires_at': 1449218652.558185 + } + """ + + def _updater(data): + token.token = data["access_token"] + token.token_secret = data.get("refresh_token", "") + token.expires_at = timezone.make_aware( + datetime.fromtimestamp(data["expires_at"]), + ) + token.save() + log.info("Updated token.", token_id=token.pk) + + return _updater + + +def get_oauth2_client(account): + """Get an OAuth2 client for the given social account.""" + token = account.socialtoken_set.first() + if token is None: + return None + + token_config = { + "access_token": token.token, + "token_type": "bearer", + } + if token.expires_at is not None: + token_expires = (token.expires_at - timezone.now()).total_seconds() + token_config.update( + { + "refresh_token": token.token_secret, + "expires_in": token_expires, + } + ) + + provider = account.get_provider() + social_app = provider.app + oauth2_adapter = provider.get_oauth2_adapter(request=provider.request) + + session = OAuth2Session( + client_id=social_app.client_id, + token=token_config, + auto_refresh_kwargs={ + "client_id": social_app.client_id, + "client_secret": social_app.secret, + }, + auto_refresh_url=oauth2_adapter.access_token_url, + token_updater=_get_token_updater(token), + ) + return session From c9eb355711d5297ec81363e58d94c13af5c1e2e0 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 13 Feb 2025 13:46:08 -0500 Subject: [PATCH 43/92] Use just the hostname for the base_api_url --- readthedocs/oauth/services/bitbucket.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 60f36ae4ac8..bf8fc90daa6 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -26,7 +26,7 @@ class BitbucketService(UserService): vcs_provider_slug = BITBUCKET allauth_provider = BitbucketOAuth2Provider - base_api_url = "https://api.bitbucket.org/2.0" + base_api_url = "https://api.bitbucket.org" # TODO replace this with a less naive check url_pattern = re.compile(r"bitbucket.org") https_url_pattern = re.compile(r"^https:\/\/[^@]+@bitbucket.org/") @@ -82,7 +82,7 @@ def sync_organizations(self): try: workspaces = self.paginate( - f"{self.base_api_url}/workspaces/", + f"{self.base_api_url}/2.0/workspaces/", role="member", ) for workspace in workspaces: @@ -236,7 +236,7 @@ def get_provider_data(self, project, integration): return integration.provider_data owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) - url = f"{self.base_api_url}/repositories/{owner}/{repo}/hooks" + url = f"{self.base_api_url}/2.0/repositories/{owner}/{repo}/hooks" rtd_webhook_url = self.get_webhook_url(project, integration) @@ -284,7 +284,7 @@ def setup_webhook(self, project, integration=None): :rtype: (Bool, Response) """ owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) - url = f"{self.base_api_url}/repositories/{owner}/{repo}/hooks" + url = f"{self.base_api_url}/2.0/repositories/{owner}/{repo}/hooks" if not integration: integration, _ = Integration.objects.get_or_create( project=project, From e0314777e6bd20f50488b5081494adb5f4b0f39a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 13 Feb 2025 22:29:38 -0500 Subject: [PATCH 44/92] Stash --- readthedocs/oauth/models.py | 5 +- readthedocs/profiles/urls/private.py | 5 ++ readthedocs/profiles/views.py | 43 ++++++++-- readthedocs/settings/base.py | 1 + .../profiles/private/migrate-to-gh-app.html | 78 +++++++++++++++++++ 5 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 readthedocs/templates/profiles/private/migrate-to-gh-app.html diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 1f447a8f4db..f920fb6e643 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -94,7 +94,7 @@ class Meta(TimeStampedModel.Meta): @cached_property def service(self): """Return the service for this installation.""" - from readthedocs.oauth.services.githubapp import GitHubAppService + from readthedocs.oauth.services import GitHubAppService return GitHubAppService(self) @@ -400,8 +400,7 @@ def get_remote_repository_relation(self, user, social_account): return remote_repository_relation def get_service_class(self): - from readthedocs.oauth.services import registry - from readthedocs.oauth.services.githubapp import GitHubAppService + from readthedocs.oauth.services import registry, GitHubAppService if self.github_app_installation: return GitHubAppService diff --git a/readthedocs/profiles/urls/private.py b/readthedocs/profiles/urls/private.py index 41b17c998e9..7f1a9558479 100644 --- a/readthedocs/profiles/urls/private.py +++ b/readthedocs/profiles/urls/private.py @@ -40,6 +40,11 @@ views.AccountAdvertisingEdit.as_view(), name="account_advertising", ), + path( + "migrate-to-github-app/", + views.MigrateToGitHubAppView.as_view(), + name="migrate_to_github_app", + ) ] urlpatterns += account_urls diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index 874a2f518de..fa69f126382 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -12,8 +12,17 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from rest_framework.authtoken.models import Token -from vanilla import CreateView, DeleteView, DetailView, FormView, ListView, UpdateView - +from vanilla import ( + CreateView, + DeleteView, + DetailView, + FormView, + ListView, + TemplateView, + UpdateView, +) + +from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider from readthedocs.audit.filters import UserSecurityLogFilter from readthedocs.audit.models import AuditLog from readthedocs.core.forms import UserAdvertisingForm, UserDeleteForm, UserProfileForm @@ -22,6 +31,7 @@ from readthedocs.core.models import UserProfile from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.extend import SettingsOverrideObject +from readthedocs.oauth.migrate import get_installation_targets_for_user, get_old_app_link from readthedocs.organizations.models import Organization from readthedocs.projects.models import Project from readthedocs.projects.utils import get_csv_file @@ -44,7 +54,6 @@ class LogoutView(SettingsOverrideObject): class ProfileEdit(PrivateViewMixin, UpdateView): - """Edit the current user's profile.""" model = UserProfile @@ -147,7 +156,6 @@ def get_success_url(self): class TokenMixin(PrivateViewMixin): - """User token to access APIv3.""" model = Token @@ -169,7 +177,6 @@ class TokenListView(TokenMixin, ListView): class TokenCreateView(TokenMixin, CreateView): - """Simple view to generate a Token object for the logged in User.""" http_method_names = ["post"] @@ -182,7 +189,6 @@ def post(self, request, *args, **kwargs): class TokenDeleteView(TokenMixin, DeleteView): - """View to delete/revoke the current Token of the logged in User.""" http_method_names = ["post"] @@ -275,3 +281,28 @@ def get_queryset(self): queryset=queryset, ) return self.filter.qs + + +class MigrateToGitHubAppView(PrivateViewMixin, TemplateView): + + template_name = "profiles/private/migrate-to-gh-app.html" + + def get(self, request, *args, **kwargs): + # TODO: check if the user already migrated all their projects, + # or if the user doesn't have projects to migrate. + return super().get(request, *args, **kwargs) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + user = self.request.user + + context["has_gh_app_social_account"] = user.socialaccount_set.filter( + provider=GitHubAppProvider.id + ).exists() + context["installation_targets"] = get_installation_targets_for_user(user) + + context["projects"] = AdminPermission.projects(user, admin=True) + + context["old_application_link"] = get_old_app_link() + + return context diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 8adac805996..a60c659d3e7 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -749,6 +749,7 @@ def SOCIALACCOUNT_PROVIDERS(self): } GITHUB_APP_ID = 1234 + GITHUB_APP_NAME = "readthedocs" GITHUB_APP_PRIVATE_KEY = "" GITHUB_APP_WEBHOOK_SECRET = "" diff --git a/readthedocs/templates/profiles/private/migrate-to-gh-app.html b/readthedocs/templates/profiles/private/migrate-to-gh-app.html new file mode 100644 index 00000000000..c10dfbe51df --- /dev/null +++ b/readthedocs/templates/profiles/private/migrate-to-gh-app.html @@ -0,0 +1,78 @@ +{% extends "profiles/base_profile_edit.html" %} + +{% load i18n %} + +{% block title %}{% trans "Migrate account to GitHub App" %}{% endblock %} + +{% block profile-admin-tokens %}active{% endblock %} + +{% block edit_content_header %} {% trans "Migrate account to GitHub App" %} {% endblock %} + +{% block edit_content %} +
    +
  1. + Connect your account to the GH app: ({% if has_gh_app_social_account %}y{% else %}n{% endif %}) +
    2. +
  2. + Grant access to all your repositories connected to a project: + {# Group by user and organization #} + + + Or select one by one: + +
    3. + +
  3. + Migrate your projects to the GH app: + + Migrate all + + or migrate one by one: + + +
    4. + +
  4. + + Revoke access to the old GH OAuth app from your account + +
    5. + +
  5. + Remove the old GH app from your RTD account + (you will no longer see this page) +
    6. +
+{% endblock %} From 6165e4cda6559ecc1a9f60983430039e48a05518 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 14 Feb 2025 00:20:50 -0500 Subject: [PATCH 45/92] Format --- readthedocs/oauth/models.py | 2 +- readthedocs/profiles/urls/private.py | 2 +- readthedocs/profiles/views.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index f920fb6e643..1b28bfbd65c 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -400,7 +400,7 @@ def get_remote_repository_relation(self, user, social_account): return remote_repository_relation def get_service_class(self): - from readthedocs.oauth.services import registry, GitHubAppService + from readthedocs.oauth.services import GitHubAppService, registry if self.github_app_installation: return GitHubAppService diff --git a/readthedocs/profiles/urls/private.py b/readthedocs/profiles/urls/private.py index 7f1a9558479..39209eb6234 100644 --- a/readthedocs/profiles/urls/private.py +++ b/readthedocs/profiles/urls/private.py @@ -44,7 +44,7 @@ "migrate-to-github-app/", views.MigrateToGitHubAppView.as_view(), name="migrate_to_github_app", - ) + ), ] urlpatterns += account_urls diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index fa69f126382..ebb9ca61120 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -31,7 +31,10 @@ from readthedocs.core.models import UserProfile from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.extend import SettingsOverrideObject -from readthedocs.oauth.migrate import get_installation_targets_for_user, get_old_app_link +from readthedocs.oauth.migrate import ( + get_installation_targets_for_user, + get_old_app_link, +) from readthedocs.organizations.models import Organization from readthedocs.projects.models import Project from readthedocs.projects.utils import get_csv_file @@ -284,7 +287,6 @@ def get_queryset(self): class MigrateToGitHubAppView(PrivateViewMixin, TemplateView): - template_name = "profiles/private/migrate-to-gh-app.html" def get(self, request, *args, **kwargs): From 37738dfc9c7e4c4a28f800b05cf6fd6a4e7e220e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 17 Feb 2025 12:25:47 -0500 Subject: [PATCH 46/92] stash --- readthedocs/oauth/migrate.py | 122 ++++++++++++++++++ readthedocs/profiles/views.py | 1 + .../profiles/private/migrate-to-gh-app.html | 13 ++ 3 files changed, 136 insertions(+) create mode 100644 readthedocs/oauth/migrate.py diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py new file mode 100644 index 00000000000..7e8abdac43c --- /dev/null +++ b/readthedocs/oauth/migrate.py @@ -0,0 +1,122 @@ +from dataclasses import dataclass + +from allauth.socialaccount.providers.github.provider import GitHubProvider +from django.conf import settings + +from readthedocs.core.permissions import AdminPermission +from readthedocs.oauth.services import GitHubAppService, GitHubService + + +def migrate_project_to_github_app(project): + # No remote repository, nothing to migrate. + if not project.remote_repository: + return + + service_class = project.get_git_service_class() + + # Already migrated, nothing to do. + if service_class == GitHubAppService: + return + + # Not a GitHub project, nothing to migrate. + if service_class != GitHubService: + return + + +def migrate_user_to_github_app(user): + pass + + +class ProjectMigration: + + def __init__(self, project): + self.project = project + + def grant_permissions_url(self): + pass + + +@dataclass +class InstallationTarget: + target_id: str + target_type: str + target_name: str + repository_ids: set[str] + + @property + def link(self): + """ + See https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/migrating-oauth-apps-to-github-apps#prompt-users-to-install-your-github-app. + """ + repository_ids = [] + for repository_id in self.repository_ids: + repository_ids.append(f"&repository_ids[]={repository_id}") + repository_ids = "".join(repository_ids) + + base_url = f"https://github.com/apps/{settings.GITHUB_APP_NAME}/installations/new/permissions" + return f"{base_url}?suggested_target_id={self.target_id}{repository_ids}" + + +def get_installation_targets_for_user(user) -> list[InstallationTarget]: + targets = {} + user_repositories = set() + for project in AdminPermission.projects(user, admin=True).select_related( + "remote_repository", "remote_repository__organization" + ): + remote_repository = project.remote_repository + if not remote_repository: + continue + + service_class = remote_repository.get_service_class() + if service_class != GitHubService: + continue + + if remote_repository.organization: + organization_id = remote_repository.organization.remote_id + if organization_id not in targets: + targets[organization_id] = InstallationTarget( + target_id=organization_id, + target_name=remote_repository.organization.slug, + target_type="organization", + repository_ids=set(), + ) + targets[organization_id].repository_ids.add(remote_repository.remote_id) + else: + user_repositories.add(remote_repository) + + # TODO: check how many users have more than one GH account connected. + # Since we don't know the ID of the owner, we create a link for each connected account. + # GH will select only the corresponding repositories for each account. + for account in user.socialaccount_set.filter(provider=GitHubProvider.id): + targets[account.uid] = InstallationTarget( + target_id=account.uid, + target_name=account.extra_data.get("login"), + target_type="user", + repository_ids={ + remote_repository.remote_id for remote_repository in user_repositories + }, + ) + + return [target for target in targets.values() if target.repository_ids] + + +def get_installation_target_for_project( + account, project +) -> tuple[str, str] | tuple[None, None]: + remote_repository = project.remote_repository + if not remote_repository: + return None, None + + service_class = remote_repository.get_service_class() + if service_class != GitHubService: + return None, None + + if remote_repository.organization: + return remote_repository.organization.remote_id, remote_repository.remote_id + + return account.uid, remote_repository.remote_id + + +def get_old_app_link(): + client_id = settings.SOCIALACCOUNT_PROVIDERS["github"]["APPS"][0]["client_id"] + return f"https://github.com/settings/connections/applications/{client_id}" diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index ebb9ca61120..01f37a2d597 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -298,6 +298,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) user = self.request.user + context["gh_app_provider"] = GitHubAppProvider context["has_gh_app_social_account"] = user.socialaccount_set.filter( provider=GitHubAppProvider.id ).exists() diff --git a/readthedocs/templates/profiles/private/migrate-to-gh-app.html b/readthedocs/templates/profiles/private/migrate-to-gh-app.html index c10dfbe51df..7c3228f665b 100644 --- a/readthedocs/templates/profiles/private/migrate-to-gh-app.html +++ b/readthedocs/templates/profiles/private/migrate-to-gh-app.html @@ -1,5 +1,6 @@ {% extends "profiles/base_profile_edit.html" %} +{% load provider_login_url from socialaccount %} {% load i18n %} {% block title %}{% trans "Migrate account to GitHub App" %}{% endblock %} @@ -12,6 +13,18 @@
  1. Connect your account to the GH app: ({% if has_gh_app_social_account %}y{% else %}n{% endif %}) + +
    + {% csrf_token %} + +
  2. Grant access to all your repositories connected to a project: From cd5015cc31e69ce855242ecf8e0e7625836644a2 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 17 Feb 2025 21:14:12 -0500 Subject: [PATCH 47/92] Basic migration page --- readthedocs/oauth/constants.py | 2 + readthedocs/oauth/migrate.py | 220 +++++++++++++----- readthedocs/oauth/querysets.py | 11 +- readthedocs/oauth/services/base.py | 24 +- readthedocs/oauth/services/github.py | 60 ++++- readthedocs/oauth/services/githubapp.py | 4 +- readthedocs/profiles/views.py | 50 +++- readthedocs/settings/docker_compose.py | 1 + .../profiles/private/migrate-to-gh-app.html | 113 ++++++--- 9 files changed, 349 insertions(+), 136 deletions(-) diff --git a/readthedocs/oauth/constants.py b/readthedocs/oauth/constants.py index 454c1e6e957..aa307b60a9c 100644 --- a/readthedocs/oauth/constants.py +++ b/readthedocs/oauth/constants.py @@ -1,9 +1,11 @@ GITHUB = "github" +GITHUB_APP = "githubapp" GITLAB = "gitlab" BITBUCKET = "bitbucket" VCS_PROVIDER_CHOICES = ( (GITHUB, "GitHub"), + (GITHUB_APP, "GitHub"), (GITLAB, "GitLab"), (BITBUCKET, "Bitbucket"), ) diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index 7e8abdac43c..d2b6bc11f85 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -4,40 +4,15 @@ from django.conf import settings from readthedocs.core.permissions import AdminPermission +from readthedocs.integrations.models import Integration +from readthedocs.oauth.constants import GITHUB, GITHUB_APP +from readthedocs.oauth.models import RemoteRepository from readthedocs.oauth.services import GitHubAppService, GitHubService - - -def migrate_project_to_github_app(project): - # No remote repository, nothing to migrate. - if not project.remote_repository: - return - - service_class = project.get_git_service_class() - - # Already migrated, nothing to do. - if service_class == GitHubAppService: - return - - # Not a GitHub project, nothing to migrate. - if service_class != GitHubService: - return - - -def migrate_user_to_github_app(user): - pass - - -class ProjectMigration: - - def __init__(self, project): - self.project = project - - def grant_permissions_url(self): - pass +from readthedocs.projects.models import Project @dataclass -class InstallationTarget: +class InstallationTargetGroup: target_id: str target_type: str target_name: str @@ -56,39 +31,45 @@ def link(self): base_url = f"https://github.com/apps/{settings.GITHUB_APP_NAME}/installations/new/permissions" return f"{base_url}?suggested_target_id={self.target_id}{repository_ids}" + @property + def installed(self): + # If we don't have any repositories, the app was already installed, + # or we don't have any repositories to install the app. + return not bool(self.repository_ids) + -def get_installation_targets_for_user(user) -> list[InstallationTarget]: +def get_installation_target_groups_for_user(user) -> list[InstallationTargetGroup]: targets = {} user_repositories = set() - for project in AdminPermission.projects(user, admin=True).select_related( - "remote_repository", "remote_repository__organization" - ): + for project, has_intallation, _ in _get_projects_for_user(user): remote_repository = project.remote_repository - if not remote_repository: - continue - - service_class = remote_repository.get_service_class() - if service_class != GitHubService: - continue - if remote_repository.organization: organization_id = remote_repository.organization.remote_id if organization_id not in targets: - targets[organization_id] = InstallationTarget( + targets[organization_id] = InstallationTargetGroup( target_id=organization_id, target_name=remote_repository.organization.slug, target_type="organization", repository_ids=set(), ) - targets[organization_id].repository_ids.add(remote_repository.remote_id) - else: + if not has_intallation: + targets[organization_id].repository_ids.add(remote_repository.remote_id) + elif not has_intallation: user_repositories.add(remote_repository) # TODO: check how many users have more than one GH account connected. + # from allauth.socialaccount.providers.github.provider import GitHubProvider + # from django.contrib.auth.models import User + # from django.db.models import Count, Q + # # Find all users that have more than one GitHub account connected. + # users = User.objects.annotate( + # num_github_accounts=Count("socialaccount", filter=Q(socialaccount__provider=GitHubProvider.id)) + # ).filter(num_github_accounts__gt=1) + # 166 on .org, 17 on .com # Since we don't know the ID of the owner, we create a link for each connected account. # GH will select only the corresponding repositories for each account. for account in user.socialaccount_set.filter(provider=GitHubProvider.id): - targets[account.uid] = InstallationTarget( + targets[account.uid] = InstallationTargetGroup( target_id=account.uid, target_name=account.extra_data.get("login"), target_type="user", @@ -97,26 +78,149 @@ def get_installation_targets_for_user(user) -> list[InstallationTarget]: }, ) - return [target for target in targets.values() if target.repository_ids] + return list(targets.values()) -def get_installation_target_for_project( - account, project -) -> tuple[str, str] | tuple[None, None]: - remote_repository = project.remote_repository - if not remote_repository: - return None, None +def _get_projects_for_user(user): + for project in get_projects_missing_migration(user): + remote_repository = project.remote_repository + has_installation = RemoteRepository.objects.filter( + remote_id=remote_repository.remote_id, + vcs_provider=GITHUB_APP, + github_app_installation__isnull=False, + ).exists() + is_admin = ( + RemoteRepository.objects.for_project_linking(user) + .filter( + remote_id=project.remote_repository.remote_id, + vcs_provider=GITHUB_APP, + github_app_installation__isnull=False, + ) + .exists() + ) + yield project, has_installation, is_admin - service_class = remote_repository.get_service_class() - if service_class != GitHubService: - return None, None - if remote_repository.organization: - return remote_repository.organization.remote_id, remote_repository.remote_id +def get_projects_missing_migration(user): + return ( + AdminPermission.projects(user, admin=True) + .filter(remote_repository__vcs_provider=GITHUB) + .select_related( + "remote_repository", + "remote_repository__organization", + ) + ) - return account.uid, remote_repository.remote_id + +@dataclass +class MigrationTarget: + project: Project + has_installation: bool + is_admin: bool + target_id: int + + @property + def installation_link(self): + """ + See https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/migrating-oauth-apps-to-github-apps + """ + base_url = f"https://github.com/apps/{settings.GITHUB_APP_NAME}/installations/new/permissions" + return f"{base_url}?suggested_target_id={self.target_id}&repository_ids[]={self.project.remote_repository.remote_id}" + + @property + def can_migrate(self): + return self.is_admin and self.has_installation + + +def get_migration_targets(user): + targets = [] + # NOTE: there are some users that have more than one GH account connected. + # so this isn't 100% accurate. + gh_account = user.socialaccount_set.filter(provider=GITHUB).first() + for project, has_installation, is_admin in _get_projects_for_user(user): + remote_repository = project.remote_repository + + if remote_repository.organization: + target_id = remote_repository.organization.remote_id + else: + target_id = gh_account.uid + targets.append( + MigrationTarget( + project=project, + has_installation=has_installation, + is_admin=is_admin, + target_id=target_id, + ) + ) + return targets def get_old_app_link(): client_id = settings.SOCIALACCOUNT_PROVIDERS["github"]["APPS"][0]["client_id"] return f"https://github.com/settings/connections/applications/{client_id}" + + +@dataclass +class MigrationResult: + webhook_removed: bool + ssh_key_removed: bool + + +class MigrationError(Exception): + pass + + +def migrate_project_to_github_app(project, user) -> MigrationResult: + # No remote repository, nothing to migrate. + if not project.remote_repository: + raise MigrationError("Project isn't connected to a repository") + + service_class = project.get_git_service_class() + + # Already migrated, nothing to do. + if service_class == GitHubAppService: + return MigrationResult(webhook_removed=True, ssh_key_removed=True) + + # Not a GitHub project, nothing to migrate. + if service_class != GitHubService: + raise MigrationError("Project isn't connected to a GitHub repository") + + new_remote_repository = RemoteRepository.objects.filter( + remote_id=project.remote_repository.remote_id, + vcs_provider=GITHUB_APP, + github_app_installation__isnull=False, + ).first() + + if not new_remote_repository: + raise MigrationError("You need to install the GitHub App on the repository") + + new_remote_repository = ( + RemoteRepository.objects.for_project_linking(user) + .filter( + remote_id=project.remote_repository.remote_id, + vcs_provider=GITHUB_APP, + github_app_installation__isnull=False, + ) + .first() + ) + if not new_remote_repository: + raise MigrationError( + "You must have admin permissions on the repository to migrate it" + ) + + webhook_removed = False + ssh_key_removed = False + for service in service_class.for_project(project): + if not webhook_removed and service.remove_webhook(project): + webhook_removed = True + + if not ssh_key_removed and service.remove_ssh_key(project): + ssh_key_removed = True + + project.integrations.filter(integration_type=Integration.GITHUB_WEBHOOK).delete() + project.remote_repository = new_remote_repository + project.save() + return MigrationResult( + webhook_removed=webhook_removed, + ssh_key_removed=ssh_key_removed, + ) diff --git a/readthedocs/oauth/querysets.py b/readthedocs/oauth/querysets.py index e123d1ce98f..5bb311ca74e 100644 --- a/readthedocs/oauth/querysets.py +++ b/readthedocs/oauth/querysets.py @@ -27,16 +27,9 @@ def for_project_linking(self, user): """ Return repositories that can be linked to a project by the given user. - Repositories can be imported if: - - - The user has read or adming access to the repository on the VCS service. - - If the repository is private, the user must be an admin. - - If the repository is public, the user doesn't need to be an admin. + Repositories can be imported if the user has admin access to the repository on the VCS service. """ - query = Q(remote_repository_relations__user=user) & ( - Q(private=False) | Q(private=True, remote_repository_relations__admin=True) - ) - return self.filter(query).distinct() + return self.filter(remote_repository_relations__user=user, remote_repository_relations__admin=True).distinct() class RemoteOrganizationQuerySet(RelatedUserQuerySet): diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 6dcfc45b628..16ec593efbd 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -251,7 +251,7 @@ def sync(self): # Skip repositories that are managed by a GH app installation. # NOTE: this is leaking the GH app logic into the parent class, # but this works for now. - remote_repository__github_app_installation=None, + # remote_repository__github_app_installation=None, ) .delete() ) @@ -261,15 +261,15 @@ def sync(self): o.remote_id for o in remote_organizations if o is not None ] - account_organizations = RemoteOrganization.objects.filter( - remote_organization_relations__account=self.account, - vcs_provider=self.vcs_provider_slug, - ) - - organizations_ids_managed_by_gh_app = GitHubAppInstallation.objects.filter( - target_id__in=account_organizations.values_list("remote_id", flat=True), - target_type=GitHubAccountType.ORGANIZATION, - ).values_list("target_id", flat=True) + # account_organizations = RemoteOrganization.objects.filter( + # remote_organization_relations__account=self.account, + # vcs_provider=self.vcs_provider_slug, + # ) + # + # organizations_ids_managed_by_gh_app = GitHubAppInstallation.objects.filter( + # target_id__in=account_organizations.values_list("remote_id", flat=True), + # target_type=GitHubAccountType.ORGANIZATION, + # ).values_list("target_id", flat=True) ( self.user.remote_organization_relations.exclude( @@ -284,8 +284,8 @@ def sync(self): # Skip organization that are managed by a GH app installation. # NOTE: this is leaking the GH app logic into the parent class, # but this works for now. - remote_organization__remote_id__in=organizations_ids_managed_by_gh_app, - remote_organization__vcs_provider=self.vcs_provider_slug, + # remote_organization__remote_id__in=organizations_ids_managed_by_gh_app, + # remote_organization__vcs_provider=self.vcs_provider_slug, ) .delete() ) diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index b9d9a26a6ae..90d603d9d13 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -113,11 +113,11 @@ def create_repository(self, fields, privacy=None): vcs_provider=self.vcs_provider_slug, ) - if repo.github_app_installation: - log.info( - "Repository is managed by a GitHub App installation, skipping.", - ) - return + # if repo.github_app_installation: + # log.info( + # "Repository is managed by a GitHub App installation, skipping.", + # ) + # return # TODO: For debugging: https://github.com/readthedocs/readthedocs.org/pull/9449. if created: @@ -199,13 +199,13 @@ def create_organization(self, fields, create_user_relationship=False): :rtype: RemoteOrganization """ remote_id = fields["id"] - if GitHubAppInstallation.objects.filter( - target_id=remote_id, target_type=GitHubAccountType.ORGANIZATION - ).exists(): - log.info( - "Organization is managed by a GitHub App installation, skipping.", - ) - return + # if GitHubAppInstallation.objects.filter( + # target_id=remote_id, target_type=GitHubAccountType.ORGANIZATION + # ).exists(): + # log.info( + # "Organization is managed by a GitHub App installation, skipping.", + # ) + # return organization, _ = RemoteOrganization.objects.get_or_create( remote_id=str(remote_id), @@ -429,6 +429,42 @@ def update_webhook(self, project, integration): return (False, resp) + def remove_webhook(self, project): + # TODO: use remote repo instead? + owner, repo = build_utils.get_github_username_repo(url=project.repo) + + try: + resp = self.session.get(f"{self.base_api_url}/repos/{owner}/{repo}/hooks") + resp.raise_for_status() + data = resp.json() + except Exception: + log.info("Failed to get GitHub webhooks for project.") + return False + + hook_targets = [ + f"{settings.PUBLIC_API_URL}/api/v2/webhook/{project.slug}/", + f"{settings.PUBLIC_API_URL}/api/v2/webhook/github/{project.slug}/", + ] + if "app." in settings.PUBLIC_API_URL: + hook_targets.append(hook_targets[0].replace("app.", "", 1)) + hook_targets.append(hook_targets[1].replace("app.", "", 1)) + + for hook in data: + hook_url = hook["config"]["url"] + for hook_target in hook_targets: + if hook_url.startswith(hook_target): + try: + self.session.delete(f"{self.base_api_url}/repos/{owner}/{repo}/hooks/{hook['id']}").raise_for_status() + except Exception: + log.info("Failed to remove GitHub webhook for project.") + return False + return True + + def remove_ssh_key(self, project): + # Overridden in corporate + return True + + def send_build_status(self, *, build, commit, status): """ Create GitHub commit status for project. diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 1151737f62b..20b3ac14e15 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -11,7 +11,7 @@ from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider from readthedocs.builds.constants import BUILD_STATUS_SUCCESS, SELECT_BUILD_STATUS from readthedocs.oauth.clients import get_gh_app_client, get_oauth2_client -from readthedocs.oauth.constants import GITHUB +from readthedocs.oauth.constants import GITHUB_APP from readthedocs.oauth.models import ( GitHubAccountType, GitHubAppInstallation, @@ -26,7 +26,7 @@ class GitHubAppService(Service): - vcs_provider_slug = GITHUB + vcs_provider_slug = GITHUB_APP allauth_provider = GitHubAppProvider def __init__(self, installation: GitHubAppInstallation): diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index 01f37a2d597..67405af9265 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -2,6 +2,8 @@ from allauth.account.views import LoginView as AllAuthLoginView from allauth.account.views import LogoutView as AllAuthLogoutView +from allauth.socialaccount.adapter import get_adapter as get_social_account_adapter +from allauth.socialaccount.providers.github.provider import GitHubProvider from django.conf import settings from django.contrib import messages from django.contrib.auth import logout @@ -32,8 +34,11 @@ from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.oauth.migrate import ( - get_installation_targets_for_user, + get_installation_target_groups_for_user, + get_migration_targets, get_old_app_link, + get_projects_missing_migration, + migrate_project_to_github_app, ) from readthedocs.organizations.models import Organization from readthedocs.projects.models import Project @@ -298,14 +303,45 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) user = self.request.user - context["gh_app_provider"] = GitHubAppProvider + # NOTE: I tried passing the GitHubAppProvider class directly to the template, + # but it doesn't work for some reason. + context["gh_app_provider"] = get_social_account_adapter().get_provider( + request=self.request, provider=GitHubAppProvider.id + ) + context["gh_provider"] = get_social_account_adapter().get_provider( + request=self.request, provider=GitHubProvider.id + ) + context["has_gh_app_social_account"] = user.socialaccount_set.filter( provider=GitHubAppProvider.id ).exists() - context["installation_targets"] = get_installation_targets_for_user(user) - - context["projects"] = AdminPermission.projects(user, admin=True) - + context["installation_target_groups"] = get_installation_target_groups_for_user( + user + ) + context["migration_targets"] = get_migration_targets(user) context["old_application_link"] = get_old_app_link() - return context + + def post(self, request, *args, **kwargs): + project_slug = request.POST.get("project") + if project_slug: + projects = AdminPermission.projects(request.user, admin=True).filter( + slug=project_slug + ) + else: + projects = get_projects_missing_migration(request.user) + + has_errors = False + for project in projects: + try: + migrate_project_to_github_app(project=project, user=request.user) + except Exception as e: + has_errors = True + messages.error(request, f"Error migrating project {project.slug}: {e}") + + if not has_errors: + messages.success(request, _("Projects migrated successfully")) + + # if has_errors: + # return HttpResponseRedirect(reverse("migrate_to_gh_app")) + return HttpResponseRedirect(reverse("migrate_to_github_app")) diff --git a/readthedocs/settings/docker_compose.py b/readthedocs/settings/docker_compose.py index f1607fdb659..883fa788bf9 100644 --- a/readthedocs/settings/docker_compose.py +++ b/readthedocs/settings/docker_compose.py @@ -226,6 +226,7 @@ def SOCIALACCOUNT_PROVIDERS(self): return providers GITHUB_APP_ID = os.environ.get("RTD_GITHUB_APP_ID") + GITHUB_APP_NAME = os.environ.get("RTD_GITHUB_APP_NAME") GITHUB_APP_WEBHOOK_SECRET = os.environ.get("RTD_GITHUB_APP_WEBHOOK_SECRET") GITHUB_APP_PRIVATE_KEY = os.environ.get("RTD_GITHUB_APP_PRIVATE_KEY") diff --git a/readthedocs/templates/profiles/private/migrate-to-gh-app.html b/readthedocs/templates/profiles/private/migrate-to-gh-app.html index 7c3228f665b..8b8afd36025 100644 --- a/readthedocs/templates/profiles/private/migrate-to-gh-app.html +++ b/readthedocs/templates/profiles/private/migrate-to-gh-app.html @@ -12,80 +12,121 @@ {% block edit_content %}
    1. - Connect your account to the GH app: ({% if has_gh_app_social_account %}y{% else %}n{% endif %}) + Connect your account to the new GitHub app: + {% url "migrate_to_github_app" as migrate_to_github_app_url %}
      + action="{% provider_login_url gh_app_provider.id process="connect" next=migrate_to_github_app_url %}"> {% csrf_token %}
    2. - Grant access to all your repositories connected to a project: - {# Group by user and organization #} - + Install app in all your repositories connected to a project: - Or select one by one: - + + Or install one by one in the next step + {% else %} +

      + You have already granted access to all your repositories. +

      + {% endif %}
    3. Migrate your projects to the GH app: - Migrate all +
        +
      • +
        + {% csrf_token %} + +
        +
        • +
      - or migrate one by one: + Or migrate one by one:
    4. - + Revoke access to the old GH OAuth app from your account
    5. - Remove the old GH app from your RTD account - (you will no longer see this page) + Disconnect the old GH app from your RTD account: + +
      + {% csrf_token %} + +
    {% endblock %} From f67e028bf5d52022f086b1d18dc9b0300d4b390f Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 18 Feb 2025 10:45:15 -0500 Subject: [PATCH 48/92] Don't use the ID to get the organization --- readthedocs/oauth/services/githubapp.py | 28 ++++--------------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 20b3ac14e15..0cec2a587d1 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -237,7 +237,7 @@ def _create_or_update_repository_from_gh( if gh_repo.owner.type == GitHubAccountType.ORGANIZATION: # NOTE: The owner object doesn't have all attributes of an organization, # so we need to fetch the organization object. - gh_organization = self._get_gh_organization(gh_repo.owner.id) + gh_organization = self._get_gh_organization(gh_repo.owner.login) remote_repo.organization = self._create_or_update_organization_from_gh( gh_organization ) @@ -248,14 +248,9 @@ def _create_or_update_repository_from_gh( # NOTE: normally, this should cache only one organization at a time, but just in case... @lru_cache(maxsize=50) - def _get_gh_organization(self, org_id: int) -> GHOrganization: - """Get a GitHub organization object given its numeric ID.""" - # NOTE: getting an organization by its numeric ID is not supported by PyGithub yet, - # see https://github.com/PyGithub/PyGithub/pull/3192. - # return self.installation_client.get_organization(org_id) - requester = self.installation_client.requester - headers, data = requester.requestJsonAndCheck("GET", f"/organizations/{org_id}") - return GHOrganization(requester, headers, data, completed=True) + def _get_gh_organization(self, login: str) -> GHOrganization: + """Get a GitHub organization object given its login identifier.""" + return self.installation_client.get_organization(login) # NOTE: normally, this should cache only one organization at a time, but just in case... @lru_cache(maxsize=50) @@ -322,21 +317,6 @@ def _get_social_accounts(self, ids): provider=self.allauth_provider.id, ).select_related("user") - def update_or_create_organization(self, org_id: int) -> RemoteOrganization | None: - try: - gh_org = self._get_gh_organization(org_id) - return self._create_or_update_organization_from_gh(gh_org) - except GithubException: - log.info( - "Failed to fetch organization from GitHub", - organization_id=org_id, - exc_info=True, - ) - # TODO: if we lost access to the organization, - # we should remove the organization from the database, - # and clean up the members and relations. - return None - def _resync_organization_members( self, gh_org: GHOrganization, remote_org: RemoteOrganization ): From 933658cdfd2757263da784356762d3f7c014243b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 18 Feb 2025 10:47:28 -0500 Subject: [PATCH 49/92] Docstring --- readthedocs/allauth/providers/githubapp/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/allauth/providers/githubapp/provider.py b/readthedocs/allauth/providers/githubapp/provider.py index c9360018347..e39fe5e91a8 100644 --- a/readthedocs/allauth/providers/githubapp/provider.py +++ b/readthedocs/allauth/providers/githubapp/provider.py @@ -7,7 +7,7 @@ class GitHubAppProvider(GitHubProvider): """ Provider for GitHub App. - We subclass the GitHubProvider to so we have two separate providers for GitHub and GitHub App. + We subclass the GitHubProvider to have two separate providers for the GitHub OAuth App and the GitHub App. """ id = "githubapp" From 346cc34da521af7e8d49a17126443690f69e86ae Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 18 Feb 2025 10:52:08 -0500 Subject: [PATCH 50/92] Fix template --- readthedocs/templates/profiles/private/migrate-to-gh-app.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/templates/profiles/private/migrate-to-gh-app.html b/readthedocs/templates/profiles/private/migrate-to-gh-app.html index 8b8afd36025..6ad33ec5196 100644 --- a/readthedocs/templates/profiles/private/migrate-to-gh-app.html +++ b/readthedocs/templates/profiles/private/migrate-to-gh-app.html @@ -95,7 +95,7 @@ {% if not migration_target.is_admin %} title="You need to have admin access to the repository" disabled - {% endif %}"> + {% endif %}> Migrate From cac95673a1df69ac80d2b47c680a363a6e9a34e8 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 18 Feb 2025 11:28:30 -0500 Subject: [PATCH 51/92] Cleanup --- readthedocs/oauth/models.py | 5 +--- readthedocs/oauth/querysets.py | 14 +++++++++-- readthedocs/oauth/services/__init__.py | 2 -- readthedocs/oauth/services/base.py | 26 -------------------- readthedocs/oauth/services/github.py | 33 ++++---------------------- readthedocs/oauth/views.py | 7 +++--- 6 files changed, 22 insertions(+), 65 deletions(-) diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 1b28bfbd65c..85989093ae8 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -400,10 +400,7 @@ def get_remote_repository_relation(self, user, social_account): return remote_repository_relation def get_service_class(self): - from readthedocs.oauth.services import GitHubAppService, registry - - if self.github_app_installation: - return GitHubAppService + from readthedocs.oauth.services import registry for service_cls in registry: if service_cls.vcs_provider_slug == self.vcs_provider: diff --git a/readthedocs/oauth/querysets.py b/readthedocs/oauth/querysets.py index 5bb311ca74e..8e52dba3e77 100644 --- a/readthedocs/oauth/querysets.py +++ b/readthedocs/oauth/querysets.py @@ -1,9 +1,9 @@ """Managers for OAuth models.""" from django.db import models -from django.db.models import Q from readthedocs.core.querysets import NoReprQuerySet +from readthedocs.oauth.constants import GITHUB, GITHUB_APP class RelatedUserQuerySet(NoReprQuerySet, models.QuerySet): @@ -29,7 +29,17 @@ def for_project_linking(self, user): Repositories can be imported if the user has admin access to the repository on the VCS service. """ - return self.filter(remote_repository_relations__user=user, remote_repository_relations__admin=True).distinct() + queryset = self.filter( + remote_repository_relations__user=user, + remote_repository_relations__admin=True, + ) + + # If the user has already started using the GitHub App, + # we shouldn't show repositories from the old GitHub integration. + if queryset.filter(vcs_provider=GITHUB_APP).exists(): + queryset = queryset.exclude(vcs_provider=GITHUB) + + return queryset.distinct() class RemoteOrganizationQuerySet(RelatedUserQuerySet): diff --git a/readthedocs/oauth/services/__init__.py b/readthedocs/oauth/services/__init__.py index 43cc2d92733..104034000cb 100644 --- a/readthedocs/oauth/services/__init__.py +++ b/readthedocs/oauth/services/__init__.py @@ -24,6 +24,4 @@ class GitHubAppService(SettingsOverrideObject): _override_setting = "OAUTH_GITHUB_APP_SERVICE" -# NOTE: GitHubAppService should be listed after GitHubService, -# since they share the same vcs_provider_slug. registry = [GitHubService, BitbucketService, GitLabService, GitHubAppService] diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 16ec593efbd..a1bb1a3e409 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -14,11 +14,6 @@ from readthedocs.core.permissions import AdminPermission from readthedocs.oauth.clients import get_oauth2_client -from readthedocs.oauth.models import ( - GitHubAccountType, - GitHubAppInstallation, - RemoteOrganization, -) log = structlog.get_logger(__name__) @@ -248,10 +243,6 @@ def sync(self): .filter( account=self.account, remote_repository__vcs_provider=self.vcs_provider_slug, - # Skip repositories that are managed by a GH app installation. - # NOTE: this is leaking the GH app logic into the parent class, - # but this works for now. - # remote_repository__github_app_installation=None, ) .delete() ) @@ -261,16 +252,6 @@ def sync(self): o.remote_id for o in remote_organizations if o is not None ] - # account_organizations = RemoteOrganization.objects.filter( - # remote_organization_relations__account=self.account, - # vcs_provider=self.vcs_provider_slug, - # ) - # - # organizations_ids_managed_by_gh_app = GitHubAppInstallation.objects.filter( - # target_id__in=account_organizations.values_list("remote_id", flat=True), - # target_type=GitHubAccountType.ORGANIZATION, - # ).values_list("target_id", flat=True) - ( self.user.remote_organization_relations.exclude( remote_organization__remote_id__in=organization_remote_ids, @@ -280,13 +261,6 @@ def sync(self): account=self.account, remote_organization__vcs_provider=self.vcs_provider_slug, ) - .exclude( - # Skip organization that are managed by a GH app installation. - # NOTE: this is leaking the GH app logic into the parent class, - # but this works for now. - # remote_organization__remote_id__in=organizations_ids_managed_by_gh_app, - # remote_organization__vcs_provider=self.vcs_provider_slug, - ) .delete() ) diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 90d603d9d13..100aabfba03 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -14,12 +14,7 @@ from readthedocs.integrations.models import Integration from ..constants import GITHUB -from ..models import ( - GitHubAccountType, - GitHubAppInstallation, - RemoteOrganization, - RemoteRepository, -) +from ..models import RemoteOrganization, RemoteRepository from .base import SyncServiceError, UserService log = structlog.get_logger(__name__) @@ -67,9 +62,6 @@ def sync_organizations(self): org_details, create_user_relationship=True, ) - if not remote_organization: - continue - remote_organizations.append(remote_organization) org_url = org["url"] @@ -113,12 +105,6 @@ def create_repository(self, fields, privacy=None): vcs_provider=self.vcs_provider_slug, ) - # if repo.github_app_installation: - # log.info( - # "Repository is managed by a GitHub App installation, skipping.", - # ) - # return - # TODO: For debugging: https://github.com/readthedocs/readthedocs.org/pull/9449. if created: _old_remote_repository = RemoteRepository.objects.filter( @@ -198,17 +184,8 @@ def create_organization(self, fields, create_user_relationship=False): will be created/updated. :rtype: RemoteOrganization """ - remote_id = fields["id"] - # if GitHubAppInstallation.objects.filter( - # target_id=remote_id, target_type=GitHubAccountType.ORGANIZATION - # ).exists(): - # log.info( - # "Organization is managed by a GitHub App installation, skipping.", - # ) - # return - organization, _ = RemoteOrganization.objects.get_or_create( - remote_id=str(remote_id), + remote_id=str(fields["id"]), vcs_provider=self.vcs_provider_slug, ) if create_user_relationship: @@ -430,7 +407,6 @@ def update_webhook(self, project, integration): return (False, resp) def remove_webhook(self, project): - # TODO: use remote repo instead? owner, repo = build_utils.get_github_username_repo(url=project.repo) try: @@ -454,7 +430,9 @@ def remove_webhook(self, project): for hook_target in hook_targets: if hook_url.startswith(hook_target): try: - self.session.delete(f"{self.base_api_url}/repos/{owner}/{repo}/hooks/{hook['id']}").raise_for_status() + self.session.delete( + f"{self.base_api_url}/repos/{owner}/{repo}/hooks/{hook['id']}" + ).raise_for_status() except Exception: log.info("Failed to remove GitHub webhook for project.") return False @@ -464,7 +442,6 @@ def remove_ssh_key(self, project): # Overridden in corporate return True - def send_build_status(self, *, build, commit, status): """ Create GitHub commit status for project. diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 2da6aa85723..af59813a272 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -1,4 +1,5 @@ import hmac +from functools import cached_property import structlog from django.conf import settings @@ -30,9 +31,9 @@ class GitHubAppWebhookView(APIView): authentication_classes = [] - def __init__(self, **kwargs): - self.gha_client = get_gh_app_client() - super().__init__(**kwargs) + @cached_property + def gha_client(self): + return get_gh_app_client() def post(self, request): if not self._is_payload_signature_valid(): From db1950f4baecf2c6c3c71ce0b12c23cb031e787a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 18 Feb 2025 14:39:55 -0500 Subject: [PATCH 52/92] More updates --- readthedocs/oauth/models.py | 59 +++++++++++++++++++++---- readthedocs/oauth/services/githubapp.py | 25 ++++++----- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 85989093ae8..cdd70bf1029 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -14,7 +14,7 @@ from readthedocs.projects.constants import REPO_CHOICES from readthedocs.projects.models import Project -from .constants import GITHUB, VCS_PROVIDER_CHOICES +from .constants import GITHUB_APP, VCS_PROVIDER_CHOICES from .querysets import RemoteOrganizationQuerySet, RemoteRepositoryQuerySet log = structlog.get_logger(__name__) @@ -111,6 +111,10 @@ def delete_orphaned_repositories(self, repository_ids: list[int] | None = None): that are not linked to a project. This is in case the user re-installs the app, they shouldn't need to manually link each project to the repository again. + All the repository relations are deleted as well, + since we want to keep the remote repository linked to the project, + but not to users. + We also remove organizations that don't have any repositories after removing the repositories. :param repository_ids: List of repository ids (remote ID) to delete. @@ -122,11 +126,11 @@ def delete_orphaned_repositories(self, repository_ids: list[int] | None = None): remote_organizations = RemoteOrganization.objects.filter( repositories__github_app_installation=self, - vcs_provider=GITHUB, + vcs_provider=GITHUB_APP, ) remote_repositories = self.repositories.filter( projects=None, - vcs_provider=GITHUB, + vcs_provider=GITHUB_APP, ) if repository_ids: remote_organizations = remote_organizations.filter( @@ -148,9 +152,18 @@ def delete_orphaned_repositories(self, repository_ids: list[int] | None = None): deleted=deleted, installation_id=self.installation_id, ) - # TODO: we should probably remove the user relation as well, - # since we want to keep the remote repository linked to the project, - # but not to the user. + + count, deleted = RemoteRepositoryRelation.objects.filter( + remote_repository__id__in=repository_ids, + remote_repository__vcs_provider=GITHUB_APP, + remote_reposotory__github_app_installation=self, + ).delete() + log.info( + "Deleted repository relations", + count=count, + deleted=deleted, + installation_id=self.installation_id, + ) count, deleted = RemoteOrganization.objects.filter( id__in=remote_organizations_ids, @@ -164,10 +177,16 @@ def delete_orphaned_repositories(self, repository_ids: list[int] | None = None): ) def delete_orphaned_organization(self, organization_id: int): - """Delete an organization and all its repositories from the database only if they are not linked to a project.""" + """ + Delete an organization and all its repositories from the database only if they are not linked to a project. + + All the organization and repository relations are deleted as well, + since we want to keep the remote repository linked to the project, + but not to users. + """ count, deleted = RemoteOrganization.objects.filter( remote_id=str(organization_id), - vcs_provider=GITHUB, + vcs_provider=GITHUB_APP, repositories__projects=None, ).delete() log.info( @@ -178,6 +197,30 @@ def delete_orphaned_organization(self, organization_id: int): installation_id=self.installation_id, ) + count, deleted = RemoteOrganizationRelation.objects.filter( + remote_organization__remote_id=str(organization_id), + remote_organization__vcs_provider=GITHUB_APP, + ).delete() + log.info( + "Deleted organization relations", + count=count, + deleted=deleted, + organization_id=organization_id, + installation_id=self.installation_id, + ) + + count, deleted = RemoteRepositoryRelation.objects.filter( + remote_repository__organization__remote_id=str(organization_id), + remote_repository__vcs_provider=GITHUB_APP, + ).delete() + log.info( + "Deleted repository relations", + count=count, + deleted=deleted, + organization_id=organization_id, + installation_id=self.installation_id, + ) + class RemoteOrganization(TimeStampedModel): """ diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 0cec2a587d1..7f945a24573 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -136,23 +136,22 @@ def sync(self): If a remote organization doesn't have any repositories after removing the repositories, we remove the organization from the database. """ - remote_repositories = [] try: - for repo in self.app_installation.get_repos(): - remote_repo = self._create_or_update_repository_from_gh(repo) - if remote_repo: - remote_repositories.append(remote_repo) + app_installation = self.app_installation except GithubException: - # TODO: if we lost access to the installations, - # we should remove the installation from the database, - # and clean up the repositories, organizations, and relations. log.info( - "Failed to sync repositories for installation", + "Failed to get installation", installation_id=self.installation.installation_id, exc_info=True, ) raise SyncServiceError() + remote_repositories = [] + for repo in app_installation.get_repos(): + remote_repo = self._create_or_update_repository_from_gh(repo) + if remote_repo: + remote_repositories.append(remote_repo) + repos_to_delete = self.installation.repositories.exclude( pk__in=[repo.pk for repo in remote_repositories], ).values_list("remote_id", flat=True) @@ -163,15 +162,17 @@ def update_or_create_repositories(self, repository_ids: list[int]): for repository_id in repository_ids: try: repo = self.installation_client.get_repo(repository_id) - except GithubException: + except GithubException as e: log.info( "Failed to fetch repository from GitHub", repository_id=repository_id, exc_info=True, ) - # TODO: if we lost access to the repository, - # we should remove the repository from the database, + # if we lost access to the repository, + # we remove the repository from the database, # and clean up the collaborators and relations. + if e.status == 404: + self.installation.delete_orphaned_repositories([repository_id]) continue self._create_or_update_repository_from_gh(repo) From 4b84536ee5fb983fa7cc1f3d390a107240c8abf7 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 19 Feb 2025 14:07:13 -0500 Subject: [PATCH 53/92] Check for githuapp --- readthedocs/projects/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 2cc9523f67e..49cd69f38ea 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1058,9 +1058,12 @@ def get_git_service_class(self, fallback_to_clone_url=False): @property def is_github_project(self): - from readthedocs.oauth.services import GitHubService + from readthedocs.oauth.services import GitHubAppService, GitHubService - return self.get_git_service_class(fallback_to_clone_url=True) == GitHubService + return self.get_git_service_class(fallback_to_clone_url=True) in [ + GitHubService, + GitHubAppService, + ] @property def is_gitlab_project(self): From 3899104c6280b094c6fbc92315dfe9f189dc3198 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 19 Feb 2025 17:21:49 -0500 Subject: [PATCH 54/92] Uninstalling the app from a repository, means unlinking it from the project --- .../oauth/migrations/0017_githubapp.py | 2 +- readthedocs/oauth/models.py | 79 +++---------------- readthedocs/oauth/views.py | 6 +- readthedocs/profiles/views.py | 21 ++++- 4 files changed, 37 insertions(+), 71 deletions(-) diff --git a/readthedocs/oauth/migrations/0017_githubapp.py b/readthedocs/oauth/migrations/0017_githubapp.py index 9f4300c8223..c8a522a5123 100644 --- a/readthedocs/oauth/migrations/0017_githubapp.py +++ b/readthedocs/oauth/migrations/0017_githubapp.py @@ -34,6 +34,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='remoterepository', name='github_app_installation', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='repositories', to='oauth.githubappinstallation', verbose_name='GitHub App Installation'), + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='repositories', to='oauth.githubappinstallation', verbose_name='GitHub App Installation'), ), ] diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index cdd70bf1029..7847c9aa785 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -99,21 +99,16 @@ def service(self): return GitHubAppService(self) def delete(self, *args, **kwargs): - """Override delete method to remove orphaned linked repositories.""" - self.delete_orphaned_repositories() + """Override delete method to remove orphaned organizations.""" + self.delete_repositories() return super().delete(*args, **kwargs) - def delete_orphaned_repositories(self, repository_ids: list[int] | None = None): + def delete_repositories(self, repository_ids: list[int] | None = None): """ - Delete orphaned repositories linked to this installation. + Delete repositories linked to this installation. When an installation is deleted, we delete all its remote repositories - that are not linked to a project. This is in case the user re-installs the app, - they shouldn't need to manually link each project to the repository again. - - All the repository relations are deleted as well, - since we want to keep the remote repository linked to the project, - but not to users. + and relations, users will need to manually link the projects to each repository again. We also remove organizations that don't have any repositories after removing the repositories. @@ -128,10 +123,7 @@ def delete_orphaned_repositories(self, repository_ids: list[int] | None = None): repositories__github_app_installation=self, vcs_provider=GITHUB_APP, ) - remote_repositories = self.repositories.filter( - projects=None, - vcs_provider=GITHUB_APP, - ) + remote_repositories = self.repositories.filter(vcs_provider=GITHUB_APP) if repository_ids: remote_organizations = remote_organizations.filter( repositories__remote_id__in=repository_ids @@ -147,19 +139,7 @@ def delete_orphaned_repositories(self, repository_ids: list[int] | None = None): count, deleted = remote_repositories.delete() log.info( - "Deleted repositories without projects", - count=count, - deleted=deleted, - installation_id=self.installation_id, - ) - - count, deleted = RemoteRepositoryRelation.objects.filter( - remote_repository__id__in=repository_ids, - remote_repository__vcs_provider=GITHUB_APP, - remote_reposotory__github_app_installation=self, - ).delete() - log.info( - "Deleted repository relations", + "Deleted repositories projects", count=count, deleted=deleted, installation_id=self.installation_id, @@ -176,45 +156,14 @@ def delete_orphaned_repositories(self, repository_ids: list[int] | None = None): installation_id=self.installation_id, ) - def delete_orphaned_organization(self, organization_id: int): - """ - Delete an organization and all its repositories from the database only if they are not linked to a project. - - All the organization and repository relations are deleted as well, - since we want to keep the remote repository linked to the project, - but not to users. - """ + def delete_organization(self, organization_id: int): + """Delete an organization and all its repositories and relations from the database.""" count, deleted = RemoteOrganization.objects.filter( remote_id=str(organization_id), vcs_provider=GITHUB_APP, - repositories__projects=None, - ).delete() - log.info( - "Deleted orphaned organization", - count=count, - deleted=deleted, - organization_id=organization_id, - installation_id=self.installation_id, - ) - - count, deleted = RemoteOrganizationRelation.objects.filter( - remote_organization__remote_id=str(organization_id), - remote_organization__vcs_provider=GITHUB_APP, ).delete() log.info( - "Deleted organization relations", - count=count, - deleted=deleted, - organization_id=organization_id, - installation_id=self.installation_id, - ) - - count, deleted = RemoteRepositoryRelation.objects.filter( - remote_repository__organization__remote_id=str(organization_id), - remote_repository__vcs_provider=GITHUB_APP, - ).delete() - log.info( - "Deleted repository relations", + "Deleted organization", count=count, deleted=deleted, organization_id=organization_id, @@ -385,11 +334,9 @@ class RemoteRepository(TimeStampedModel): related_name="repositories", null=True, blank=True, - # When an installation is deleted, we don't delete the repository - # if it's linked to a project. This is in case the user re-installs the app, - # they shouldn't need to manually link each project to the repository again. - # NOTE: I also see how this may be unexpected behavior in some cases. - on_delete=models.SET_NULL, + # When an installation is deleted, we delete all its remote repositories + # and relations, users will need to manually link the projects to each repository again. + on_delete=models.CASCADE, ) objects = RemoteRepositoryQuerySet.as_manager() diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index af59813a272..08406a06669 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -153,7 +153,7 @@ def _handle_installation_event(self): if action == "deleted": # NOTE: When an installation is deleted, this doesn't trigger an installation_repositories event. - # So we need to call the delete method explicitly here, so we delete orphan remote repositories. + # So we need to call the delete method explicitly here, so we delete its repositories. installation = GitHubAppInstallation.objects.filter( installation_id=installation_id ).first() @@ -253,7 +253,7 @@ def _handle_installation_repositories_event(self): return if action == "removed": - installation.delete_orphaned_repositories( + installation.delete_repositories( [repo["id"] for repo in data["repositories_removed"]] ) return @@ -438,7 +438,7 @@ def _handle_organization_event(self): # when the organization is deleted. # I didn't see GH send the deleted action for the organization event... # But handle it just in case. - installation.delete_orphaned_organization(data["organization"]["id"]) + installation.delete_organization(data["organization"]["id"]) return # Ignore other actions: diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index 67405af9265..a737e874288 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -334,7 +334,26 @@ def post(self, request, *args, **kwargs): has_errors = False for project in projects: try: - migrate_project_to_github_app(project=project, user=request.user) + result = migrate_project_to_github_app(project=project, user=request.user) + if not result.webhook_removed: + messages.warning( + request, + _( + "The webhook from the old GitHub integration " + "was not removed for project {project}. " + "Please remove it manually." + ).format(project=project.slug), + ) + + if not result.ssh_key_removed: + messages.warning( + request, + _( + "The SSH key from the old GitHub integration " + "was not removed for project {project}. " + "Please remove it manually." + ).format(project=project.slug), + ) except Exception as e: has_errors = True messages.error(request, f"Error migrating project {project.slug}: {e}") From acfda4c896642f8f3bd69a563d7d00814c0c18e6 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 19 Feb 2025 17:24:15 -0500 Subject: [PATCH 55/92] Format --- readthedocs/profiles/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index a737e874288..60c7dbb98d2 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -334,7 +334,9 @@ def post(self, request, *args, **kwargs): has_errors = False for project in projects: try: - result = migrate_project_to_github_app(project=project, user=request.user) + result = migrate_project_to_github_app( + project=project, user=request.user + ) if not result.webhook_removed: messages.warning( request, From d1e3cb47449f80bb157eea47d2c67bcf8270bb7b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 20 Feb 2025 15:58:29 -0500 Subject: [PATCH 56/92] Add a sync repositories permissions method --- .../commands/reconnect_remoterepositories.py | 9 +++--- readthedocs/oauth/services/base.py | 21 ++++++++++++-- readthedocs/oauth/services/githubapp.py | 29 +++++++++++++++++-- readthedocs/oauth/tasks.py | 10 +++---- 4 files changed, 55 insertions(+), 14 deletions(-) diff --git a/readthedocs/oauth/management/commands/reconnect_remoterepositories.py b/readthedocs/oauth/management/commands/reconnect_remoterepositories.py index b8bd9c523aa..df45191f2dd 100644 --- a/readthedocs/oauth/management/commands/reconnect_remoterepositories.py +++ b/readthedocs/oauth/management/commands/reconnect_remoterepositories.py @@ -41,11 +41,10 @@ def add_arguments(self, parser): def _force_owners_social_resync(self, organization): for owner in organization.owners.all(): for service_cls in registry: - for service in service_cls.for_user(owner): - try: - service.sync() - except SyncServiceError: - print(f"Service {service} failed while syncing. Skipping...") + try: + service_cls.sync_user_access(owner) + except SyncServiceError: + print(f"Service {service_cls.allauth_provider.name} failed while syncing. Skipping...") def _connect_repositories(self, organization, no_dry_run, only_owners): connected_projects = [] diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index a1bb1a3e409..2d2ded5241b 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -38,15 +38,20 @@ class Service: supports_build_status = False @classmethod - def for_project(self, project): + def for_project(cls, project): """Return an iterator of services that can be used for the project.""" raise NotImplementedError @classmethod - def for_user(self, user): + def for_user(cls, user): """Return an iterator of services that belong to the user.""" raise NotImplementedError + @classmethod + def sync_user_access(cls, user): + """Sync the user's access to the provider repositories and organizations.""" + raise NotImplementedError + def sync(self): """ Sync remote repositories and organizations. @@ -152,6 +157,18 @@ def for_user(cls, user): for account in accounts: yield cls(user=user, account=account) + @classmethod + def sync_user_access(cls, user): + """ + Sync the user's access to the provider repositories and organizations. + + Since UserService makes use of the user's OAuth token, + we can just sync the user's repositories in order to + update the user access to repositories and organizations. + """ + for service in cls.for_user(user): + service.sync() + @cached_property def session(self): return get_oauth2_client(self.account) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 7f945a24573..2bbec5329be 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -126,6 +126,31 @@ def for_user(cls, user): if installation: yield cls(installation) + @classmethod + def sync_user_access(cls, user): + """ + Sync the user's access to the provider repositories and organizations. + + Since we are using a GitHub App, we don't have a way to check all the repositories and organizations + the user has access to or if it lost access to a repository or organization. + + Our webhooks should keep permissions in sync, but just in case, + we first sync the repositories from all installations accessible to the user (refresh access to new repositories), + and then we sync each repository the user has access to (check if the user lost access to a repository, or his access level changed). + """ + # Refresh access to all installations accessible to the user. + for service in cls.for_user(user): + service.sync() + + # Update the access to each repository the user has access to. + queryset = RemoteRepository.objects.filter( + remote_repository_relations__user=user, + vcs_provider=cls.vcs_provider_slug, + ) + for repository in queryset: + service = cls(repository.github_app_installation) + service.update_or_create_repositories([repository.remote_id]) + def sync(self): """ Sync all repositories and organizations that are accessible to the installation. @@ -155,7 +180,7 @@ def sync(self): repos_to_delete = self.installation.repositories.exclude( pk__in=[repo.pk for repo in remote_repositories], ).values_list("remote_id", flat=True) - self.installation.delete_orphaned_repositories(repos_to_delete) + self.installation.delete_repositories(repos_to_delete) def update_or_create_repositories(self, repository_ids: list[int]): """Update or create repositories from the given list of repository IDs.""" @@ -172,7 +197,7 @@ def update_or_create_repositories(self, repository_ids: list[int]): # we remove the repository from the database, # and clean up the collaborators and relations. if e.status == 404: - self.installation.delete_orphaned_repositories([repository_id]) + self.installation.delete_repositories([repository_id]) continue self._create_or_update_repository_from_gh(repo) diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index cff086804d4..597cae9d025 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -45,11 +45,11 @@ def sync_remote_repositories(user_id): failed_services = set() for service_cls in registry: - for service in service_cls.for_user(user): - try: - service.sync() - except SyncServiceError: - failed_services.add(service_cls.allauth_provider.name) + try: + service_cls.sync_user_access(user) + except SyncServiceError: + failed_services.add(service_cls.allauth_provider.name) + if failed_services: raise SyncServiceError( SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format( From b3ad1a278bedde4b0dd1e3061e05c66960b7e636 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 25 Feb 2025 17:07:43 -0500 Subject: [PATCH 57/92] Add tests for webhook --- readthedocs/core/views/hooks.py | 2 +- readthedocs/oauth/services/__init__.py | 16 +- .../oauth/tests/test_githubapp_webhook.py | 775 ++++++++++++++++++ readthedocs/oauth/views.py | 20 +- readthedocs/settings/test.py | 2 + 5 files changed, 800 insertions(+), 15 deletions(-) create mode 100644 readthedocs/oauth/tests/test_githubapp_webhook.py diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 2621c89304d..47a13c794d4 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -184,6 +184,7 @@ def get_or_create_external_version(project, version_data): external_version.identifier = version_data.commit # If the PR was previously closed it was marked as closed external_version.state = EXTERNAL_VERSION_STATE_OPEN + external_version.active = True external_version.save() log.info( "External version updated.", @@ -209,7 +210,6 @@ def close_external_version(project, version_data): project.versions(manager=EXTERNAL) .filter( verbose_name=version_data.id, - identifier=version_data.commit, ) .first() ) diff --git a/readthedocs/oauth/services/__init__.py b/readthedocs/oauth/services/__init__.py index 104034000cb..84c1d385d72 100644 --- a/readthedocs/oauth/services/__init__.py +++ b/readthedocs/oauth/services/__init__.py @@ -1,7 +1,16 @@ """Conditional classes for OAuth services.""" from readthedocs.core.utils.extend import SettingsOverrideObject -from readthedocs.oauth.services import bitbucket, github, githubapp, gitlab +from readthedocs.oauth.services import bitbucket, github, gitlab +from readthedocs.oauth.services.githubapp import GitHubAppService + +__all__ = [ + "GitHubService", + "BitbucketService", + "GitLabService", + "GitHubAppService", + "registry", +] class GitHubService(SettingsOverrideObject): @@ -19,9 +28,4 @@ class GitLabService(SettingsOverrideObject): _override_setting = "OAUTH_GITLAB_SERVICE" -class GitHubAppService(SettingsOverrideObject): - _default_class = githubapp.GitHubAppService - _override_setting = "OAUTH_GITHUB_APP_SERVICE" - - registry = [GitHubService, BitbucketService, GitLabService, GitHubAppService] diff --git a/readthedocs/oauth/tests/test_githubapp_webhook.py b/readthedocs/oauth/tests/test_githubapp_webhook.py new file mode 100644 index 00000000000..d074d3673ff --- /dev/null +++ b/readthedocs/oauth/tests/test_githubapp_webhook.py @@ -0,0 +1,775 @@ +import json +from unittest import mock + +from allauth.socialaccount.models import SocialAccount +from django.conf import settings +from django.contrib.auth.models import User +from django.test import TestCase +from django.urls import reverse +from django_dynamic_fixture import get + +from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider +from readthedocs.api.v2.views.integrations import ( + GITHUB_EVENT_HEADER, + GITHUB_SIGNATURE_HEADER, + WebhookMixin, +) +from readthedocs.builds.constants import ( + BRANCH, + EXTERNAL, + EXTERNAL_VERSION_STATE_CLOSED, + EXTERNAL_VERSION_STATE_OPEN, + LATEST, + TAG, +) +from readthedocs.builds.models import Version +from readthedocs.oauth.models import ( + GitHubAccountType, + GitHubAppInstallation, + RemoteOrganization, + RemoteRepository, +) +from readthedocs.oauth.services import GitHubAppService +from readthedocs.projects.models import Project + + +def get_signature(payload): + if not isinstance(payload, str): + payload = json.dumps(payload) + return "sha256=" + WebhookMixin.get_digest( + secret=settings.GITHUB_APP_WEBHOOK_SECRET, + msg=payload, + ) + + +class TestGitHubAppWebhook(TestCase): + def setUp(self): + self.user = get(User) + self.project = get(Project, users=[self.user], default_branch="main") + self.version_latest = self.project.versions.get(slug=LATEST) + self.version = get( + Version, project=self.project, verbose_name="1.0", type=BRANCH, active=True + ) + self.version_main = get( + Version, project=self.project, verbose_name="main", type=BRANCH, active=True + ) + self.version_tag = get( + Version, project=self.project, verbose_name="2.0", type=TAG, active=True + ) + self.socialaccount = get( + SocialAccount, user=self.user, provider=GitHubAppProvider.id + ) + self.installation = get( + GitHubAppInstallation, + installation_id=1111, + target_id=1111, + target_type=GitHubAccountType.USER, + ) + + self.remote_repository = get( + RemoteRepository, + remote_id="1234", + name="repo", + full_name="user/repo", + vcs_provider=GitHubAppProvider.id, + github_app_installation=self.installation, + ) + self.project.remote_repository = self.remote_repository + self.project.save() + self.url = reverse("github_app_webhook") + + def post_webhook(self, event, payload): + headers = { + GITHUB_EVENT_HEADER: event, + GITHUB_SIGNATURE_HEADER: get_signature(payload), + } + return self.client.post( + self.url, data=payload, content_type="application/json", headers=headers + ) + + @mock.patch.object(GitHubAppService, "sync") + def test_installation_created(self, sync): + new_installation_id = 2222 + assert not GitHubAppInstallation.objects.filter( + installation_id=new_installation_id + ).exists() + payload = { + "action": "created", + "installation": { + "id": new_installation_id, + "target_id": 2222, + "target_type": GitHubAccountType.USER, + }, + } + r = self.post_webhook("installation", payload) + assert r.status_code == 200 + + installation = GitHubAppInstallation.objects.get( + installation_id=new_installation_id + ) + assert installation.target_id == 2222 + assert installation.target_type == GitHubAccountType.USER + sync.assert_called_once() + + @mock.patch.object(GitHubAppService, "sync") + def test_installation_created_with_existing_installation(self, sync): + paylod = { + "action": "created", + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + } + r = self.post_webhook("installation", paylod) + assert r.status_code == 200 + sync.assert_called_once() + self.installation.refresh_from_db() + assert self.installation.target_id == 1111 + assert self.installation.target_type == GitHubAccountType.USER + assert GitHubAppInstallation.objects.count() == 1 + + def test_installation_deleted(self): + payload = { + "action": "deleted", + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + } + r = self.post_webhook("installation", payload) + assert r.status_code == 200 + assert not GitHubAppInstallation.objects.filter( + installation_id=self.installation.installation_id + ).exists() + + def test_installation_deleted_with_non_existing_installation(self): + install_id = 2222 + assert not GitHubAppInstallation.objects.filter( + installation_id=install_id + ).exists() + payload = { + "action": "deleted", + "installation": { + "id": install_id, + "target_id": 2222, + "target_type": GitHubAccountType.USER, + }, + } + r = self.post_webhook("installation", payload) + assert r.status_code == 200 + assert not GitHubAppInstallation.objects.filter(installation_id=2222).exists() + + def test_installation_new_permissions_accepted(self): + payload = { + "action": "new_permissions_accepted", + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + } + r = self.post_webhook("installation", payload) + assert r.status_code == 200 + + @mock.patch.object(GitHubAppService, "update_or_create_repositories") + def test_installation_repositories_added(self, update_or_create_repositories): + payload = { + "action": "added", + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "repository_selection": "selected", + "repositories_added": [ + { + "id": 1234, + "name": "repo1", + "full_name": "user/repo1", + "private": False, + }, + { + "id": 5678, + "name": "repo2", + "full_name": "user/repo2", + "private": True, + }, + ], + } + r = self.post_webhook("installation_repositories", payload) + assert r.status_code == 200 + update_or_create_repositories.assert_called_once_with([1234, 5678]) + + @mock.patch.object(GitHubAppService, "sync") + def test_installation_repositories_added_all(self, sync): + payload = { + "action": "added", + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "repository_selection": "all", + } + r = self.post_webhook("installation_repositories", payload) + assert r.status_code == 200 + sync.assert_called_once() + + def test_installation_repositories_removed(self): + assert self.installation.repositories.count() == 1 + payload = { + "action": "removed", + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "repository_selection": "selected", + "repositories_removed": [ + { + "id": 1234, + "name": "repo1", + "full_name": "user/repo1", + "private": False, + }, + { + "id": 5678, + "name": "repo2", + "full_name": "user/repo2", + "private": True, + }, + ], + } + r = self.post_webhook("installation_repositories", payload) + assert r.status_code == 200 + assert self.installation.repositories.count() == 0 + + @mock.patch.object(GitHubAppService, "sync") + def test_installation_target(self, sync): + payload = { + "action": "renamed", + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + } + r = self.post_webhook("installation_target", payload) + assert r.status_code == 200 + sync.assert_called_once() + + @mock.patch("readthedocs.core.views.hooks.sync_repository_task") + def test_push_branch_created(self, sync_repository_task): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "created": True, + "deleted": False, + "ref": "refs/heads/branch", + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("push", payload) + assert r.status_code == 200 + sync_repository_task.apply_async.assert_called_once_with( + args=[self.version_latest.pk], + kwargs={"build_api_key": mock.ANY}, + ) + + @mock.patch("readthedocs.core.views.hooks.sync_repository_task") + def test_push_tag_created(self, sync_repository_task): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "created": True, + "deleted": False, + "ref": "refs/tags/tag", + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("push", payload) + assert r.status_code == 200 + sync_repository_task.apply_async.assert_called_once_with( + args=[self.version_latest.pk], + kwargs={"build_api_key": mock.ANY}, + ) + + @mock.patch("readthedocs.core.views.hooks.sync_repository_task") + def test_push_branch_deleted(self, sync_repository_task): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "created": False, + "deleted": True, + "ref": "refs/heads/branch", + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("push", payload) + assert r.status_code == 200 + sync_repository_task.apply_async.assert_called_once_with( + args=[self.version_latest.pk], + kwargs={"build_api_key": mock.ANY}, + ) + + @mock.patch("readthedocs.core.views.hooks.sync_repository_task") + def test_push_tag_deleted(self, sync_repository_task): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "created": False, + "deleted": True, + "ref": "refs/tags/tag", + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("push", payload) + assert r.status_code == 200 + sync_repository_task.apply_async.assert_called_once_with( + args=[self.version_latest.pk], + kwargs={"build_api_key": mock.ANY}, + ) + + @mock.patch("readthedocs.core.views.hooks.trigger_build") + def test_push_branch(self, trigger_build): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "created": False, + "deleted": False, + "ref": "refs/heads/main", + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("push", payload) + assert r.status_code == 200 + trigger_build.assert_has_calls( + [ + mock.call(project=self.project, version=self.version_main), + mock.call(project=self.project, version=self.version_latest), + ] + ) + + @mock.patch("readthedocs.core.views.hooks.trigger_build") + def test_push_tag(self, trigger_build): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "created": False, + "deleted": False, + "ref": "refs/tags/2.0", + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("push", payload) + assert r.status_code == 200 + trigger_build.assert_called_once_with( + project=self.project, + version=self.version_tag, + ) + + @mock.patch("readthedocs.core.views.hooks.trigger_build") + def test_pull_request_opened(self, trigger_build): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "opened", + "pull_request": { + "number": 1, + "head": { + "ref": "new-feature", + "sha": "1234abcd", + }, + "base": { + "ref": "main", + }, + }, + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("pull_request", payload) + assert r.status_code == 200 + external_version = self.project.versions.get(verbose_name="1", type=EXTERNAL) + assert external_version.identifier == "1234abcd" + assert external_version.state == EXTERNAL_VERSION_STATE_OPEN + assert external_version.active + trigger_build.assert_called_once_with( + project=self.project, + version=external_version, + commit=external_version.identifier, + ) + + @mock.patch("readthedocs.core.views.hooks.trigger_build") + def test_pull_request_reopened(self, trigger_build): + external_version = get( + Version, + project=self.project, + verbose_name="1", + type=EXTERNAL, + active=True, + identifier="1234changeme", + ) + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "reopened", + "pull_request": { + "number": 1, + "head": { + "ref": "new-feature", + "sha": "1234abcd", + }, + "base": { + "ref": "main", + }, + }, + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("pull_request", payload) + assert r.status_code == 200 + + external_version.refresh_from_db() + assert external_version.identifier == "1234abcd" + assert external_version.state == EXTERNAL_VERSION_STATE_OPEN + assert external_version.active + trigger_build.assert_called_once_with( + project=self.project, + version=external_version, + commit=external_version.identifier, + ) + + @mock.patch("readthedocs.core.views.hooks.trigger_build") + def test_pull_request_synchronize(self, trigger_build): + external_version = get( + Version, + project=self.project, + verbose_name="1", + type=EXTERNAL, + active=True, + identifier="1234changeme", + ) + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "synchronize", + "pull_request": { + "number": 1, + "head": { + "ref": "new-feature", + "sha": "1234abcd", + }, + "base": { + "ref": "main", + }, + }, + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("pull_request", payload) + assert r.status_code == 200 + + external_version.refresh_from_db() + assert external_version.identifier == "1234abcd" + assert external_version.state == EXTERNAL_VERSION_STATE_OPEN + assert external_version.active + trigger_build.assert_called_once_with( + project=self.project, + version=external_version, + commit=external_version.identifier, + ) + + def test_pull_request_closed(self): + external_version = get( + Version, + project=self.project, + verbose_name="1", + type=EXTERNAL, + active=True, + identifier="1234abcd", + ) + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "closed", + "pull_request": { + "number": 1, + "head": { + "ref": "new-feature", + "sha": "1234abcd", + }, + "base": { + "ref": "main", + }, + }, + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("pull_request", payload) + assert r.status_code == 200 + + external_version.refresh_from_db() + assert external_version.identifier == "1234abcd" + assert external_version.state == EXTERNAL_VERSION_STATE_CLOSED + assert external_version.active + + def test_pull_request_edited(self): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "edited", + "pull_request": { + "number": 1, + "head": { + "ref": "new-feature", + "sha": "1234abcd", + }, + "base": { + "ref": "main", + }, + }, + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("pull_request", payload) + assert r.status_code == 200 + + @mock.patch.object(GitHubAppService, "update_or_create_repositories") + def test_repository_edited(self, update_or_create_repositories): + actions = ["edited", "renamed", "transferred", "privatized", "publicized"] + for action in actions: + update_or_create_repositories.reset_mock() + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": action, + "changes": {}, + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("repository", payload) + assert r.status_code == 200 + update_or_create_repositories.assert_called_once_with( + [self.remote_repository.remote_id] + ) + + def test_repository_created(self): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "created", + "repository": { + "id": 5678, + "full_name": "user/repo2", + }, + } + r = self.post_webhook("repository", payload) + assert r.status_code == 200 + + @mock.patch.object(GitHubAppService, "sync") + def test_organization_member_added(self, sync): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "member_added", + } + r = self.post_webhook("organization", payload) + assert r.status_code == 200 + sync.assert_called_once() + + @mock.patch.object(GitHubAppService, "sync") + def test_organization_member_removed(self, sync): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "member_removed", + } + r = self.post_webhook("organization", payload) + assert r.status_code == 200 + sync.assert_called_once() + + @mock.patch.object(GitHubAppService, "sync") + def test_organization_renamed(self, sync): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "renamed", + } + r = self.post_webhook("organization", payload) + assert r.status_code == 200 + sync.assert_called_once() + + def test_organization_deleted(self): + organization_id = 1234 + get( + RemoteOrganization, + remote_id=str(organization_id), + name="org", + vcs_provider=GitHubAppProvider.id, + ) + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "deleted", + "organization": { + "id": organization_id, + "login": "org", + }, + } + r = self.post_webhook("organization", payload) + assert r.status_code == 200 + assert not RemoteOrganization.objects.filter( + remote_id=str(organization_id) + ).exists() + + def test_organization_member_invited(self): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "member_invited", + } + r = self.post_webhook("organization", payload) + assert r.status_code == 200 + + @mock.patch.object(GitHubAppService, "update_or_create_repositories") + def test_member_added(self, update_or_create_repositories): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "added", + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("member", payload) + assert r.status_code == 200 + update_or_create_repositories.assert_called_once_with( + [self.remote_repository.remote_id] + ) + + @mock.patch.object(GitHubAppService, "update_or_create_repositories") + def test_member_edited(self, update_or_create_repositories): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "edited", + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("member", payload) + assert r.status_code == 200 + update_or_create_repositories.assert_called_once_with( + [self.remote_repository.remote_id] + ) + + @mock.patch.object(GitHubAppService, "update_or_create_repositories") + def test_member_removed(self, update_or_create_repositories): + payload = { + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + "action": "removed", + "repository": { + "id": self.remote_repository.remote_id, + "full_name": self.remote_repository.full_name, + }, + } + r = self.post_webhook("member", payload) + assert r.status_code == 200 + update_or_create_repositories.assert_called_once_with( + [self.remote_repository.remote_id] + ) + + def test_github_app_authorization(self): + pass diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 08406a06669..bef3edd83d6 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -146,10 +146,11 @@ def _handle_installation_event(self): installation_id = gh_installation["id"] if action == "created": - _, created = self._get_or_create_installation() - if not created: - log.info("Installation already exists") - return + installation, created = self._get_or_create_installation() + # If the installation was just created, we already synced the repositories. + if created: + return + installation.service.sync() if action == "deleted": # NOTE: When an installation is deleted, this doesn't trigger an installation_repositories event. @@ -275,6 +276,8 @@ def _handle_installation_target_event(self): (maybe a bug?). When this happens, we re-sync all the repositories, so they use the new name. + + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#installation_target. """ installation, created = self._get_or_create_installation() @@ -301,7 +304,7 @@ def _handle_repository_event(self): if created: return - if action in ("edited", "privatized", "publicized", "renamed", "trasferred"): + if action in ("edited", "privatized", "publicized", "renamed", "transferred"): installation.service.update_or_create_repositories( [data["repository"]["id"]] ) @@ -406,7 +409,7 @@ def _handle_organization_event(self): Triggered when an member is added or removed from an organization, or when the organization is renamed or deleted. - See https://docs.github.com/en/webhooks/webhook-events-and-payloads#organization + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#organization. """ data = self.request.data action = data["action"] @@ -450,7 +453,7 @@ def _handle_member_event(self): Triggered when a user is added or removed from a repository. - See https://docs.github.com/en/webhooks/webhook-events-and-payloads#member + See https://docs.github.com/en/webhooks/webhook-events-and-payloads#member. """ data = self.request.data action = data["action"] @@ -466,6 +469,7 @@ def _handle_member_event(self): installation.service.update_or_create_repositories( [data["repository"]["id"]] ) + return # NOTE: this should never happen. raise ValidationError(f"Unsupported action: {action}") @@ -506,7 +510,7 @@ def _get_remote_repository(self): data = self.request.data remote_id = data["repository"]["id"] installation, _ = self._get_or_create_installation() - return installation.repositories.filter(remote_id=remote_id).first() + return installation.repositories.filter(remote_id=str(remote_id)).first() def _get_or_create_installation(self, sync_repositories_on_create: bool = True): """ diff --git a/readthedocs/settings/test.py b/readthedocs/settings/test.py index 985abef831e..0cf495654a1 100644 --- a/readthedocs/settings/test.py +++ b/readthedocs/settings/test.py @@ -37,6 +37,8 @@ class CommunityTestSettings(CommunityBaseSettings): } } + GITHUB_APP_WEBHOOK_SECRET = "secret" + @property def PASSWORD_HASHERS(self): # Speed up tests by using a fast password hasher as the default. From 3fb108383dd8a2b56256caee2870bc108275543a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 26 Feb 2025 11:03:42 -0500 Subject: [PATCH 58/92] Stash --- readthedocs/rtd_tests/tests/test_oauth.py | 1 + readthedocs/rtd_tests/tests/test_oauth_sync.py | 1 + readthedocs/rtd_tests/tests/test_oauth_tasks.py | 1 + 3 files changed, 3 insertions(+) diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index d32f470fb34..6e1dda39976 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -19,6 +19,7 @@ from readthedocs.projects.models import Project +# TODO: add tests here class GitHubOAuthTests(TestCase): fixtures = ["eric", "test_data"] diff --git a/readthedocs/rtd_tests/tests/test_oauth_sync.py b/readthedocs/rtd_tests/tests/test_oauth_sync.py index 5773f6e6357..14fa7b5cb2b 100644 --- a/readthedocs/rtd_tests/tests/test_oauth_sync.py +++ b/readthedocs/rtd_tests/tests/test_oauth_sync.py @@ -16,6 +16,7 @@ from readthedocs.projects.models import Project +# TODO: add tests here class GitHubOAuthSyncTests(TestCase): payload_user_repos = [ { diff --git a/readthedocs/rtd_tests/tests/test_oauth_tasks.py b/readthedocs/rtd_tests/tests/test_oauth_tasks.py index 8b255e84bc7..a8dfbde39e6 100644 --- a/readthedocs/rtd_tests/tests/test_oauth_tasks.py +++ b/readthedocs/rtd_tests/tests/test_oauth_tasks.py @@ -21,6 +21,7 @@ from readthedocs.sso.models import SSOIntegration +# TODO: add tests here for the githubapp class SyncRemoteRepositoriesTests(TestCase): def setUp(self): self.user = get(User) From eacf0580702310120b36b8d176f2b53f53e67af5 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 27 Feb 2025 18:45:41 -0500 Subject: [PATCH 59/92] Tests --- .../oauth/migrations/0017_githubapp.py | 39 --- .../oauth/migrations/0018_githubapp.py | 88 +++++ readthedocs/oauth/services/githubapp.py | 13 +- readthedocs/rtd_tests/tests/test_oauth.py | 301 +++++++++++++++++- readthedocs/settings/test.py | 57 ++++ 5 files changed, 451 insertions(+), 47 deletions(-) delete mode 100644 readthedocs/oauth/migrations/0017_githubapp.py create mode 100644 readthedocs/oauth/migrations/0018_githubapp.py diff --git a/readthedocs/oauth/migrations/0017_githubapp.py b/readthedocs/oauth/migrations/0017_githubapp.py deleted file mode 100644 index c8a522a5123..00000000000 --- a/readthedocs/oauth/migrations/0017_githubapp.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 4.2.18 on 2025-02-03 21:58 -from django_safemigrate import Safe - -from django.db import migrations, models -import django.db.models.deletion -import django_extensions.db.fields - - -class Migration(migrations.Migration): - - safe = Safe.before_deploy - - dependencies = [ - ('oauth', '0016_deprecate_old_vcs'), - ] - - operations = [ - migrations.CreateModel( - name='GitHubAppInstallation', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), - ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), - ('installation_id', models.PositiveBigIntegerField(db_index=True, help_text='The application installation ID', unique=True)), - ('target_id', models.PositiveBigIntegerField(help_text='A GitHub account ID, it can be from a user or an organization')), - ('target_type', models.CharField(choices=[('User', 'User'), ('Organization', 'Organization')], help_text='Account type that the target_id belongs to (user or organization)', max_length=255)), - ('extra_data', models.JSONField(default=dict, help_text='Extra data returned by the webhook when the installation is created')), - ], - options={ - 'get_latest_by': 'modified', - 'abstract': False, - }, - ), - migrations.AddField( - model_name='remoterepository', - name='github_app_installation', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='repositories', to='oauth.githubappinstallation', verbose_name='GitHub App Installation'), - ), - ] diff --git a/readthedocs/oauth/migrations/0018_githubapp.py b/readthedocs/oauth/migrations/0018_githubapp.py new file mode 100644 index 00000000000..ac6b30f7d4a --- /dev/null +++ b/readthedocs/oauth/migrations/0018_githubapp.py @@ -0,0 +1,88 @@ +# Generated by Django 4.2.18 on 2025-02-03 21:58 +import django.db.models.deletion +import django_extensions.db.fields +from django.db import migrations, models +from django_safemigrate import Safe + + +class Migration(migrations.Migration): + + safe = Safe.before_deploy + + dependencies = [ + ("oauth", "0017_remove_unused_indexes"), + ] + + operations = [ + migrations.CreateModel( + name="GitHubAppInstallation", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "installation_id", + models.PositiveBigIntegerField( + db_index=True, + help_text="The application installation ID", + unique=True, + ), + ), + ( + "target_id", + models.PositiveBigIntegerField( + help_text="A GitHub account ID, it can be from a user or an organization" + ), + ), + ( + "target_type", + models.CharField( + choices=[("User", "User"), ("Organization", "Organization")], + help_text="Account type that the target_id belongs to (user or organization)", + max_length=255, + ), + ), + ( + "extra_data", + models.JSONField( + default=dict, + help_text="Extra data returned by the webhook when the installation is created", + ), + ), + ], + options={ + "get_latest_by": "modified", + "abstract": False, + }, + ), + migrations.AddField( + model_name="remoterepository", + name="github_app_installation", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="repositories", + to="oauth.githubappinstallation", + verbose_name="GitHub App Installation", + ), + ), + ] diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 2bbec5329be..c23f1710e5b 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -117,14 +117,16 @@ def for_user(cls, user): continue for gh_installation in resp.json()["installations"]: - installation = GitHubAppInstallation.objects.get_or_create_installation( + ( + installation, + _, + ) = GitHubAppInstallation.objects.get_or_create_installation( installation_id=gh_installation["id"], target_id=gh_installation["target_id"], target_type=gh_installation["target_type"], extra_data={"installation": gh_installation}, - ).first() - if installation: - yield cls(installation) + ) + yield cls(installation) @classmethod def sync_user_access(cls, user): @@ -151,6 +153,9 @@ def sync_user_access(cls, user): service = cls(repository.github_app_installation) service.update_or_create_repositories([repository.remote_id]) + # TODO: maybe also refresh the organizations the user has access to? + # But doesn't look like we are using that relation for anything? + def sync(self): """ Sync all repositories and organizations that are accessible to the installation. diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index 6e1dda39976..bae8fa0042d 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -1,7 +1,8 @@ import copy from unittest import mock -from allauth.socialaccount.models import SocialAccount +import requests_mock +from allauth.socialaccount.models import SocialAccount, SocialToken from django.conf import settings from django.contrib.auth.models import User from django.test import TestCase @@ -9,17 +10,309 @@ from django.urls import reverse from django_dynamic_fixture import get +from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider from readthedocs.builds.constants import BUILD_STATUS_SUCCESS, EXTERNAL from readthedocs.builds.models import Build, Version from readthedocs.integrations.models import GitHubWebhook, GitLabWebhook -from readthedocs.oauth.constants import BITBUCKET, GITHUB, GITLAB -from readthedocs.oauth.models import RemoteOrganization, RemoteRepository +from readthedocs.oauth.constants import BITBUCKET, GITHUB, GITHUB_APP, GITLAB +from readthedocs.oauth.models import ( + GitHubAccountType, + GitHubAppInstallation, + RemoteOrganization, + RemoteOrganizationRelation, + RemoteRepository, + RemoteRepositoryRelation, +) from readthedocs.oauth.services import BitbucketService, GitHubService, GitLabService +from readthedocs.oauth.services.githubapp import GitHubAppService from readthedocs.projects import constants from readthedocs.projects.models import Project -# TODO: add tests here +@override_settings() +class GitHubAppTests(TestCase): + def setUp(self): + self.user = get(User) + self.account = get( + SocialAccount, + uid="1111", + user=self.user, + provider=GitHubAppProvider.id, + ) + get( + SocialToken, + account=self.account, + ) + self.installation = get( + GitHubAppInstallation, + installation_id=1111, + target_id=int(self.account.uid), + target_type=GitHubAccountType.USER, + ) + self.remote_repository = get( + RemoteRepository, + name="repo", + full_name="user/repo", + vcs_provider=GITHUB_APP, + github_app_installation=self.installation, + ) + get( + RemoteRepositoryRelation, + remote_repository=self.remote_repository, + user=self.user, + account=self.account, + admin=True, + ) + self.project = get( + Project, users=[self.user], remote_repository=self.remote_repository + ) + + self.remote_organization = get( + RemoteOrganization, + slug="org", + remote_id="2222", + ) + get( + RemoteOrganizationRelation, + remote_organization=self.remote_organization, + user=self.user, + account=self.account, + ) + self.organization_installation = get( + GitHubAppInstallation, + installation_id=2222, + target_id=int(self.remote_organization.remote_id), + target_type=GitHubAccountType.ORGANIZATION, + ) + self.remote_repository_with_org = get( + RemoteRepository, + name="repo", + full_name="org/repo", + vcs_provider=GITHUB_APP, + github_app_installation=self.organization_installation, + organization=self.remote_organization, + ) + get( + RemoteRepositoryRelation, + remote_repository=self.remote_repository_with_org, + user=self.user, + account=self.account, + admin=True, + ) + self.project_with_org = get( + Project, + users=[self.user], + remote_repository=self.remote_repository_with_org, + ) + + def _merge_dicts(self, a, b): + for k, v in b.items(): + if k in a and isinstance(a[k], dict): + self._merge_dicts(a[k], v) + else: + a[k] = v + + def _get_access_token_json(self, **kwargs): + default = { + "token": "ghs_16C7e42F292c6912E7710c838347Ae178B4a", + "expires_at": "2016-07-11T22:14:10Z", + "permissions": {"issues": "write", "contents": "read"}, + } + self._merge_dicts(default, kwargs) + return default + + def _get_installation_json(self, id, **kwargs): + default = { + "id": id, + "account": { + "login": "user", + "id": 1111, + "type": "User", + }, + "html_url": f"https://github.com/organizations/github/settings/installations/{id}", + "app_id": 1, + "target_id": 1111, + "target_type": "User", + "permissions": {"checks": "write", "metadata": "read", "contents": "read"}, + "events": ["push", "pull_request"], + "repository_selection": "all", + "created_at": "2017-07-08T16:18:44-04:00", + "updated_at": "2017-07-08T16:18:44-04:00", + "app_slug": "github-actions", + "suspended_at": None, + "suspended_by": None, + } + self._merge_dicts(default, kwargs) + return default + + def _get_repository_json(self, full_name, **kwargs): + user, repo = full_name.split("/") + default = { + "id": 1111, + "name": repo, + "full_name": full_name, + "ssh_url": f"git@github.com:{full_name}.git", + "clone_url": f"https://github.com/{full_name}.git", + "html_url": f"https://github.com/{full_name}", + "private": False, + "default_branch": "main", + "url": f"https://api.github.com/repos/{full_name}", + "description": "Some description", + "owner": { + "login": user, + "id": 1111, + "type": "User", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "url": f"https://api.github.com/users/{user}", + }, + } + self._merge_dicts(default, kwargs) + return default + + def _get_collaborator_json(self, **kwargs): + default = { + "id": 1111, + "login": "user", + "permissions": {"admin": True}, + } + self._merge_dicts(default, kwargs) + return default + + def test_for_project(self): + services = list(GitHubAppService.for_project(self.project)) + assert len(services) == 1 + service = services[0] + assert service.installation == self.installation + + services = list(GitHubAppService.for_project(self.project_with_org)) + assert len(services) == 1 + service = services[0] + assert service.installation == self.organization_installation + + @requests_mock.Mocker(kw="request") + def test_for_user(self, request): + new_installation_id = 3333 + assert not GitHubAppInstallation.objects.filter( + installation_id=new_installation_id + ).exists() + request.get( + "https://api.github.com/user/installations", + json={ + "installations": [ + self._get_installation_json( + id=self.installation.installation_id, + account={"id": self.account.uid}, + ), + self._get_installation_json( + id=new_installation_id, + target_id=2, + target_type="Organization", + account={"login": "octocat", "id": 2, "type": "Organization"}, + ), + ] + }, + ) + services = list(GitHubAppService.for_user(self.user)) + assert len(services) == 2 + + self.installation.refresh_from_db() + assert self.installation.installation_id == 1111 + assert self.installation.target_id == int(self.account.uid) + assert self.installation.target_type == GitHubAccountType.USER + + new_installation = GitHubAppInstallation.objects.get( + installation_id=new_installation_id + ) + assert new_installation.target_id == 2 + assert new_installation.target_type == GitHubAccountType.ORGANIZATION + + @requests_mock.Mocker(kw="request") + @mock.patch.object(GitHubAppService, "sync") + @mock.patch.object(GitHubAppService, "update_or_create_repositories") + def test_sync_user_access(self, update_or_create_repositories, sync, request): + request.get( + "https://api.github.com/user/installations", + json={ + "installations": [ + self._get_installation_json( + id=self.installation.installation_id, + account={"id": self.account.uid}, + ), + ], + }, + ) + + GitHubAppService.sync_user_access(self.user) + sync.assert_called_once() + update_or_create_repositories.assert_has_calls( + [ + mock.call([self.remote_repository.remote_id]), + mock.call([self.remote_repository_with_org.remote_id]), + ], + any_order=True, + ) + + @requests_mock.Mocker(kw="request") + def test_create_repository(self, request): + new_repo_id = 4444 + assert not RemoteRepository.objects.filter( + remote_id=new_repo_id, vcs_provider=GitHubAppProvider.id + ).exists() + + api_url = "https://api.github.com:443" + request.post( + f"{api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{api_url}/repositories/4444", + json=self._get_repository_json( + full_name="user/repo", id=4444, owner={"id": int(self.account.uid)} + ), + ) + request.get( + f"{api_url}/repos/user/repo/collaborators", + json=[self._get_collaborator_json()], + ) + + service = self.installation.service + service.update_or_create_repositories([new_repo_id]) + + repo = RemoteRepository.objects.get( + remote_id=new_repo_id, vcs_provider=GitHubAppProvider.id + ) + assert repo.name == "repo" + assert repo.full_name == "user/repo" + assert repo.organization is None + assert repo.description == "Some description" + assert repo.avatar_url == "https://github.com/images/error/octocat_happy.gif" + assert repo.ssh_url == "git@github.com:user/repo.git" + assert repo.html_url == "https://github.com/user/repo" + assert repo.clone_url == "https://github.com/user/repo.git" + assert not repo.private + assert repo.default_branch == "main" + assert repo.github_app_installation == self.installation + + relations = repo.remote_repository_relations.all() + assert relations.count() == 1 + relation = relations[0] + assert relation.user == self.user + assert relation.account == self.account + assert relation.admin + + def test_update_repository(self): + pass + + def test_sync(self): + pass + + def test_send_build_status(self): + pass + + def test_get_clone_token(self): + pass + + class GitHubOAuthTests(TestCase): fixtures = ["eric", "test_data"] diff --git a/readthedocs/settings/test.py b/readthedocs/settings/test.py index 0cf495654a1..7bf9189fac1 100644 --- a/readthedocs/settings/test.py +++ b/readthedocs/settings/test.py @@ -1,4 +1,5 @@ import os +import textwrap from .base import CommunityBaseSettings @@ -37,6 +38,62 @@ class CommunityTestSettings(CommunityBaseSettings): } } + # Random private RSA key for testing + # $ openssl genpkey -algorithm RSA -out private-key.pem -pkeyopt rsa_keygen_bits:4096 + GITHUB_APP_PRIVATE_KEY = textwrap.dedent(""" + -----BEGIN PRIVATE KEY----- + MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDChDHFcZRVl/DT + M+gWhptJRqLC5Yuq93OxA7BrAQpknNa4kDSVObrtD3ZlyI90VvlCEb/BcgX+lZGb + lw9K93t+XgD/ZspVyfYXKh2OOd93v+xYpgjiu1idyOeblOldTnyQLaFvooU44hGZ + tuIvtJMKiE/Tnd6uq2CypzBS8CGkTJ/YQuIQmsXHL/DS7Oi00M3UWNe0sq8C4T5u + HohCIVmXFOtduNYwNGCAdRW6qcn9VwtJumDnAy884qX5YYHQk52PeAq2R1nS/Dnv + yhIt4opzY7PXvhUw7Jg16hDz96GlAhEin1HYhQuVDZpBwfQ74VzEFEZ3+dD/WWC0 + bA2o/M4SB7ccyC6CPDY+DZgKDQlHJwRge3mQjyFO3J6q9DokgTVqGxF1YY+UVwTP + JBuW4xowAddMJoUiljr6U/Rv4Hz+nk4YuKkDmqL+EmPr6iGRvQiT/C8cVJsD/E8K + pbG9TYMkluN13NoSWN4odVH8dPkd1LeS4IjcVKmnnraSg/X93gBOQe8L7J1KMutE + V3UFHDrYeSJeieW59NLgMwwpXBfFNRADbVBz40Ok4qERu2lh3bJLOgV9lpedJ25N + LlwTgDY138BDdT6oYZabjw2MUW1uebCzl3yu0yzTjeYxNrgj+wxsqMhKg/bDE96i + pzKL0Z6PR4lvgYcOp4Wh/+kmxdIyZQIDAQABAoICAFbYp9IkQlqu4nahw7se8UEX + mP7UdvXn0o8TexZjWg0O221+8QM5ScSi9TU/hREn7dT6ULehXZzLkb26hbjuYwRK + Gz7s2WTRLZ8tDhIcs7HnDjKMOwZkKA4Wj5Xut/yRWNsUjHHnyXxarwoG1dj/0fDP + aHiukShCWwOY0uIM1bBiB7IKNp28RJaIyIib/tAQM/3dhr1mU+5Au9t1pVeFRVdH + n0hyiKrwD6/61q9HNGh4jxEldjNeQB56gSklSEzkQ2I1ce7tT2T8eS+e9FvpO/CF + 8Ntfwl1cHR9hOJ18j/64vAbNxECcMk4jyx4V5yI/HehrtwTFFHOVp7AWWEj9SlGJ + A4BxcCvyGAjPKgjaIxMG9bHM8Je4ushlroUgYXroHdSVd9Su7JBoLmRFHf+3oAAj + IieDEXBeQbUx8lIQmsO71l+eX2EvouNduQ0NY+oCF9ookd4dDXA/rkexZD4so7Ee + lb79z3i1HTidYZHoejlEyTHQFXOeonzmKgVGDHxJrHm5clnAjk8HdmyWgFxcgSVO + fnPWdntTTGUQNowqkaJLyMhYHIP5BGZZ7oGCfY2WuUfBSMgsssnOmUmavN5hbpbk + Ohikn4sKIeZQTf2HJiJNBPdV5FJ3Y1TT8MW5lvNnDDS1M+ezEBSWd6NaN3PXlQpk + pGEIuoVv3Yo/eIJPjqinAoIBAQD5U5FW9fbOUTpxfI6Cg1ktBmQCRALeiCbU+jZf + PVIU+HZBkChkL50Qpq0D9KjJqVvZGyMFG3jkJbdKoAADkcuRptf678GWlOjAzERa + Z43Hh4vsF8PnXNojslpgoMRRWMhckcEV6VrhdvtqupBNW2GnyLofCPomwLL5DSfO + M+3pgxlqav+3WMZkTowHWu5NP8v1+X/6O2ASIWe8XNSvrcchNj6wvPVaZZB5OV28 + IcsnKOhveVy9fmXmuNZfWzEGiccra5DmqfkXwz2ZZokOTSfr2R7IsVNj5Z686oWW + FawmEMx7zFQE6BRpDgGTZF/e1ve3sjmarX/jTYks7rBnW5IzAoIBAQDHuQ7Jf0tf + BX7fZtDPy0JzwRzg6pYXJktHpXsPvUwf93Y991dDYdXWo/JDnj73G5bgJ7g0jTXq + AYdndK1KUnKeZcV7rFJsqEjIociRSeKFMKS9lWP+XKoJDzNtygAKyCaIZXlOKSXE + xWIUzeigVWnom6fwOuDe3/8TGE1aJSINCnSZ0TZLwsH8+lewjALPOt2e8bZ6ePpe + ypysvVWnASio3OUoLmhbC7YV/lAvLgp8b4vB/9EPzmlwIKjN9Uurq4LOjTwRP/MD + SHSPkiFe47zDyT0S+DOODxNC9bKh26NzOZ1Nbuqy1flvjTlhk68ih0CMEUWPv6wd + sOFXn8AVRQEHAoIBAQCVCyDB9E0yrpoaR1RFrtE7OivEsvVoI8na3SxtqJGN2a2P + qeaLZW8mCg05ZSMVUjmGwlMf9XlCIU29vYHkoF4p1qwb5QE7zA6LWlCuHmNB2MSL + QPWqM/ZvCmo+gzx4SHOV6sebGqFqUJ8hAR/MLollLHgen1YynlUezn9yI9bgFa+2 + zvnIl7gZNF8+8lusMCv0Ac9APghDLlb94hx+XIrCTtQRARRGkppX7TQch7MS2MCC + CvGmkY3G682yuSfIecpnKWk4inlOfDcxoXri4rqvoV5mqKJqAFTxJ9ztiE0dgENM + 6it7t2SkHGxSuNkatDTnShJnZboinjIXeyRW1QXDAoIBACikJ7YpCRVU8PRU37jp + C6Syb0X1doVPbZIuwlP5mTwIBy+k3UUA65q50dqgoP93xcPnUTygX5A2r28F9x1g + maJR41W/QyaJOAZbpYyrFEU2GM/bTnW8NX2SckytBkUrZWvr+jtFdEIOSF8jZ2r4 + 9ow24H2p/Yhc3HLuRw9I7xzoO8HxKLNR9lecOavbUdcJi3+EgDV72LbhU/BytrM9 + MSDrklYS23lrcKoZDggLvmaD7FSV0dz9i8cdXjxK5hMQ25VceBSqhrDsVYvBmLjO + buMIWD079IG735eIl8kIAMK5vqC7KVcq448nlb2dZ84G58OY4CbYQhXooHJMN7Ic + UJECggEAJuMo2+TjqSKP3NgPknDboWm4rpi6u9u4/5Lp6gFVr0wXbFVEcQWic9Gt + pb7+hgm3x08s7RBWr8SsDkslT0rFs6v05nYIsALFUu0BXtYqh652BRY8hLD8cDew + V1YR0bULHRFbN8AyjNVNS/68R89kb9kYgAjsJP/30AIdAopP2UMSCj9cq8BUrOHf + 1JhQ9/uq2YJo3XEz2ypjitUMgCtpLxu9WKDU0sYyqZaOxbr8q3HLOAttNzp0Ai3a + wFZ8cpFd+mMwHGsM9+WqUFnnZkHbw2ylLo/Kv3eHLA0MEYyyF8hLvPH4JV+ftnDS + agcfZZcGZQjnJO+V/MWnsSY4obY8Ag== + -----END PRIVATE KEY----- + """).strip() GITHUB_APP_WEBHOOK_SECRET = "secret" @property From 619858cc330a14fe365f40a1a77b31330985cb83 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Mar 2025 14:13:07 -0500 Subject: [PATCH 60/92] Tests --- readthedocs/rtd_tests/tests/test_oauth.py | 101 +++++++++++++++++++++- 1 file changed, 97 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index bae8fa0042d..fe233d34f8f 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -55,6 +55,7 @@ def setUp(self): full_name="user/repo", vcs_provider=GITHUB_APP, github_app_installation=self.installation, + description="Some description", ) get( RemoteRepositoryRelation, @@ -300,11 +301,103 @@ def test_create_repository(self, request): assert relation.account == self.account assert relation.admin - def test_update_repository(self): - pass + @requests_mock.Mocker(kw="request") + def test_update_repository(self, request): + assert self.remote_repository.description == "Some description" + api_url = "https://api.github.com:443" + request.post( + f"{api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{api_url}/repositories/{self.remote_repository.remote_id}", + json=self._get_repository_json( + full_name="user/repo", + id=int(self.remote_repository.remote_id), + owner={"id": int(self.account.uid)}, + description="New description", + ), + ) + request.get( + f"{api_url}/repos/user/repo/collaborators", + json=[self._get_collaborator_json()], + ) - def test_sync(self): - pass + service = self.installation.service + service.update_or_create_repositories([int(self.remote_repository.remote_id)]) + + self.remote_repository.refresh_from_db() + assert self.remote_repository.description == "New description" + + @requests_mock.Mocker(kw="request") + def test_sync(self, request): + assert self.installation.repositories.count() == 1 + api_url = "https://api.github.com:443" + request.get( + f"{api_url}/app/installations/1111", + json=self._get_installation_json(id=1111), + ) + request.post( + f"{api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{api_url}/installation/repositories", + json={ + "repositories": [ + self._get_repository_json( + full_name="user/repo", id=int(self.remote_repository.remote_id) + ), + self._get_repository_json(full_name="user/repo2", id=2222), + self._get_repository_json(full_name="user/repo3", id=3333), + ] + }, + ) + + request.get( + f"{api_url}/repos/user/repo/collaborators", + json=[self._get_collaborator_json()], + ) + request.get( + f"{api_url}/repos/user/repo2/collaborators", + json=[self._get_collaborator_json()], + ) + request.get( + f"{api_url}/repos/user/repo3/collaborators", + json=[self._get_collaborator_json()], + ) + + service = self.installation.service + service.sync() + + assert self.installation.repositories.count() == 3 + + repo = self.installation.repositories.get(full_name="user/repo") + assert repo.name == "repo" + assert repo.remote_id == self.remote_repository.remote_id + assert repo.remote_repository_relations.count() == 1 + relation = repo.remote_repository_relations.first() + assert relation.user == self.user + assert relation.account == self.account + assert relation.admin + + repo = self.installation.repositories.get(full_name="user/repo2") + assert repo.name == "repo2" + assert repo.remote_id == "2222" + assert repo.remote_repository_relations.count() == 1 + relation = repo.remote_repository_relations.first() + assert relation.user == self.user + assert relation.account == self.account + assert relation.admin + + repo = self.installation.repositories.get(full_name="user/repo3") + assert repo.name == "repo3" + assert repo.remote_id == "3333" + assert repo.remote_repository_relations.count() == 1 + relation = repo.remote_repository_relations.first() + assert relation.user == self.user + assert relation.account == self.account + assert relation.admin def test_send_build_status(self): pass From 80e6c88b6a5922207febee474204e7bd975132fa Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Mar 2025 16:44:05 -0500 Subject: [PATCH 61/92] More tests --- readthedocs/rtd_tests/tests/test_oauth.py | 215 +++++++++++++++++++--- 1 file changed, 194 insertions(+), 21 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index fe233d34f8f..a4a060c1714 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -11,7 +11,13 @@ from django_dynamic_fixture import get from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider -from readthedocs.builds.constants import BUILD_STATUS_SUCCESS, EXTERNAL +from readthedocs.builds.constants import ( + BUILD_STATUS_FAILURE, + BUILD_STATUS_PENDING, + BUILD_STATUS_SUCCESS, + EXTERNAL, + LATEST, +) from readthedocs.builds.models import Build, Version from readthedocs.integrations.models import GitHubWebhook, GitLabWebhook from readthedocs.oauth.constants import BITBUCKET, GITHUB, GITHUB_APP, GITLAB @@ -29,7 +35,10 @@ from readthedocs.projects.models import Project -@override_settings() +@override_settings( + PRODUCTION_DOMAIN="readthedocs.org", + PUBLIC_DOMAIN="readthedocs.io", +) class GitHubAppTests(TestCase): def setUp(self): self.user = get(User) @@ -105,6 +114,7 @@ def setUp(self): users=[self.user], remote_repository=self.remote_repository_with_org, ) + self.api_url = "https://api.github.com:443" def _merge_dicts(self, a, b): for k, v in b.items(): @@ -179,6 +189,63 @@ def _get_collaborator_json(self, **kwargs): self._merge_dicts(default, kwargs) return default + def _get_commit_json(self, commit, **kwargs): + default = { + "url": f"https://api.github.com/repos/user/repo/commits/{commit}", + "sha": commit, + "html_url": f"https://github.com/user/repo/commit/{commit}", + # "comments_url": "https://api.github.com/repos/octocat/Hello-World/commits/6dcb09b5b57875f334f61aebed695e2e4193db5e/comments", + "commit": { + "url": f"https://api.github.com/repos/octocat/Hello-World/git/commits/{commit}", + "author": { + "name": "Monalisa Octocat", + "email": "mona@github.com", + "date": "2011-04-14T16:00:49Z", + }, + "committer": { + "name": "Monalisa Octocat", + "email": "mona@github.com", + "date": "2011-04-14T16:00:49Z", + }, + "message": "Fix all the bugs", + "tree": { + "url": f"https://api.github.com/repos/user/repo/tree/{commit}", + "sha": commit, + }, + "comment_count": 0, + }, + "author": { + "login": "octocat", + "id": 1, + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "type": "User", + "site_admin": False, + }, + "committer": { + "login": "octocat", + "id": 1, + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "type": "User", + "site_admin": False, + }, + "parents": [ + { + "url": "https://api.github.com/repos/user/repo/commits/6dcb09b5b57875f334f61aebed695e2e4193db5e", + "sha": "6dcb09b5b57875f334f61aebed695e2e4193db5e", + } + ], + "stats": {"additions": 104, "deletions": 4, "total": 108}, + "files": [], + } + self._merge_dicts(default, kwargs) + return default + def test_for_project(self): services = list(GitHubAppService.for_project(self.project)) assert len(services) == 1 @@ -260,19 +327,18 @@ def test_create_repository(self, request): remote_id=new_repo_id, vcs_provider=GitHubAppProvider.id ).exists() - api_url = "https://api.github.com:443" request.post( - f"{api_url}/app/installations/1111/access_tokens", + f"{self.api_url}/app/installations/1111/access_tokens", json=self._get_access_token_json(), ) request.get( - f"{api_url}/repositories/4444", + f"{self.api_url}/repositories/4444", json=self._get_repository_json( full_name="user/repo", id=4444, owner={"id": int(self.account.uid)} ), ) request.get( - f"{api_url}/repos/user/repo/collaborators", + f"{self.api_url}/repos/user/repo/collaborators", json=[self._get_collaborator_json()], ) @@ -304,13 +370,12 @@ def test_create_repository(self, request): @requests_mock.Mocker(kw="request") def test_update_repository(self, request): assert self.remote_repository.description == "Some description" - api_url = "https://api.github.com:443" request.post( - f"{api_url}/app/installations/1111/access_tokens", + f"{self.api_url}/app/installations/1111/access_tokens", json=self._get_access_token_json(), ) request.get( - f"{api_url}/repositories/{self.remote_repository.remote_id}", + f"{self.api_url}/repositories/{self.remote_repository.remote_id}", json=self._get_repository_json( full_name="user/repo", id=int(self.remote_repository.remote_id), @@ -319,7 +384,7 @@ def test_update_repository(self, request): ), ) request.get( - f"{api_url}/repos/user/repo/collaborators", + f"{self.api_url}/repos/user/repo/collaborators", json=[self._get_collaborator_json()], ) @@ -332,17 +397,16 @@ def test_update_repository(self, request): @requests_mock.Mocker(kw="request") def test_sync(self, request): assert self.installation.repositories.count() == 1 - api_url = "https://api.github.com:443" request.get( - f"{api_url}/app/installations/1111", + f"{self.api_url}/app/installations/1111", json=self._get_installation_json(id=1111), ) request.post( - f"{api_url}/app/installations/1111/access_tokens", + f"{self.api_url}/app/installations/1111/access_tokens", json=self._get_access_token_json(), ) request.get( - f"{api_url}/installation/repositories", + f"{self.api_url}/installation/repositories", json={ "repositories": [ self._get_repository_json( @@ -355,15 +419,15 @@ def test_sync(self, request): ) request.get( - f"{api_url}/repos/user/repo/collaborators", + f"{self.api_url}/repos/user/repo/collaborators", json=[self._get_collaborator_json()], ) request.get( - f"{api_url}/repos/user/repo2/collaborators", + f"{self.api_url}/repos/user/repo2/collaborators", json=[self._get_collaborator_json()], ) request.get( - f"{api_url}/repos/user/repo3/collaborators", + f"{self.api_url}/repos/user/repo3/collaborators", json=[self._get_collaborator_json()], ) @@ -399,11 +463,120 @@ def test_sync(self, request): assert relation.account == self.account assert relation.admin - def test_send_build_status(self): - pass + @requests_mock.Mocker(kw="request") + def test_send_build_status_pending(self, request): + commit = "1234abc" + version = self.project.versions.get(slug=LATEST) + build = get( + Build, + project=self.project, + version=version, + ) + request.post( + f"{self.api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{self.api_url}/repositories/{self.remote_repository.remote_id}/commits/{commit}", + json=self._get_commit_json(commit=commit), + ) + status_api_request = request.post( + f"{self.api_url}/repos/user/repo/statuses/{commit}", + json={}, + ) + + service = self.installation.service + assert service.send_build_status( + build=build, commit=commit, status=BUILD_STATUS_PENDING + ) + assert status_api_request.called + assert status_api_request.last_request.json() == { + "context": f"docs/readthedocs:{self.project.slug}", + "description": "Read the Docs build is in progress!", + "state": "pending", + "target_url": f"https://readthedocs.org/projects/{self.project.slug}/builds/{build.pk}/", + } + + @requests_mock.Mocker(kw="request") + def test_send_build_status_success(self, request): + commit = "1234abc" + version = self.project.versions.get(slug=LATEST) + version.built = True + version.save() + build = get( + Build, + project=self.project, + version=version, + ) + request.post( + f"{self.api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{self.api_url}/repositories/{self.remote_repository.remote_id}/commits/{commit}", + json=self._get_commit_json(commit=commit), + ) + status_api_request = request.post( + f"{self.api_url}/repos/user/repo/statuses/{commit}", + json={}, + ) + + service = self.installation.service + assert service.send_build_status( + build=build, commit=commit, status=BUILD_STATUS_SUCCESS + ) + assert status_api_request.called + assert status_api_request.last_request.json() == { + "context": f"docs/readthedocs:{self.project.slug}", + "description": "Read the Docs build succeeded!", + "state": "success", + "target_url": f"http://{self.project.slug}.readthedocs.io/en/latest/", + } + + @requests_mock.Mocker(kw="request") + def test_send_build_status_failure(self, request): + commit = "1234abc" + version = self.project.versions.get(slug=LATEST) + build = get( + Build, + project=self.project, + version=version, + ) + request.post( + f"{self.api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{self.api_url}/repositories/{self.remote_repository.remote_id}/commits/{commit}", + json=self._get_commit_json(commit=commit), + ) + status_api_request = request.post( + f"{self.api_url}/repos/user/repo/statuses/{commit}", + json={}, + ) + + service = self.installation.service + assert service.send_build_status( + build=build, commit=commit, status=BUILD_STATUS_FAILURE + ) + assert status_api_request.called + assert status_api_request.last_request.json() == { + "context": f"docs/readthedocs:{self.project.slug}", + "description": "Read the Docs build failed!", + "state": "failure", + "target_url": f"https://readthedocs.org/projects/{self.project.slug}/builds/{build.pk}/", + } - def test_get_clone_token(self): - pass + @requests_mock.Mocker(kw="request") + def test_get_clone_token(self, request): + token = "ghs_16C7e42F292c6912E7710c838347Ae178B4a" + request.post( + f"{self.api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(toke=token), + ) + service = self.installation.service + clone_token = service.get_clone_token(self.project) + assert clone_token == f"x-access-token:{token}" class GitHubOAuthTests(TestCase): From 926ed6d1962d95ce5d66f6a1b4a850c4a94aea74 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 10 Mar 2025 17:54:12 -0500 Subject: [PATCH 62/92] Tests for tasks --- .../rtd_tests/tests/test_oauth_tasks.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_oauth_tasks.py b/readthedocs/rtd_tests/tests/test_oauth_tasks.py index a8dfbde39e6..7e0ef6980a9 100644 --- a/readthedocs/rtd_tests/tests/test_oauth_tasks.py +++ b/readthedocs/rtd_tests/tests/test_oauth_tasks.py @@ -10,6 +10,7 @@ from django.test import TestCase from django_dynamic_fixture import get +from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider from readthedocs.builds.models import Version from readthedocs.oauth.services.base import SyncServiceError from readthedocs.oauth.tasks import ( @@ -21,7 +22,6 @@ from readthedocs.sso.models import SSOIntegration -# TODO: add tests here for the githubapp class SyncRemoteRepositoriesTests(TestCase): def setUp(self): self.user = get(User) @@ -32,6 +32,11 @@ def setUp(self): user=self.user, provider=GitHubOAuth2Adapter.provider_id, ) + self.socialaccount_ghapp = get( + SocialAccount, + user=self.user, + provider=GitHubAppProvider.id, + ) self.socialaccount_gl = get( SocialAccount, user=self.user, @@ -43,20 +48,23 @@ def setUp(self): provider=BitbucketOAuth2Adapter.provider_id, ) + @patch("readthedocs.oauth.services.githubapp.GitHubAppService.sync_user_access") @patch("readthedocs.oauth.services.github.GitHubService.sync") @patch("readthedocs.oauth.services.gitlab.GitLabService.sync") @patch("readthedocs.oauth.services.bitbucket.BitbucketService.sync") - def test_sync_repository(self, sync_bb, sync_gl, sync_gh): + def test_sync_repository(self, sync_bb, sync_gl, sync_gh, sync_ghapp): r = sync_remote_repositories(self.user.pk) self.assertNotIn("error", r) sync_bb.assert_called_once() sync_gl.assert_called_once() sync_gh.assert_called_once() + sync_ghapp.assert_called_once() + @patch("readthedocs.oauth.services.githubapp.GitHubAppService.sync_user_access") @patch("readthedocs.oauth.services.github.GitHubService.sync") @patch("readthedocs.oauth.services.gitlab.GitLabService.sync") @patch("readthedocs.oauth.services.bitbucket.BitbucketService.sync") - def test_sync_repository_failsync(self, sync_bb, sync_gl, sync_gh): + def test_sync_repository_failsync(self, sync_bb, sync_gl, sync_gh, sync_ghapp): sync_gh.side_effect = SyncServiceError r = sync_remote_repositories(self.user.pk) self.assertIn("GitHub", r["error"]) @@ -65,11 +73,15 @@ def test_sync_repository_failsync(self, sync_bb, sync_gl, sync_gh): sync_bb.assert_called_once() sync_gl.assert_called_once() sync_gh.assert_called_once() + sync_ghapp.assert_called_once() + @patch("readthedocs.oauth.services.githubapp.GitHubAppService.sync_user_access") @patch("readthedocs.oauth.services.github.GitHubService.sync") @patch("readthedocs.oauth.services.gitlab.GitLabService.sync") @patch("readthedocs.oauth.services.bitbucket.BitbucketService.sync") - def test_sync_repository_failsync_more_than_one(self, sync_bb, sync_gl, sync_gh): + def test_sync_repository_failsync_more_than_one( + self, sync_bb, sync_gl, sync_gh, sync_ghapp + ): sync_gh.side_effect = SyncServiceError sync_bb.side_effect = SyncServiceError r = sync_remote_repositories(self.user.pk) @@ -79,6 +91,7 @@ def test_sync_repository_failsync_more_than_one(self, sync_bb, sync_gl, sync_gh) sync_bb.assert_called_once() sync_gl.assert_called_once() sync_gh.assert_called_once() + sync_ghapp.assert_called_once() @patch("readthedocs.oauth.tasks.sync_remote_repositories") def test_sync_remote_repository_organizations_slugs( From 0c338ab2e174659829c0bad6a59390744e4c1e69 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Mar 2025 11:33:28 -0500 Subject: [PATCH 63/92] stash --- readthedocs/oauth/services/githubapp.py | 7 ++++++- readthedocs/rtd_tests/tests/test_oauth_sync.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index c23f1710e5b..35268e04f52 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -168,12 +168,17 @@ def sync(self): """ try: app_installation = self.app_installation - except GithubException: + except GithubException as e: log.info( "Failed to get installation", installation_id=self.installation.installation_id, exc_info=True, ) + + if e.status == 404: + # The installation is no longer accessible, + # we remove it from the database. + self.installation.delete() raise SyncServiceError() remote_repositories = [] diff --git a/readthedocs/rtd_tests/tests/test_oauth_sync.py b/readthedocs/rtd_tests/tests/test_oauth_sync.py index 14fa7b5cb2b..e8b270ca3d0 100644 --- a/readthedocs/rtd_tests/tests/test_oauth_sync.py +++ b/readthedocs/rtd_tests/tests/test_oauth_sync.py @@ -16,7 +16,7 @@ from readthedocs.projects.models import Project -# TODO: add tests here +# TODO: port these tests to the GitHub app. class GitHubOAuthSyncTests(TestCase): payload_user_repos = [ { From 616166e8b9ba9805b2d86a408e21fa25016d98ce Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Mar 2025 11:36:00 -0500 Subject: [PATCH 64/92] Format to avoid conflicts --- readthedocs/oauth/admin.py | 16 +++------ readthedocs/oauth/models.py | 40 +++++++-------------- readthedocs/oauth/services/__init__.py | 5 ++- readthedocs/oauth/services/base.py | 18 +++------- readthedocs/profiles/views.py | 50 +++++++++++--------------- 5 files changed, 48 insertions(+), 81 deletions(-) diff --git a/readthedocs/oauth/admin.py b/readthedocs/oauth/admin.py index 80865f598ed..cad63237a67 100644 --- a/readthedocs/oauth/admin.py +++ b/readthedocs/oauth/admin.py @@ -2,13 +2,11 @@ from django.contrib import admin -from .models import ( - GitHubAppInstallation, - RemoteOrganization, - RemoteOrganizationRelation, - RemoteRepository, - RemoteRepositoryRelation, -) +from .models import GitHubAppInstallation +from .models import RemoteOrganization +from .models import RemoteOrganizationRelation +from .models import RemoteRepository +from .models import RemoteRepositoryRelation @admin.register(GitHubAppInstallation) @@ -22,7 +20,6 @@ class GitHubAppInstallationAdmin(admin.ModelAdmin): @admin.register(RemoteRepository) class RemoteRepositoryAdmin(admin.ModelAdmin): - """Admin configuration for the RemoteRepository model.""" readonly_fields = ( @@ -54,7 +51,6 @@ class RemoteRepositoryAdmin(admin.ModelAdmin): @admin.register(RemoteOrganization) class RemoteOrganizationAdmin(admin.ModelAdmin): - """Admin configuration for the RemoteOrganization model.""" readonly_fields = ( @@ -80,7 +76,6 @@ class RemoteOrganizationAdmin(admin.ModelAdmin): @admin.register(RemoteRepositoryRelation) class RemoteRepositoryRelationAdmin(admin.ModelAdmin): - """Admin configuration for the RemoteRepositoryRelation model.""" raw_id_fields = ( @@ -96,7 +91,6 @@ class RemoteRepositoryRelationAdmin(admin.ModelAdmin): @admin.register(RemoteOrganizationRelation) class RemoteOrganizationRelationAdmin(admin.ModelAdmin): - """Admin configuration for the RemoteOrganizationRelation model.""" raw_id_fields = ( diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index ea7c8a1112b..59b452b17e5 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -14,8 +14,11 @@ from readthedocs.projects.constants import REPO_CHOICES from readthedocs.projects.models import Project -from .constants import GITHUB_APP, VCS_PROVIDER_CHOICES -from .querysets import RemoteOrganizationQuerySet, RemoteRepositoryQuerySet +from .constants import GITHUB_APP +from .constants import VCS_PROVIDER_CHOICES +from .querysets import RemoteOrganizationQuerySet +from .querysets import RemoteRepositoryQuerySet + log = structlog.get_logger(__name__) @@ -40,10 +43,7 @@ def get_or_create_installation( ) # NOTE: An installation can't change its target_id or target_type. # This should never happen, unless this assumption is wrong. - if ( - installation.target_id != target_id - or installation.target_type != target_type - ): + if installation.target_id != target_id or installation.target_type != target_type: log.exception( "Installation target_id or target_type changed", installation_id=installation.installation_id, @@ -73,16 +73,12 @@ class GitHubAppInstallation(TimeStampedModel): help_text=_("A GitHub account ID, it can be from a user or an organization"), ) target_type = models.CharField( - help_text=_( - "Account type that the target_id belongs to (user or organization)" - ), + help_text=_("Account type that the target_id belongs to (user or organization)"), choices=GitHubAccountType.choices, max_length=255, ) extra_data = models.JSONField( - help_text=_( - "Extra data returned by the webhook when the installation is created" - ), + help_text=_("Extra data returned by the webhook when the installation is created"), default=dict, ) @@ -128,14 +124,10 @@ def delete_repositories(self, repository_ids: list[int] | None = None): remote_organizations = remote_organizations.filter( repositories__remote_id__in=repository_ids ) - remote_repositories = remote_repositories.filter( - remote_id__in=repository_ids - ) + remote_repositories = remote_repositories.filter(remote_id__in=repository_ids) # Fetch all IDs before deleting the repositories, so we can filter the organizations later. - remote_organizations_ids = list( - remote_organizations.values_list("id", flat=True) - ) + remote_organizations_ids = list(remote_organizations.values_list("id", flat=True)) count, deleted = remote_repositories.delete() log.info( @@ -201,9 +193,7 @@ class RemoteOrganization(TimeStampedModel): ) # VCS provider organization id remote_id = models.CharField(db_index=True, max_length=128) - vcs_provider = models.CharField( - _("VCS provider"), choices=VCS_PROVIDER_CHOICES, max_length=32 - ) + vcs_provider = models.CharField(_("VCS provider"), choices=VCS_PROVIDER_CHOICES, max_length=32) objects = RemoteOrganizationQuerySet.as_manager() @@ -324,9 +314,7 @@ class RemoteRepository(TimeStampedModel): ) # VCS provider repository id remote_id = models.CharField(max_length=128) - vcs_provider = models.CharField( - _("VCS provider"), choices=VCS_PROVIDER_CHOICES, max_length=32 - ) + vcs_provider = models.CharField(_("VCS provider"), choices=VCS_PROVIDER_CHOICES, max_length=32) github_app_installation = models.ForeignKey( GitHubAppInstallation, @@ -397,9 +385,7 @@ def get_service_class(self): return service_cls # NOTE: this should never happen, but we log it just in case - log.exception( - "Service not found for the VCS provider", vcs_provider=self.vcs_provider - ) + log.exception("Service not found for the VCS provider", vcs_provider=self.vcs_provider) return None diff --git a/readthedocs/oauth/services/__init__.py b/readthedocs/oauth/services/__init__.py index 84c1d385d72..bd5174a559c 100644 --- a/readthedocs/oauth/services/__init__.py +++ b/readthedocs/oauth/services/__init__.py @@ -1,9 +1,12 @@ """Conditional classes for OAuth services.""" from readthedocs.core.utils.extend import SettingsOverrideObject -from readthedocs.oauth.services import bitbucket, github, gitlab +from readthedocs.oauth.services import bitbucket +from readthedocs.oauth.services import github +from readthedocs.oauth.services import gitlab from readthedocs.oauth.services.githubapp import GitHubAppService + __all__ = [ "GitHubService", "BitbucketService", diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index b90115694c1..066c1aa08b5 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -15,6 +15,7 @@ from readthedocs.core.permissions import AdminPermission from readthedocs.oauth.clients import get_oauth2_client + log = structlog.get_logger(__name__) @@ -119,10 +120,7 @@ def is_project_service(cls, project): :py:class:`RemoteRepository` to the project instance. This is a slight improvement on the legacy check for webhooks """ - return ( - cls.url_pattern is not None - and cls.url_pattern.search(project.repo) is not None - ) + return cls.url_pattern is not None and cls.url_pattern.search(project.repo) is not None class UserService(Service): @@ -248,12 +246,8 @@ def sync(self): # Delete RemoteRepository where the user doesn't have access anymore # (skip RemoteRepository tied to a Project on this user) - all_remote_repositories = ( - remote_repositories + remote_repositories_organizations - ) - repository_remote_ids = [ - r.remote_id for r in all_remote_repositories if r is not None - ] + all_remote_repositories = remote_repositories + remote_repositories_organizations + repository_remote_ids = [r.remote_id for r in all_remote_repositories if r is not None] ( self.user.remote_repository_relations.exclude( remote_repository__remote_id__in=repository_remote_ids, @@ -267,9 +261,7 @@ def sync(self): ) # Delete RemoteOrganization where the user doesn't have access anymore - organization_remote_ids = [ - o.remote_id for o in remote_organizations if o is not None - ] + organization_remote_ids = [o.remote_id for o in remote_organizations if o is not None] ( self.user.remote_organization_relations.exclude( diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index 60c7dbb98d2..fa10bff473e 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -9,37 +9,36 @@ from django.contrib.auth import logout from django.contrib.auth.models import User from django.contrib.messages.views import SuccessMessageMixin -from django.http import Http404, HttpResponseRedirect +from django.http import Http404 +from django.http import HttpResponseRedirect from django.urls import reverse from django.utils import timezone from django.utils.translation import gettext_lazy as _ from rest_framework.authtoken.models import Token -from vanilla import ( - CreateView, - DeleteView, - DetailView, - FormView, - ListView, - TemplateView, - UpdateView, -) +from vanilla import CreateView +from vanilla import DeleteView +from vanilla import DetailView +from vanilla import FormView +from vanilla import ListView +from vanilla import TemplateView +from vanilla import UpdateView from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider from readthedocs.audit.filters import UserSecurityLogFilter from readthedocs.audit.models import AuditLog -from readthedocs.core.forms import UserAdvertisingForm, UserDeleteForm, UserProfileForm +from readthedocs.core.forms import UserAdvertisingForm +from readthedocs.core.forms import UserDeleteForm +from readthedocs.core.forms import UserProfileForm from readthedocs.core.history import set_change_reason from readthedocs.core.mixins import PrivateViewMixin from readthedocs.core.models import UserProfile from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.extend import SettingsOverrideObject -from readthedocs.oauth.migrate import ( - get_installation_target_groups_for_user, - get_migration_targets, - get_old_app_link, - get_projects_missing_migration, - migrate_project_to_github_app, -) +from readthedocs.oauth.migrate import get_installation_target_groups_for_user +from readthedocs.oauth.migrate import get_migration_targets +from readthedocs.oauth.migrate import get_old_app_link +from readthedocs.oauth.migrate import get_projects_missing_migration +from readthedocs.oauth.migrate import migrate_project_to_github_app from readthedocs.organizations.models import Organization from readthedocs.projects.models import Project from readthedocs.projects.utils import get_csv_file @@ -250,8 +249,7 @@ def _get_csv_data(self): end=timezone.datetime.strftime(end_date, "%Y-%m-%d"), ) csv_data = [ - [timezone.datetime.strftime(date, "%Y-%m-%d %H:%M:%S"), *rest] - for date, *rest in data + [timezone.datetime.strftime(date, "%Y-%m-%d %H:%M:%S"), *rest] for date, *rest in data ] csv_data.insert(0, [header for header, _ in values]) return get_csv_file(filename=filename, csv_data=csv_data) @@ -315,9 +313,7 @@ def get_context_data(self, **kwargs): context["has_gh_app_social_account"] = user.socialaccount_set.filter( provider=GitHubAppProvider.id ).exists() - context["installation_target_groups"] = get_installation_target_groups_for_user( - user - ) + context["installation_target_groups"] = get_installation_target_groups_for_user(user) context["migration_targets"] = get_migration_targets(user) context["old_application_link"] = get_old_app_link() return context @@ -325,18 +321,14 @@ def get_context_data(self, **kwargs): def post(self, request, *args, **kwargs): project_slug = request.POST.get("project") if project_slug: - projects = AdminPermission.projects(request.user, admin=True).filter( - slug=project_slug - ) + projects = AdminPermission.projects(request.user, admin=True).filter(slug=project_slug) else: projects = get_projects_missing_migration(request.user) has_errors = False for project in projects: try: - result = migrate_project_to_github_app( - project=project, user=request.user - ) + result = migrate_project_to_github_app(project=project, user=request.user) if not result.webhook_removed: messages.warning( request, From b58c3b8ffca2d1a6890b24afa4c6ead186b83a70 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Mar 2025 11:38:41 -0500 Subject: [PATCH 65/92] Format --- readthedocs/oauth/clients.py | 3 +- readthedocs/oauth/migrate.py | 22 +++--- .../oauth/migrations/0018_githubapp.py | 4 +- readthedocs/oauth/models.py | 1 + readthedocs/oauth/querysets.py | 3 +- readthedocs/oauth/services/githubapp.py | 78 +++++++------------ readthedocs/oauth/urls.py | 1 + readthedocs/oauth/views.py | 38 ++++----- readthedocs/projects/models.py | 3 +- 9 files changed, 66 insertions(+), 87 deletions(-) diff --git a/readthedocs/oauth/clients.py b/readthedocs/oauth/clients.py index 3857087712e..5d058123c11 100644 --- a/readthedocs/oauth/clients.py +++ b/readthedocs/oauth/clients.py @@ -3,7 +3,8 @@ import structlog from django.conf import settings from django.utils import timezone -from github import Auth, GithubIntegration +from github import Auth +from github import GithubIntegration from requests_oauthlib import OAuth2Session diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index d2b6bc11f85..83e929f6199 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -5,9 +5,11 @@ from readthedocs.core.permissions import AdminPermission from readthedocs.integrations.models import Integration -from readthedocs.oauth.constants import GITHUB, GITHUB_APP +from readthedocs.oauth.constants import GITHUB +from readthedocs.oauth.constants import GITHUB_APP from readthedocs.oauth.models import RemoteRepository -from readthedocs.oauth.services import GitHubAppService, GitHubService +from readthedocs.oauth.services import GitHubAppService +from readthedocs.oauth.services import GitHubService from readthedocs.projects.models import Project @@ -28,7 +30,9 @@ def link(self): repository_ids.append(f"&repository_ids[]={repository_id}") repository_ids = "".join(repository_ids) - base_url = f"https://github.com/apps/{settings.GITHUB_APP_NAME}/installations/new/permissions" + base_url = ( + f"https://github.com/apps/{settings.GITHUB_APP_NAME}/installations/new/permissions" + ) return f"{base_url}?suggested_target_id={self.target_id}{repository_ids}" @property @@ -73,9 +77,7 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou target_id=account.uid, target_name=account.extra_data.get("login"), target_type="user", - repository_ids={ - remote_repository.remote_id for remote_repository in user_repositories - }, + repository_ids={remote_repository.remote_id for remote_repository in user_repositories}, ) return list(targets.values()) @@ -124,7 +126,9 @@ def installation_link(self): """ See https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/migrating-oauth-apps-to-github-apps """ - base_url = f"https://github.com/apps/{settings.GITHUB_APP_NAME}/installations/new/permissions" + base_url = ( + f"https://github.com/apps/{settings.GITHUB_APP_NAME}/installations/new/permissions" + ) return f"{base_url}?suggested_target_id={self.target_id}&repository_ids[]={self.project.remote_repository.remote_id}" @property @@ -204,9 +208,7 @@ def migrate_project_to_github_app(project, user) -> MigrationResult: .first() ) if not new_remote_repository: - raise MigrationError( - "You must have admin permissions on the repository to migrate it" - ) + raise MigrationError("You must have admin permissions on the repository to migrate it") webhook_removed = False ssh_key_removed = False diff --git a/readthedocs/oauth/migrations/0018_githubapp.py b/readthedocs/oauth/migrations/0018_githubapp.py index ac6b30f7d4a..d46b7443c8a 100644 --- a/readthedocs/oauth/migrations/0018_githubapp.py +++ b/readthedocs/oauth/migrations/0018_githubapp.py @@ -1,12 +1,12 @@ # Generated by Django 4.2.18 on 2025-02-03 21:58 import django.db.models.deletion import django_extensions.db.fields -from django.db import migrations, models +from django.db import migrations +from django.db import models from django_safemigrate import Safe class Migration(migrations.Migration): - safe = Safe.before_deploy dependencies = [ diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 412a72a3a72..59b452b17e5 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -1,6 +1,7 @@ """OAuth service models.""" from functools import cached_property + import structlog from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import User diff --git a/readthedocs/oauth/querysets.py b/readthedocs/oauth/querysets.py index 5b2c9277614..667c0a237be 100644 --- a/readthedocs/oauth/querysets.py +++ b/readthedocs/oauth/querysets.py @@ -3,7 +3,8 @@ from django.db import models from readthedocs.core.querysets import NoReprQuerySet -from readthedocs.oauth.constants import GITHUB, GITHUB_APP +from readthedocs.oauth.constants import GITHUB +from readthedocs.oauth.constants import GITHUB_APP class RelatedUserQuerySet(NoReprQuerySet, models.QuerySet): diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 35268e04f52..543770498a3 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -1,26 +1,30 @@ -from functools import cached_property, lru_cache +from functools import cached_property +from functools import lru_cache import structlog from allauth.socialaccount.models import SocialAccount from django.conf import settings -from github import Github, GithubException +from github import Github +from github import GithubException from github.Installation import Installation as GHInstallation from github.Organization import Organization as GHOrganization from github.Repository import Repository as GHRepository from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider -from readthedocs.builds.constants import BUILD_STATUS_SUCCESS, SELECT_BUILD_STATUS -from readthedocs.oauth.clients import get_gh_app_client, get_oauth2_client +from readthedocs.builds.constants import BUILD_STATUS_SUCCESS +from readthedocs.builds.constants import SELECT_BUILD_STATUS +from readthedocs.oauth.clients import get_gh_app_client +from readthedocs.oauth.clients import get_oauth2_client from readthedocs.oauth.constants import GITHUB_APP -from readthedocs.oauth.models import ( - GitHubAccountType, - GitHubAppInstallation, - RemoteOrganization, - RemoteOrganizationRelation, - RemoteRepository, - RemoteRepositoryRelation, -) -from readthedocs.oauth.services.base import Service, SyncServiceError +from readthedocs.oauth.models import GitHubAccountType +from readthedocs.oauth.models import GitHubAppInstallation +from readthedocs.oauth.models import RemoteOrganization +from readthedocs.oauth.models import RemoteOrganizationRelation +from readthedocs.oauth.models import RemoteRepository +from readthedocs.oauth.models import RemoteRepositoryRelation +from readthedocs.oauth.services.base import Service +from readthedocs.oauth.services.base import SyncServiceError + log = structlog.get_logger(__name__) @@ -52,9 +56,7 @@ def app_installation(self) -> GHInstallation: @cached_property def installation_client(self) -> Github: """Return a client authenticated as the GitHub installation to interact with the GH API.""" - return self.gha_client.get_github_for_installation( - self.installation.installation_id - ) + return self.gha_client.get_github_for_installation(self.installation.installation_id) @classmethod def for_project(cls, project): @@ -65,10 +67,7 @@ def for_project(cls, project): and are linked to a GitHub App installation, this returns only one service or None. """ - if ( - not project.remote_repository - or not project.remote_repository.github_app_installation - ): + if not project.remote_repository or not project.remote_repository.github_app_installation: return None yield cls(project.remote_repository.github_app_installation) @@ -242,18 +241,14 @@ def _create_or_update_repository_from_gh( remote_repo.name = gh_repo.name remote_repo.full_name = gh_repo.full_name remote_repo.description = gh_repo.description - remote_repo.avatar_url = ( - gh_repo.owner.avatar_url or self.default_user_avatar_url - ) + remote_repo.avatar_url = gh_repo.owner.avatar_url or self.default_user_avatar_url remote_repo.ssh_url = gh_repo.ssh_url remote_repo.html_url = gh_repo.html_url remote_repo.private = gh_repo.private remote_repo.default_branch = gh_repo.default_branch # TODO: Do we need the SSH URL for private repositories now that we can clone using a token? - remote_repo.clone_url = ( - gh_repo.ssh_url if gh_repo.private else gh_repo.clone_url - ) + remote_repo.clone_url = gh_repo.ssh_url if gh_repo.private else gh_repo.clone_url # NOTE: Only one installation of our APP should give access to a repository. # This should only happen if our data is out of sync. @@ -274,9 +269,7 @@ def _create_or_update_repository_from_gh( # NOTE: The owner object doesn't have all attributes of an organization, # so we need to fetch the organization object. gh_organization = self._get_gh_organization(gh_repo.owner.login) - remote_repo.organization = self._create_or_update_organization_from_gh( - gh_organization - ) + remote_repo.organization = self._create_or_update_organization_from_gh(gh_organization) remote_repo.save() self._resync_collaborators(gh_repo, remote_repo) @@ -290,9 +283,7 @@ def _get_gh_organization(self, login: str) -> GHOrganization: # NOTE: normally, this should cache only one organization at a time, but just in case... @lru_cache(maxsize=50) - def _create_or_update_organization_from_gh( - self, gh_org: GHOrganization - ) -> RemoteOrganization: + def _create_or_update_organization_from_gh(self, gh_org: GHOrganization) -> RemoteOrganization: """ Create or update a remote organization from a GitHub organization object. @@ -314,17 +305,14 @@ def _create_or_update_organization_from_gh( self._resync_organization_members(gh_org, remote_org) return remote_org - def _resync_collaborators( - self, gh_repo: GHRepository, remote_repo: RemoteRepository - ): + def _resync_collaborators(self, gh_repo: GHRepository, remote_repo: RemoteRepository): """ Sync collaborators of a repository with the database. This method will remove collaborators that are no longer in the list. """ collaborators = { - collaborator.id: collaborator - for collaborator in gh_repo.get_collaborators() + collaborator.id: collaborator for collaborator in gh_repo.get_collaborators() } remote_repo_relations_ids = [] for account in self._get_social_accounts(collaborators.keys()): @@ -333,9 +321,7 @@ def _resync_collaborators( user=account.user, account=account, ) - remote_repo_relation.admin = collaborators[ - int(account.uid) - ].permissions.admin + remote_repo_relation.admin = collaborators[int(account.uid)].permissions.admin remote_repo_relation.save() remote_repo_relations_ids.append(remote_repo_relation.pk) @@ -353,9 +339,7 @@ def _get_social_accounts(self, ids): provider=self.allauth_provider.id, ).select_related("user") - def _resync_organization_members( - self, gh_org: GHOrganization, remote_org: RemoteOrganization - ): + def _resync_organization_members(self, gh_org: GHOrganization, remote_org: RemoteOrganization): """ Sync members of an organization with the database. @@ -399,9 +383,7 @@ def send_build_status(self, *, build, commit, status): try: # NOTE: we use the lazy option to avoid fetching the repository object, # since we only need the object to interact with the commit status API. - gh_repo = self.installation_client.get_repo( - int(remote_repo.remote_id), lazy=True - ) + gh_repo = self.installation_client.get_repo(int(remote_repo.remote_id), lazy=True) gh_repo.get_commit(commit).create_status( state=state, target_url=target_url, @@ -434,9 +416,7 @@ def get_clone_token(self, project): # We can also pass a specific permissions object to get a token with specific permissions # if we want to scope this token even more. try: - access_token = self.gha_client.get_access_token( - self.installation.installation_id - ) + access_token = self.gha_client.get_access_token(self.installation.installation_id) return f"x-access-token:{access_token.token}" except GithubException: log.info( diff --git a/readthedocs/oauth/urls.py b/readthedocs/oauth/urls.py index a464d303c16..6068fcac221 100644 --- a/readthedocs/oauth/urls.py +++ b/readthedocs/oauth/urls.py @@ -2,6 +2,7 @@ from readthedocs.oauth.views import GitHubAppWebhookView + urlpatterns = [ path("githubapp/", GitHubAppWebhookView.as_view(), name="github_app_webhook"), ] diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index bef3edd83d6..04654a0e442 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -7,24 +7,22 @@ from rest_framework.response import Response from rest_framework.views import APIView -from readthedocs.api.v2.views.integrations import ( - GITHUB_EVENT_HEADER, - GITHUB_SIGNATURE_HEADER, - ExternalVersionData, - WebhookMixin, -) -from readthedocs.builds.constants import BRANCH, TAG -from readthedocs.core.views.hooks import ( - build_external_version, - build_versions_from_names, - close_external_version, - get_or_create_external_version, - trigger_sync_versions, -) +from readthedocs.api.v2.views.integrations import GITHUB_EVENT_HEADER +from readthedocs.api.v2.views.integrations import GITHUB_SIGNATURE_HEADER +from readthedocs.api.v2.views.integrations import ExternalVersionData +from readthedocs.api.v2.views.integrations import WebhookMixin +from readthedocs.builds.constants import BRANCH +from readthedocs.builds.constants import TAG +from readthedocs.core.views.hooks import build_external_version +from readthedocs.core.views.hooks import build_versions_from_names +from readthedocs.core.views.hooks import close_external_version +from readthedocs.core.views.hooks import get_or_create_external_version +from readthedocs.core.views.hooks import trigger_sync_versions from readthedocs.oauth.models import GitHubAppInstallation from readthedocs.oauth.services.githubapp import get_gh_app_client from readthedocs.projects.models import Project + log = structlog.get_logger(__name__) @@ -254,9 +252,7 @@ def _handle_installation_repositories_event(self): return if action == "removed": - installation.delete_repositories( - [repo["id"] for repo in data["repositories_removed"]] - ) + installation.delete_repositories([repo["id"] for repo in data["repositories_removed"]]) return # NOTE: this should never happen. @@ -305,9 +301,7 @@ def _handle_repository_event(self): return if action in ("edited", "privatized", "publicized", "renamed", "transferred"): - installation.service.update_or_create_repositories( - [data["repository"]["id"]] - ) + installation.service.update_or_create_repositories([data["repository"]["id"]]) return # Ignore other actions: @@ -466,9 +460,7 @@ def _handle_member_event(self): if action in ("added", "edited", "removed"): # Sync collaborators - installation.service.update_or_create_repositories( - [data["repository"]["id"]] - ) + installation.service.update_or_create_repositories([data["repository"]["id"]]) return # NOTE: this should never happen. diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 9d6a578e97d..17917838fad 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1034,7 +1034,8 @@ def get_git_service_class(self, fallback_to_clone_url=False): @property def is_github_project(self): - from readthedocs.oauth.services import GitHubAppService, GitHubService + from readthedocs.oauth.services import GitHubAppService + from readthedocs.oauth.services import GitHubService return self.get_git_service_class(fallback_to_clone_url=True) in [ GitHubService, From d10627793bc69e9fccf0b49f9f6c7d2829f683bb Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Mar 2025 12:06:14 -0500 Subject: [PATCH 66/92] Update migration --- .../oauth/migrations/0018_githubapp.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/readthedocs/oauth/migrations/0018_githubapp.py b/readthedocs/oauth/migrations/0018_githubapp.py index d46b7443c8a..e8b28ec0924 100644 --- a/readthedocs/oauth/migrations/0018_githubapp.py +++ b/readthedocs/oauth/migrations/0018_githubapp.py @@ -70,6 +70,7 @@ class Migration(migrations.Migration): ], options={ "get_latest_by": "modified", + "verbose_name": "GitHub app installation", "abstract": False, }, ), @@ -85,4 +86,32 @@ class Migration(migrations.Migration): verbose_name="GitHub App Installation", ), ), + migrations.AlterField( + model_name="remoteorganization", + name="vcs_provider", + field=models.CharField( + choices=[ + ("github", "GitHub"), + ("githubapp", "GitHub"), + ("gitlab", "GitLab"), + ("bitbucket", "Bitbucket"), + ], + max_length=32, + verbose_name="VCS provider", + ), + ), + migrations.AlterField( + model_name="remoterepository", + name="vcs_provider", + field=models.CharField( + choices=[ + ("github", "GitHub"), + ("githubapp", "GitHub"), + ("gitlab", "GitLab"), + ("bitbucket", "Bitbucket"), + ], + max_length=32, + verbose_name="VCS provider", + ), + ), ] From c2a89d7208d6e90086ec93b25f55a8d60cf73792 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Mar 2025 13:07:45 -0500 Subject: [PATCH 67/92] Remove suspended installations --- readthedocs/oauth/services/githubapp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 543770498a3..21dea8bfefb 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -174,9 +174,10 @@ def sync(self): exc_info=True, ) - if e.status == 404: - # The installation is no longer accessible, - # we remove it from the database. + # 404: installation was deleted/uninstalled + # 403: installation was suspended + if e.status in [404, 403]: + # The installation is no longer accessible, we remove it from the database. self.installation.delete() raise SyncServiceError() From 7fffcea37ed7a4519556c4622bd3da130237491c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Mar 2025 13:15:22 -0500 Subject: [PATCH 68/92] Delete installation when suspended --- .../oauth/tests/test_githubapp_webhook.py | 74 +++++++++++++++++++ readthedocs/oauth/views.py | 7 +- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/readthedocs/oauth/tests/test_githubapp_webhook.py b/readthedocs/oauth/tests/test_githubapp_webhook.py index d074d3673ff..62607945daf 100644 --- a/readthedocs/oauth/tests/test_githubapp_webhook.py +++ b/readthedocs/oauth/tests/test_githubapp_webhook.py @@ -129,6 +129,48 @@ def test_installation_created_with_existing_installation(self, sync): assert self.installation.target_type == GitHubAccountType.USER assert GitHubAppInstallation.objects.count() == 1 + @mock.patch.object(GitHubAppService, "sync") + def test_installation_unsuspended(self, sync): + new_installation_id = 2222 + assert not GitHubAppInstallation.objects.filter( + installation_id=new_installation_id + ).exists() + payload = { + "action": "unsuspended", + "installation": { + "id": new_installation_id, + "target_id": 2222, + "target_type": GitHubAccountType.USER, + }, + } + r = self.post_webhook("installation", payload) + assert r.status_code == 200 + + installation = GitHubAppInstallation.objects.get( + installation_id=new_installation_id + ) + assert installation.target_id == 2222 + assert installation.target_type == GitHubAccountType.USER + sync.assert_called_once() + + @mock.patch.object(GitHubAppService, "sync") + def test_installation_unsuspended_with_existing_installation(self, sync): + paylod = { + "action": "unsuspended", + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + } + r = self.post_webhook("installation", paylod) + assert r.status_code == 200 + sync.assert_called_once() + self.installation.refresh_from_db() + assert self.installation.target_id == 1111 + assert self.installation.target_type == GitHubAccountType.USER + assert GitHubAppInstallation.objects.count() == 1 + def test_installation_deleted(self): payload = { "action": "deleted", @@ -161,6 +203,38 @@ def test_installation_deleted_with_non_existing_installation(self): assert r.status_code == 200 assert not GitHubAppInstallation.objects.filter(installation_id=2222).exists() + def test_installation_suspended(self): + payload = { + "action": "suspended", + "installation": { + "id": self.installation.installation_id, + "target_id": self.installation.target_id, + "target_type": self.installation.target_type, + }, + } + r = self.post_webhook("installation", payload) + assert r.status_code == 200 + assert not GitHubAppInstallation.objects.filter( + installation_id=self.installation.installation_id + ).exists() + + def test_installation_suspended_with_non_existing_installation(self): + install_id = 2222 + assert not GitHubAppInstallation.objects.filter( + installation_id=install_id + ).exists() + payload = { + "action": "suspended", + "installation": { + "id": install_id, + "target_id": 2222, + "target_type": GitHubAccountType.USER, + }, + } + r = self.post_webhook("installation", payload) + assert r.status_code == 200 + assert not GitHubAppInstallation.objects.filter(installation_id=2222).exists() + def test_installation_new_permissions_accepted(self): payload = { "action": "new_permissions_accepted", diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 04654a0e442..4eadaafd4da 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -143,15 +143,15 @@ def _handle_installation_event(self): gh_installation = data["installation"] installation_id = gh_installation["id"] - if action == "created": + if action in ["created", "unsuspended"]: installation, created = self._get_or_create_installation() # If the installation was just created, we already synced the repositories. if created: return installation.service.sync() - if action == "deleted": - # NOTE: When an installation is deleted, this doesn't trigger an installation_repositories event. + if action in ["deleted", "suspended"]: + # NOTE: When an installation is deleted/suspended, it doesn't trigger an installation_repositories event. # So we need to call the delete method explicitly here, so we delete its repositories. installation = GitHubAppInstallation.objects.filter( installation_id=installation_id @@ -165,7 +165,6 @@ def _handle_installation_event(self): # Ignore other actions: # - new_permissions_accepted: We don't need to do anything here for now. - # - suspended/unsuspended: We don't do anything with suspended installations. return def _handle_installation_repositories_event(self): From 0adc4b489c5bad2ef5065ea338695deb64a08756 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 11 Mar 2025 13:30:22 -0500 Subject: [PATCH 69/92] Handle authorization event --- readthedocs/oauth/views.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 4eadaafd4da..d5912a6cd63 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -467,13 +467,19 @@ def _handle_member_event(self): def _handle_github_app_authorization_event(self): """ - Revoking the authorization of a GitHub App does not uninstall the GitHub App. - You should program your GitHub App so that when it receives this webhook, - it stops calling the API on behalf of the person who revoked the token. + Handle the github_app_authorization event. + + Triggered when a user revokes the authorization of a GitHub App ("log in with GitHub" will no longer work). + + .. note:: + + Revoking the authorization of a GitHub App does not uninstall the GitHub App, + it only revokes the OAuth2 token. See https://docs.github.com/en/webhooks/webhook-events-and-payloads#github_app_authorization. """ - # TODO: what to do here? + # A GitHub App receives this webhook by default and cannot unsubscribe from this event. + # We don't need to do anything here for now. def _get_projects(self): """ From 7e3f3f4d820cbfcb49d23009805360aa0e60f5ba Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Mar 2025 12:11:46 -0500 Subject: [PATCH 70/92] Tests --- readthedocs/oauth/services/githubapp.py | 21 +- .../oauth/tests/test_githubapp_webhook.py | 9 +- readthedocs/rtd_tests/tests/test_oauth.py | 288 ++++++++++++++++++ .../rtd_tests/tests/test_oauth_sync.py | 1 - requirements/deploy.txt | 16 +- requirements/docker.txt | 21 +- requirements/docs.txt | 10 +- requirements/pip.txt | 9 +- requirements/testing.txt | 18 +- 9 files changed, 321 insertions(+), 72 deletions(-) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 21dea8bfefb..05ece8c800c 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -139,6 +139,7 @@ def sync_user_access(cls, user): we first sync the repositories from all installations accessible to the user (refresh access to new repositories), and then we sync each repository the user has access to (check if the user lost access to a repository, or his access level changed). """ + # TODO: don't stop at the first exception. # Refresh access to all installations accessible to the user. for service in cls.for_user(user): service.sync() @@ -173,14 +174,22 @@ def sync(self): installation_id=self.installation.installation_id, exc_info=True, ) - - # 404: installation was deleted/uninstalled - # 403: installation was suspended - if e.status in [404, 403]: - # The installation is no longer accessible, we remove it from the database. + if e.status == 404: + # The app was uninstalled, we remove the installation from the database. self.installation.delete() raise SyncServiceError() + if app_installation.suspended_at is not None: + log.info( + "Installation is suspended", + installation_id=self.installation.installation_id, + suspended_at=app_installation.suspended_at, + ) + # The installation is suspended, we don't have access to it anymore, + # so we just delete it from the database. + self.installation.delete() + raise SyncServiceError() + remote_repositories = [] for repo in app_installation.get_repos(): remote_repo = self._create_or_update_repository_from_gh(repo) @@ -206,7 +215,7 @@ def update_or_create_repositories(self, repository_ids: list[int]): # if we lost access to the repository, # we remove the repository from the database, # and clean up the collaborators and relations. - if e.status == 404: + if e.status in [404, 403]: self.installation.delete_repositories([repository_id]) continue self._create_or_update_repository_from_gh(repo) diff --git a/readthedocs/oauth/tests/test_githubapp_webhook.py b/readthedocs/oauth/tests/test_githubapp_webhook.py index 62607945daf..cce303a435e 100644 --- a/readthedocs/oauth/tests/test_githubapp_webhook.py +++ b/readthedocs/oauth/tests/test_githubapp_webhook.py @@ -846,4 +846,11 @@ def test_member_removed(self, update_or_create_repositories): ) def test_github_app_authorization(self): - pass + payload = { + "action": "revoked", + "sender": { + "login": "user", + }, + } + r = self.post_webhook("github_app_authorization", payload) + assert r.status_code == 200 diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index a4a060c1714..f83815c2527 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -30,6 +30,7 @@ RemoteRepositoryRelation, ) from readthedocs.oauth.services import BitbucketService, GitHubService, GitLabService +from readthedocs.oauth.services.base import SyncServiceError from readthedocs.oauth.services.githubapp import GitHubAppService from readthedocs.projects import constants from readthedocs.projects.models import Project @@ -81,6 +82,7 @@ def setUp(self): RemoteOrganization, slug="org", remote_id="2222", + vcs_provider=GITHUB_APP, ) get( RemoteOrganizationRelation, @@ -189,6 +191,38 @@ def _get_collaborator_json(self, **kwargs): self._merge_dicts(default, kwargs) return default + def _get_organization_json(self, **kwargs): + default = { + "login": "org", + "id": 2222, + "url": "https://api.github.com/orgs/org", + "repos_url": "https://api.github.com/orgs/org/repos", + "events_url": "https://api.github.com/orgs/org/events", + "hooks_url": "https://api.github.com/orgs/org/hooks", + "issues_url": "https://api.github.com/orgs/org/issues", + "members_url": "https://api.github.com/orgs/org/members{/member}", + "public_members_url": "https://api.github.com/orgs/org/public_members{/member}", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "description": "Some organization", + "name": "Organization", + "email": "org@example.com", + "html_url": "https://github.com/org", + "created_at": "2008-01-14T04:33:35Z", + "type": "Organization", + "billing_email": "mona@github.com", + } + self._merge_dicts(default, kwargs) + return default + + def _get_user_json(self, **kwargs): + default = { + "login": "user", + "id": 1111, + "type": "User", + } + self._merge_dicts(default, kwargs) + return default + def _get_commit_json(self, commit, **kwargs): default = { "url": f"https://api.github.com/repos/user/repo/commits/{commit}", @@ -394,6 +428,23 @@ def test_update_repository(self, request): self.remote_repository.refresh_from_db() assert self.remote_repository.description == "New description" + @requests_mock.Mocker(kw="request") + def test_update_invalid_repository(self, request): + request.post( + f"{self.api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{self.api_url}/repositories/{self.remote_repository.remote_id}", + status_code=404, + ) + + service = self.installation.service + service.update_or_create_repositories([int(self.remote_repository.remote_id)]) + assert not RemoteRepository.objects.filter( + id=self.remote_repository.id + ).exists() + @requests_mock.Mocker(kw="request") def test_sync(self, request): assert self.installation.repositories.count() == 1 @@ -439,6 +490,9 @@ def test_sync(self, request): repo = self.installation.repositories.get(full_name="user/repo") assert repo.name == "repo" assert repo.remote_id == self.remote_repository.remote_id + assert repo.default_branch == "main" + assert repo.vcs_provider == GITHUB_APP + assert repo.private is False assert repo.remote_repository_relations.count() == 1 relation = repo.remote_repository_relations.first() assert relation.user == self.user @@ -449,6 +503,9 @@ def test_sync(self, request): assert repo.name == "repo2" assert repo.remote_id == "2222" assert repo.remote_repository_relations.count() == 1 + assert repo.default_branch == "main" + assert repo.vcs_provider == GITHUB_APP + assert repo.private is False relation = repo.remote_repository_relations.first() assert relation.user == self.user assert relation.account == self.account @@ -458,10 +515,241 @@ def test_sync(self, request): assert repo.name == "repo3" assert repo.remote_id == "3333" assert repo.remote_repository_relations.count() == 1 + assert repo.default_branch == "main" + assert repo.vcs_provider == GITHUB_APP + assert repo.private is False + relation = repo.remote_repository_relations.first() + assert relation.user == self.user + assert relation.account == self.account + assert relation.admin + + @requests_mock.Mocker(kw="request") + def test_sync_delete_remote_repositories(self, request): + assert self.installation.repositories.count() == 1 + request.get( + f"{self.api_url}/app/installations/1111", + json=self._get_installation_json(id=1111), + ) + request.post( + f"{self.api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{self.api_url}/installation/repositories", + json={ + "repositories": [ + self._get_repository_json(full_name="user/repo2", id=2222), + ] + }, + ) + request.get( + f"{self.api_url}/repos/user/repo2/collaborators", + json=[self._get_collaborator_json()], + ) + + service = self.installation.service + service.sync() + + assert self.installation.repositories.count() == 1 + repo = self.installation.repositories.get(full_name="user/repo2") + assert repo.name == "repo2" + assert repo.remote_id == "2222" + assert repo.remote_repository_relations.count() == 1 + assert repo.default_branch == "main" + assert repo.vcs_provider == GITHUB_APP + assert repo.private is False + relation = repo.remote_repository_relations.first() + assert relation.user == self.user + assert relation.account == self.account + assert relation.admin + + @requests_mock.Mocker(kw="request") + def test_sync_repositories_with_organization(self, request): + assert self.organization_installation.repositories.count() == 1 + request.get( + f"{self.api_url}/app/installations/{self.organization_installation.installation_id}", + json=self._get_installation_json( + id=self.organization_installation.installation_id, + account={"id": 2222, "login": "org", "type": "Organization"}, + target_type="Organization", + target_id=2222, + ), + ) + request.post( + f"{self.api_url}/app/installations/{self.organization_installation.installation_id}/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{self.api_url}/installation/repositories", + json={ + "repositories": [ + self._get_repository_json( + full_name="org/repo2", + id=2222, + owner={"id": 2222, "login": "org", "type": "Organization"}, + ), + ] + }, + ) + request.get( + f"{self.api_url}/repos/org/repo2/collaborators", + json=[self._get_collaborator_json()], + ) + request.get( + f"{self.api_url}/orgs/org", + json=self._get_organization_json(login="org", id=2222), + ) + request.get( + f"{self.api_url}/orgs/org/members", + json=[self._get_user_json(id=1111, login="user")], + ) + + service = self.organization_installation.service + service.sync() + + assert self.organization_installation.repositories.count() == 1 + repo = self.organization_installation.repositories.get(full_name="org/repo2") + assert repo.name == "repo2" + assert repo.remote_id == "2222" + assert repo.remote_repository_relations.count() == 1 + assert repo.default_branch == "main" + assert repo.vcs_provider == GITHUB_APP + assert repo.private is False relation = repo.remote_repository_relations.first() assert relation.user == self.user assert relation.account == self.account assert relation.admin + remote_organization = repo.organization + assert remote_organization.slug == "org" + assert remote_organization.remote_id == "2222" + assert remote_organization.name == "Organization" + assert remote_organization.email == "org@example.com" + assert remote_organization.url == "https://github.com/org" + assert ( + remote_organization.avatar_url + == "https://github.com/images/error/octocat_happy.gif" + ) + assert remote_organization.vcs_provider == GITHUB_APP + + @requests_mock.Mocker(kw="request") + def test_sync_repo_moved_from_org_to_user(self, request): + assert self.installation.repositories.count() == 1 + assert self.organization_installation.repositories.count() == 1 + assert self.remote_repository_with_org.organization == self.remote_organization + assert ( + self.remote_repository_with_org.github_app_installation + == self.organization_installation + ) + + request.get( + f"{self.api_url}/app/installations/1111", + json=self._get_installation_json(id=1111), + ) + request.post( + f"{self.api_url}/app/installations/1111/access_tokens", + json=self._get_access_token_json(), + ) + + request.get( + f"{self.api_url}/installation/repositories", + json={ + "repositories": [ + self._get_repository_json( + full_name="user/repo", id=int(self.remote_repository.remote_id) + ), + self._get_repository_json( + full_name="user/repo2", + id=int(self.remote_repository_with_org.remote_id), + ), + ] + }, + ) + request.get( + f"{self.api_url}/repos/user/repo/collaborators", + json=[self._get_collaborator_json()], + ) + request.get( + f"{self.api_url}/repos/user/repo2/collaborators", + json=[self._get_collaborator_json()], + ) + + service = self.installation.service + service.sync() + assert self.installation.repositories.count() == 2 + assert self.organization_installation.repositories.count() == 0 + + self.remote_repository_with_org.refresh_from_db() + assert self.remote_repository_with_org.organization is None + assert ( + self.remote_repository_with_org.github_app_installation == self.installation + ) + assert self.remote_repository_with_org.name == "repo2" + assert self.remote_repository_with_org.full_name == "user/repo2" + + @requests_mock.Mocker(kw="request") + def test_sync_orphan_organization_is_deleted(self, request): + assert self.organization_installation.repositories.count() == 1 + request.get( + f"{self.api_url}/app/installations/{self.organization_installation.installation_id}", + json=self._get_installation_json( + id=self.organization_installation.installation_id, + account={"id": 2222, "login": "org", "type": "Organization"}, + target_type="Organization", + target_id=2222, + ), + ) + request.post( + f"{self.api_url}/app/installations/{self.organization_installation.installation_id}/access_tokens", + json=self._get_access_token_json(), + ) + request.get( + f"{self.api_url}/installation/repositories", + json={"repositories": []}, + ) + + service = self.organization_installation.service + service.sync() + + assert self.organization_installation.repositories.count() == 0 + assert not RemoteOrganization.objects.filter( + id=self.remote_organization.id + ).exists() + + @requests_mock.Mocker(kw="request") + def test_sync_with_uninstalled_installation(self, request): + assert self.installation.repositories.count() == 1 + request.get( + f"{self.api_url}/app/installations/1111", + status_code=404, + ) + service = self.installation.service + with self.assertRaises(SyncServiceError): + service.sync() + + assert not GitHubAppInstallation.objects.filter( + id=self.installation.id + ).exists() + assert not RemoteRepository.objects.filter( + id=self.remote_repository.id + ).exists() + + @requests_mock.Mocker(kw="request") + def test_sync_with_suspended_installation(self, request): + assert self.installation.repositories.count() == 1 + request.get( + f"{self.api_url}/app/installations/1111", + json=self._get_installation_json(id=1111, suspended_at="2021-01-01T00:00:00Z"), + ) + service = self.installation.service + with self.assertRaises(SyncServiceError): + service.sync() + + assert not GitHubAppInstallation.objects.filter( + id=self.installation.id + ).exists() + assert not RemoteRepository.objects.filter( + id=self.remote_repository.id + ).exists() @requests_mock.Mocker(kw="request") def test_send_build_status_pending(self, request): diff --git a/readthedocs/rtd_tests/tests/test_oauth_sync.py b/readthedocs/rtd_tests/tests/test_oauth_sync.py index e8b270ca3d0..5773f6e6357 100644 --- a/readthedocs/rtd_tests/tests/test_oauth_sync.py +++ b/readthedocs/rtd_tests/tests/test_oauth_sync.py @@ -16,7 +16,6 @@ from readthedocs.projects.models import Project -# TODO: port these tests to the GitHub app. class GitHubOAuthSyncTests(TestCase): payload_user_repos = [ { diff --git a/requirements/deploy.txt b/requirements/deploy.txt index 506f4c12c0a..4ddbcdeef94 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/deploy.txt requirements/deploy.in @@ -20,10 +20,6 @@ asgiref==3.8.1 # django-cors-headers asttokens==3.0.0 # via stack-data -async-timeout==5.0.1 - # via - # -r requirements/pip.txt - # redis billiard==4.2.1 # via # -r requirements/pip.txt @@ -218,8 +214,6 @@ elasticsearch-dsl==8.17.1 # via # -r requirements/pip.txt # django-elasticsearch-dsl -exceptiongroup==1.2.2 - # via ipython executing==2.2.0 # via stack-data fido2==1.2.0 @@ -327,7 +321,7 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic -pygithub==2.5.0 +pygithub==2.6.1 # via -r requirements/pip.txt pygments==2.19.1 # via @@ -430,10 +424,6 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver -tomli==2.2.1 - # via - # -r requirements/pip.txt - # dparse traitlets==5.14.3 # via # ipython @@ -441,9 +431,7 @@ traitlets==5.14.3 typing-extensions==4.12.2 # via # -r requirements/pip.txt - # asgiref # elasticsearch-dsl - # ipython # psycopg # psycopg-pool # pydantic diff --git a/requirements/docker.txt b/requirements/docker.txt index f3cedcb12cd..239961421e7 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/docker.txt requirements/docker.in @@ -20,10 +20,6 @@ asgiref==3.8.1 # django-cors-headers asttokens==3.0.0 # via stack-data -async-timeout==5.0.1 - # via - # -r requirements/pip.txt - # redis attrs==25.1.0 # via wmctrl billiard==4.2.1 @@ -228,8 +224,6 @@ elasticsearch-dsl==8.17.1 # via # -r requirements/pip.txt # django-elasticsearch-dsl -exceptiongroup==1.2.2 - # via ipython executing==2.2.0 # via stack-data fancycompleter==0.9.1 @@ -351,7 +345,7 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic -pygithub==2.5.0 +pygithub==2.6.1 # via -r requirements/pip.txt pygments==2.19.1 # via @@ -457,13 +451,6 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver -tomli==2.2.1 - # via - # -r requirements/pip.txt - # dparse - # ipdb - # pyproject-api - # tox tox==4.24.1 # via -r requirements/docker.in traitlets==5.14.3 @@ -473,16 +460,12 @@ traitlets==5.14.3 typing-extensions==4.12.2 # via # -r requirements/pip.txt - # asgiref # elasticsearch-dsl - # ipython # psycopg # psycopg-pool # pydantic # pydantic-core # pygithub - # rich - # tox tzdata==2025.1 # via # -r requirements/pip.txt diff --git a/requirements/docs.txt b/requirements/docs.txt index b0ebd7d6cbe..d1f58aa56c2 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/docs.txt requirements/docs.in @@ -37,8 +37,6 @@ docutils==0.21.2 # sphinx-prompt # sphinx-rtd-theme # sphinx-tabs -exceptiongroup==1.2.2 - # via anyio fonttools==4.55.8 # via matplotlib h11==0.14.0 @@ -161,12 +159,8 @@ sphinxext-opengraph==0.9.1 # via -r requirements/docs.in starlette==0.46.0 # via sphinx-autobuild -tomli==2.2.1 - # via sphinx typing-extensions==4.12.2 - # via - # anyio - # uvicorn + # via anyio urllib3==2.3.0 # via # requests diff --git a/requirements/pip.txt b/requirements/pip.txt index daec9523309..e931a4d7ad0 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/pip.txt requirements/pip.in @@ -13,8 +13,6 @@ asgiref==3.8.1 # django # django-allauth # django-cors-headers -async-timeout==5.0.1 - # via redis billiard==4.2.1 # via celery boto3==1.36.26 @@ -234,7 +232,7 @@ pydantic==2.10.6 # via -r requirements/pip.in pydantic-core==2.27.2 # via pydantic -pygithub==2.5.0 +pygithub==2.6.1 # via -r requirements/pip.in pygments==2.19.1 # via -r requirements/pip.in @@ -313,11 +311,8 @@ structlog==23.2.0 # django-structlog toml==0.10.2 # via bumpver -tomli==2.2.1 - # via dparse typing-extensions==4.12.2 # via - # asgiref # elasticsearch-dsl # psycopg # psycopg-pool diff --git a/requirements/testing.txt b/requirements/testing.txt index 0c4c030f837..ae05239ca9b 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/testing.txt requirements/testing.in @@ -20,10 +20,6 @@ asgiref==3.8.1 # django # django-allauth # django-cors-headers -async-timeout==5.0.1 - # via - # -r requirements/pip.txt - # redis babel==2.17.0 # via sphinx billiard==4.2.1 @@ -227,8 +223,6 @@ elasticsearch-dsl==8.17.1 # via # -r requirements/pip.txt # django-elasticsearch-dsl -exceptiongroup==1.2.2 - # via pytest fido2==1.2.0 # via # -r requirements/pip.txt @@ -329,7 +323,7 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic -pygithub==2.5.0 +pygithub==2.6.1 # via -r requirements/pip.txt pygments==2.19.1 # via @@ -461,17 +455,9 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver -tomli==2.2.1 - # via - # -r requirements/pip.txt - # coverage - # dparse - # pytest - # sphinx typing-extensions==4.12.2 # via # -r requirements/pip.txt - # asgiref # elasticsearch-dsl # psycopg # psycopg-pool From fa1b70cdc517df627688e99c43d984ac3bc6f1a7 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 12 Mar 2025 12:38:21 -0500 Subject: [PATCH 71/92] Small changes --- readthedocs/oauth/services/githubapp.py | 12 ++++++++++-- readthedocs/oauth/views.py | 9 +++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index 05ece8c800c..bc921b3268d 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -139,10 +139,15 @@ def sync_user_access(cls, user): we first sync the repositories from all installations accessible to the user (refresh access to new repositories), and then we sync each repository the user has access to (check if the user lost access to a repository, or his access level changed). """ - # TODO: don't stop at the first exception. + has_error = False # Refresh access to all installations accessible to the user. for service in cls.for_user(user): - service.sync() + try: + service.sync() + except SyncServiceError: + # Don't stop the sync if one installation fails, + # as we should try to sync all installations. + has_error = True # Update the access to each repository the user has access to. queryset = RemoteRepository.objects.filter( @@ -156,6 +161,9 @@ def sync_user_access(cls, user): # TODO: maybe also refresh the organizations the user has access to? # But doesn't look like we are using that relation for anything? + if has_error: + raise SyncServiceError() + def sync(self): """ Sync all repositories and organizations that are accessible to the installation. diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index d5912a6cd63..27682d897e2 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -27,6 +27,15 @@ class GitHubAppWebhookView(APIView): + """ + Handle GitHub App webhooks. + + All event handlers try to create the installation object if it doesn't exist in our database, + except for events related to the installation being deleted or suspended. + This guarantees that our application can easily recover if we missed an event + in case our application or GitHub were down. + """ + authentication_classes = [] @cached_property From 339cf7378a14e310b98763a706399eca94d98fd4 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 18 Mar 2025 17:37:22 -0500 Subject: [PATCH 72/92] Complete migrate view --- readthedocs/oauth/migrate.py | 5 ++ readthedocs/oauth/services/githubapp.py | 7 ++- readthedocs/oauth/signals.py | 8 +++ readthedocs/profiles/views.py | 81 ++++++++++++++++++++----- 4 files changed, 83 insertions(+), 18 deletions(-) diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index 83e929f6199..37c683d6b66 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -1,3 +1,5 @@ +"""This module contains the logic to help users migrate from the GitHub OAuth App to the GitHub App.""" + from dataclasses import dataclass from allauth.socialaccount.providers.github.provider import GitHubProvider @@ -141,6 +143,9 @@ def get_migration_targets(user): # NOTE: there are some users that have more than one GH account connected. # so this isn't 100% accurate. gh_account = user.socialaccount_set.filter(provider=GITHUB).first() + if not gh_account: + return targets + for project, has_installation, is_admin in _get_projects_for_user(user): remote_repository = project.remote_repository diff --git a/readthedocs/oauth/services/githubapp.py b/readthedocs/oauth/services/githubapp.py index bc921b3268d..22e2a546eeb 100644 --- a/readthedocs/oauth/services/githubapp.py +++ b/readthedocs/oauth/services/githubapp.py @@ -156,7 +156,7 @@ def sync_user_access(cls, user): ) for repository in queryset: service = cls(repository.github_app_installation) - service.update_or_create_repositories([repository.remote_id]) + service.update_or_create_repositories([int(repository.remote_id)]) # TODO: maybe also refresh the organizations the user has access to? # But doesn't look like we are using that relation for anything? @@ -213,7 +213,10 @@ def update_or_create_repositories(self, repository_ids: list[int]): """Update or create repositories from the given list of repository IDs.""" for repository_id in repository_ids: try: - repo = self.installation_client.get_repo(repository_id) + # NOTE: we save the repository ID as a string in our database, + # in order for PyGithub to use the API to fetch the repository by ID (not by name). + # it needs to be an integer, so just in case we cast it to an integer. + repo = self.installation_client.get_repo(int(repository_id)) except GithubException as e: log.info( "Failed to fetch repository from GitHub", diff --git a/readthedocs/oauth/signals.py b/readthedocs/oauth/signals.py index 5ef23940682..1fcbb1eaad9 100644 --- a/readthedocs/oauth/signals.py +++ b/readthedocs/oauth/signals.py @@ -1,5 +1,7 @@ import structlog from allauth.account.signals import user_logged_in +from allauth.socialaccount.models import SocialLogin +from allauth.socialaccount.signals import social_account_added from django.contrib.auth.models import User from django.db.models.signals import post_save from django.dispatch import receiver @@ -28,6 +30,12 @@ def sync_remote_repositories_on_login(sender, request, user, *args, **kwargs): sync_remote_repositories.delay(user.pk) +@receiver(social_account_added, sender=SocialLogin) +def sync_remote_repositories_on_social_account_added(sender, request, sociallogin, *args, **kwargs): + """Sync remote repositories when a new social account is added.""" + sync_remote_repositories.delay(sociallogin.user.pk) + + @receiver(post_save, sender=RemoteRepository) def update_project_clone_url(sender, instance, created, *args, **kwargs): """Update the clone URL for all projects linked to this RemoteRepository.""" diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index fa10bff473e..d5344cc4746 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -1,8 +1,10 @@ """Views for creating, editing and viewing site-specific user profiles.""" +from enum import StrEnum +from enum import auto + from allauth.account.views import LoginView as AllAuthLoginView from allauth.account.views import LogoutView as AllAuthLogoutView -from allauth.socialaccount.adapter import get_adapter as get_social_account_adapter from allauth.socialaccount.providers.github.provider import GitHubProvider from django.conf import settings from django.contrib import messages @@ -34,6 +36,8 @@ from readthedocs.core.models import UserProfile from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.extend import SettingsOverrideObject +from readthedocs.oauth.clients import get_oauth2_client +from readthedocs.oauth.constants import GITHUB_APP from readthedocs.oauth.migrate import get_installation_target_groups_for_user from readthedocs.oauth.migrate import get_migration_targets from readthedocs.oauth.migrate import get_old_app_link @@ -289,35 +293,80 @@ def get_queryset(self): return self.filter.qs +class MigrationSteps(StrEnum): + overview = auto() + connect = auto() + install = auto() + migrate = auto() + revoke = auto() + disconnect = auto() + + class MigrateToGitHubAppView(PrivateViewMixin, TemplateView): - template_name = "profiles/private/migrate-to-gh-app.html" + template_name = "profiles/private/migrate_to_gh_app.html" def get(self, request, *args, **kwargs): - # TODO: check if the user already migrated all their projects, - # or if the user doesn't have projects to migrate. + if self._get_old_github_account() is None: + if self._get_new_github_account(): + msg = _("You have already migrated your account to the new GitHub App.") + else: + msg = _("You don't have any GitHub account connected.") + messages.info(request, msg) + return HttpResponseRedirect(reverse("homepage")) + return super().get(request, *args, **kwargs) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - user = self.request.user - # NOTE: I tried passing the GitHubAppProvider class directly to the template, - # but it doesn't work for some reason. - context["gh_app_provider"] = get_social_account_adapter().get_provider( - request=self.request, provider=GitHubAppProvider.id - ) - context["gh_provider"] = get_social_account_adapter().get_provider( - request=self.request, provider=GitHubProvider.id - ) + step = self.request.GET.get("step", MigrationSteps.overview) + if step not in MigrationSteps: + step = MigrationSteps.overview + context["step"] = step - context["has_gh_app_social_account"] = user.socialaccount_set.filter( - provider=GitHubAppProvider.id - ).exists() + user = self.request.user + + context["step_connect_completed"] = self._has_new_account_for_old_account() context["installation_target_groups"] = get_installation_target_groups_for_user(user) + context["gh_app_name"] = settings.GITHUB_APP_NAME context["migration_targets"] = get_migration_targets(user) + context["migrated_projects"] = ( + AdminPermission.projects(user, admin=True) + .filter(remote_repository__vcs_provider=GITHUB_APP) + .select_related( + "remote_repository", + ) + ) context["old_application_link"] = get_old_app_link() + context["step_revoke_completed"] = self._is_access_to_old_github_account_revoked() + context["old_github_account"] = self._get_old_github_account() return context + def _is_access_to_old_github_account_revoked(self): + old_account = self._get_old_github_account() + client = get_oauth2_client(old_account) + if client is None: + return True + + resp = client.get("https://api.github.com/user") + if resp.status_code == 401: + return True + + return False + + def _has_new_account_for_old_account(self): + old_account = self._get_old_github_account() + return self.request.user.socialaccount_set.filter( + provider=GitHubAppProvider.id, + uid=old_account.uid, + ).exists() + + def _get_new_github_account(self): + return self.request.user.socialaccount_set.filter(provider=GitHubAppProvider.id).first() + + def _get_old_github_account(self): + return self.request.user.socialaccount_set.filter(provider=GitHubProvider.id).first() + def post(self, request, *args, **kwargs): project_slug = request.POST.get("project") if project_slug: From 7ff1e961d6bb21d1a850706eda5e114a3cd4f901 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 18 Mar 2025 18:07:14 -0500 Subject: [PATCH 73/92] Create notifications when an error happens during the migration --- readthedocs/oauth/notifications.py | 29 +++++++++++++++++ readthedocs/profiles/views.py | 52 +++++++++++++----------------- 2 files changed, 52 insertions(+), 29 deletions(-) diff --git a/readthedocs/oauth/notifications.py b/readthedocs/oauth/notifications.py index 968c1be8ebd..496a7c83c9b 100644 --- a/readthedocs/oauth/notifications.py +++ b/readthedocs/oauth/notifications.py @@ -5,6 +5,7 @@ from django.utils.translation import gettext_lazy as _ from readthedocs.notifications.constants import ERROR +from readthedocs.notifications.constants import WARNING from readthedocs.notifications.messages import Message from readthedocs.notifications.messages import registry @@ -14,6 +15,8 @@ MESSAGE_OAUTH_WEBHOOK_INVALID = "oauth:webhook:invalid" MESSAGE_OAUTH_BUILD_STATUS_FAILURE = "oauth:status:send-failed" MESSAGE_OAUTH_DEPLOY_KEY_ATTACHED_FAILED = "oauth:deploy-key:attached-failed" +MESSAGE_OAUTH_WEBHOOK_NOT_REMOVED = "oauth:migration:webhook-not-removed" +MESSAGE_OAUTH_DEPLOY_KEY_NOT_REMOVED = "oauth:migration:ssh-key-not-removed" messages = [ Message( @@ -83,5 +86,31 @@ ), type=ERROR, ), + Message( + id=MESSAGE_OAUTH_WEBHOOK_NOT_REMOVED, + header=_("Failed to remove webhook"), + body=_( + textwrap.dedent( + """ + Failed to remove webhook from the {{ repo_full_name }} repository, please remove it manually + from the repository settings (search for a webhook containing "{{ project_slug }}" in the URL). + """ + ).strip(), + ), + type=WARNING, + ), + Message( + id=MESSAGE_OAUTH_DEPLOY_KEY_NOT_REMOVED, + header=_("Failed to remove deploy key"), + body=_( + textwrap.dedent( + """ + Failed to remove deploy key from the {{ repo_full_name }} repository, please remove it manually + from the repository settings (search for a deploy key containing "{{ project_slug }}" in the title). + """ + ) + ), + type=WARNING, + ), ] registry.add(messages) diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index d5344cc4746..61a6ed62750 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -36,6 +36,7 @@ from readthedocs.core.models import UserProfile from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.extend import SettingsOverrideObject +from readthedocs.notifications.models import Notification from readthedocs.oauth.clients import get_oauth2_client from readthedocs.oauth.constants import GITHUB_APP from readthedocs.oauth.migrate import get_installation_target_groups_for_user @@ -43,6 +44,8 @@ from readthedocs.oauth.migrate import get_old_app_link from readthedocs.oauth.migrate import get_projects_missing_migration from readthedocs.oauth.migrate import migrate_project_to_github_app +from readthedocs.oauth.notifications import MESSAGE_OAUTH_DEPLOY_KEY_NOT_REMOVED +from readthedocs.oauth.notifications import MESSAGE_OAUTH_WEBHOOK_NOT_REMOVED from readthedocs.organizations.models import Organization from readthedocs.projects.models import Project from readthedocs.projects.utils import get_csv_file @@ -374,36 +377,27 @@ def post(self, request, *args, **kwargs): else: projects = get_projects_missing_migration(request.user) - has_errors = False for project in projects: - try: - result = migrate_project_to_github_app(project=project, user=request.user) - if not result.webhook_removed: - messages.warning( - request, - _( - "The webhook from the old GitHub integration " - "was not removed for project {project}. " - "Please remove it manually." - ).format(project=project.slug), - ) - + result = migrate_project_to_github_app(project=project, user=request.user) + if not result.webhook_removed: + Notification.objects.add( + message_id=MESSAGE_OAUTH_WEBHOOK_NOT_REMOVED, + attached_to=request.user, + dismissable=True, + format_values={ + "repo_full_name": project.remote_repository.full_name, + "project_slug": project.slug, + }, + ) if not result.ssh_key_removed: - messages.warning( - request, - _( - "The SSH key from the old GitHub integration " - "was not removed for project {project}. " - "Please remove it manually." - ).format(project=project.slug), + Notification.objects.add( + message_id=MESSAGE_OAUTH_DEPLOY_KEY_NOT_REMOVED, + attached_to=request.user, + dismissable=True, + format_values={ + "repo_full_name": project.remote_repository.full_name, + "project_slug": project.slug, + }, ) - except Exception as e: - has_errors = True - messages.error(request, f"Error migrating project {project.slug}: {e}") - - if not has_errors: - messages.success(request, _("Projects migrated successfully")) - # if has_errors: - # return HttpResponseRedirect(reverse("migrate_to_gh_app")) - return HttpResponseRedirect(reverse("migrate_to_github_app")) + return HttpResponseRedirect(request.get_full_path()) From 9b14e95b8f2a72993cbd43500877520d7dadcbc2 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 18 Mar 2025 18:50:55 -0500 Subject: [PATCH 74/92] Fixes --- readthedocs/oauth/migrate.py | 29 ++++++++++++++++------------- readthedocs/profiles/views.py | 28 ++++++++++++++-------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index 37c683d6b66..aef21d06acb 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -9,7 +9,7 @@ from readthedocs.integrations.models import Integration from readthedocs.oauth.constants import GITHUB from readthedocs.oauth.constants import GITHUB_APP -from readthedocs.oauth.models import RemoteRepository +from readthedocs.oauth.models import GitHubAccountType, RemoteRepository from readthedocs.oauth.services import GitHubAppService from readthedocs.oauth.services import GitHubService from readthedocs.projects.models import Project @@ -55,7 +55,7 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou targets[organization_id] = InstallationTargetGroup( target_id=organization_id, target_name=remote_repository.organization.slug, - target_type="organization", + target_type=GitHubAccountType.ORGANIZATION, repository_ids=set(), ) if not has_intallation: @@ -78,7 +78,7 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou targets[account.uid] = InstallationTargetGroup( target_id=account.uid, target_name=account.extra_data.get("login"), - target_type="user", + target_type=GitHubAccountType.USER, repository_ids={remote_repository.remote_id for remote_repository in user_repositories}, ) @@ -86,7 +86,15 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou def _get_projects_for_user(user): - for project in get_projects_missing_migration(user): + projects = ( + AdminPermission.projects(user, admin=True) + .filter(remote_repository__vcs_provider=GITHUB) + .select_related( + "remote_repository", + "remote_repository__organization", + ) + ) + for project in projects: remote_repository = project.remote_repository has_installation = RemoteRepository.objects.filter( remote_id=remote_repository.remote_id, @@ -105,15 +113,10 @@ def _get_projects_for_user(user): yield project, has_installation, is_admin -def get_projects_missing_migration(user): - return ( - AdminPermission.projects(user, admin=True) - .filter(remote_repository__vcs_provider=GITHUB) - .select_related( - "remote_repository", - "remote_repository__organization", - ) - ) +def get_valid_projects_missing_migration(user): + for project, has_installation, is_admin in _get_projects_for_user(user): + if has_installation and is_admin: + yield project @dataclass diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index 61a6ed62750..0d4e820b897 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -42,7 +42,7 @@ from readthedocs.oauth.migrate import get_installation_target_groups_for_user from readthedocs.oauth.migrate import get_migration_targets from readthedocs.oauth.migrate import get_old_app_link -from readthedocs.oauth.migrate import get_projects_missing_migration +from readthedocs.oauth.migrate import get_valid_projects_missing_migration from readthedocs.oauth.migrate import migrate_project_to_github_app from readthedocs.oauth.notifications import MESSAGE_OAUTH_DEPLOY_KEY_NOT_REMOVED from readthedocs.oauth.notifications import MESSAGE_OAUTH_WEBHOOK_NOT_REMOVED @@ -375,7 +375,7 @@ def post(self, request, *args, **kwargs): if project_slug: projects = AdminPermission.projects(request.user, admin=True).filter(slug=project_slug) else: - projects = get_projects_missing_migration(request.user) + projects = get_valid_projects_missing_migration(request.user) for project in projects: result = migrate_project_to_github_app(project=project, user=request.user) @@ -389,15 +389,15 @@ def post(self, request, *args, **kwargs): "project_slug": project.slug, }, ) - if not result.ssh_key_removed: - Notification.objects.add( - message_id=MESSAGE_OAUTH_DEPLOY_KEY_NOT_REMOVED, - attached_to=request.user, - dismissable=True, - format_values={ - "repo_full_name": project.remote_repository.full_name, - "project_slug": project.slug, - }, - ) - - return HttpResponseRedirect(request.get_full_path()) + if not result.ssh_key_removed: + Notification.objects.add( + message_id=MESSAGE_OAUTH_DEPLOY_KEY_NOT_REMOVED, + attached_to=request.user, + dismissable=True, + format_values={ + "repo_full_name": project.remote_repository.full_name, + "project_slug": project.slug, + }, + ) + + return HttpResponseRedirect(reverse("migrate_to_github_app") + "?step=migrate") From 704ecb4b8c5eecfbaf5b6a3e3a93ef82d27888cf Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 19 Mar 2025 12:54:50 -0500 Subject: [PATCH 75/92] Check for users with multiple GH accounts --- readthedocs/oauth/migrate.py | 2 +- readthedocs/profiles/views.py | 1 + .../profiles/private/migrate-to-gh-app.html | 132 ------------------ 3 files changed, 2 insertions(+), 133 deletions(-) delete mode 100644 readthedocs/templates/profiles/private/migrate-to-gh-app.html diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index aef21d06acb..a646b762486 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -137,7 +137,7 @@ def installation_link(self): return f"{base_url}?suggested_target_id={self.target_id}&repository_ids[]={self.project.remote_repository.remote_id}" @property - def can_migrate(self): + def can_be_migrated(self): return self.is_admin and self.has_installation diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index 0d4e820b897..ad9c4bbdf04 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -329,6 +329,7 @@ def get_context_data(self, **kwargs): user = self.request.user + context["has_multiple_github_accounts"] = user.socialaccount_set.filter(provider=GitHubProvider.id).count() > 1 context["step_connect_completed"] = self._has_new_account_for_old_account() context["installation_target_groups"] = get_installation_target_groups_for_user(user) context["gh_app_name"] = settings.GITHUB_APP_NAME diff --git a/readthedocs/templates/profiles/private/migrate-to-gh-app.html b/readthedocs/templates/profiles/private/migrate-to-gh-app.html deleted file mode 100644 index 6ad33ec5196..00000000000 --- a/readthedocs/templates/profiles/private/migrate-to-gh-app.html +++ /dev/null @@ -1,132 +0,0 @@ -{% extends "profiles/base_profile_edit.html" %} - -{% load provider_login_url from socialaccount %} -{% load i18n %} - -{% block title %}{% trans "Migrate account to GitHub App" %}{% endblock %} - -{% block profile-admin-tokens %}active{% endblock %} - -{% block edit_content_header %} {% trans "Migrate account to GitHub App" %} {% endblock %} - -{% block edit_content %} -
      -
    1. - Connect your account to the new GitHub app: - - {% url "migrate_to_github_app" as migrate_to_github_app_url %} -
      - {% csrf_token %} - -
      -
      2. -
    2. - Install app in all your repositories connected to a project: - - {% if installation_target_groups %} -
        - {% for installation_target_group in installation_target_groups %} -
      • - {{ installation_target_group.target_name }} ({{ installation_target_group.target_type }}): - {% if installation_target_group.installed %} - installed - {% else %} - - install - - {% endif %} -
        • - {% endfor %} -
      - - Or install one by one in the next step - {% else %} -

      - You have already granted access to all your repositories. -

      - {% endif %} -
      3. - -
    3. - Migrate your projects to the GH app: - -
        -
      • -
        - {% csrf_token %} - -
        -
        • -
      - - Or migrate one by one: - - -
      4. - -
    4. - - Revoke access to the old GH OAuth app from your account - -
      5. - -
    5. - Disconnect the old GH app from your RTD account: - -
      - {% csrf_token %} - -
      -
      6. -
    -{% endblock %} From 19dc240995a73360cbd6474d3114c561bb7c0660 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 19 Mar 2025 13:20:29 -0500 Subject: [PATCH 76/92] Better default for accounts linked to multiple GH accounts --- readthedocs/oauth/migrate.py | 49 ++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index a646b762486..cd584826d37 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -46,41 +46,34 @@ def installed(self): def get_installation_target_groups_for_user(user) -> list[InstallationTargetGroup]: targets = {} - user_repositories = set() + account = user.socialaccount_set.filter(provider=GitHubProvider.id).first() + + # Since we don't save the ID of the owner of each repository, we group all repositories + # that don't have an organization under the user account, + # GitHub will ignore the repositories that the user doesn't own. + targets[account.uid] = InstallationTargetGroup( + target_id=account.uid, + target_name=account.extra_data.get("login", "unknown"), + target_type=GitHubAccountType.USER, + repository_ids=set(), + ) + for project, has_intallation, _ in _get_projects_for_user(user): remote_repository = project.remote_repository if remote_repository.organization: - organization_id = remote_repository.organization.remote_id - if organization_id not in targets: - targets[organization_id] = InstallationTargetGroup( - target_id=organization_id, + target_id = remote_repository.organization.remote_id + if target_id not in targets: + targets[target_id] = InstallationTargetGroup( + target_id=target_id, target_name=remote_repository.organization.slug, target_type=GitHubAccountType.ORGANIZATION, repository_ids=set(), ) - if not has_intallation: - targets[organization_id].repository_ids.add(remote_repository.remote_id) - elif not has_intallation: - user_repositories.add(remote_repository) - - # TODO: check how many users have more than one GH account connected. - # from allauth.socialaccount.providers.github.provider import GitHubProvider - # from django.contrib.auth.models import User - # from django.db.models import Count, Q - # # Find all users that have more than one GitHub account connected. - # users = User.objects.annotate( - # num_github_accounts=Count("socialaccount", filter=Q(socialaccount__provider=GitHubProvider.id)) - # ).filter(num_github_accounts__gt=1) - # 166 on .org, 17 on .com - # Since we don't know the ID of the owner, we create a link for each connected account. - # GH will select only the corresponding repositories for each account. - for account in user.socialaccount_set.filter(provider=GitHubProvider.id): - targets[account.uid] = InstallationTargetGroup( - target_id=account.uid, - target_name=account.extra_data.get("login"), - target_type=GitHubAccountType.USER, - repository_ids={remote_repository.remote_id for remote_repository in user_repositories}, - ) + else: + target_id = account.uid + + if not has_intallation: + targets[target_id].repository_ids.add(remote_repository.remote_id) return list(targets.values()) From 5a5d532f8d15b0502722c48052ca70d61228a1d5 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 20 Mar 2025 14:38:22 -0500 Subject: [PATCH 77/92] Rollback changes to avoid merge conflicts --- readthedocs/oauth/signals.py | 4 ++ requirements/deploy.txt | 82 ++++++++++++++------------------- requirements/docker.txt | 89 +++++++++++++++++------------------- requirements/docs.txt | 22 +++++---- requirements/pip.txt | 71 ++++++++++++---------------- requirements/testing.txt | 82 +++++++++++++++------------------ 6 files changed, 160 insertions(+), 190 deletions(-) diff --git a/readthedocs/oauth/signals.py b/readthedocs/oauth/signals.py index 1fcbb1eaad9..3735ca549c6 100644 --- a/readthedocs/oauth/signals.py +++ b/readthedocs/oauth/signals.py @@ -33,6 +33,10 @@ def sync_remote_repositories_on_login(sender, request, user, *args, **kwargs): @receiver(social_account_added, sender=SocialLogin) def sync_remote_repositories_on_social_account_added(sender, request, sociallogin, *args, **kwargs): """Sync remote repositories when a new social account is added.""" + log.info( + "Triggering remote repositories sync in background on social account added.", + user_username=sociallogin.user.username, + ) sync_remote_repositories.delay(sociallogin.user.pk) diff --git a/requirements/deploy.txt b/requirements/deploy.txt index 4ddbcdeef94..d5108fddeaf 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=requirements/deploy.txt requirements/deploy.in @@ -20,15 +20,19 @@ asgiref==3.8.1 # django-cors-headers asttokens==3.0.0 # via stack-data +async-timeout==5.0.1 + # via + # -r requirements/pip.txt + # redis billiard==4.2.1 # via # -r requirements/pip.txt # celery -boto3==1.36.26 +boto3==1.37.13 # via # -r requirements/pip.txt # django-storages -botocore==1.36.26 +botocore==1.37.13 # via # -r requirements/pip.txt # boto3 @@ -49,7 +53,6 @@ cffi==1.17.1 # via # -r requirements/pip.txt # cryptography - # pynacl charset-normalizer==3.4.1 # via # -r requirements/pip.txt @@ -82,21 +85,17 @@ cron-descriptor==1.4.5 # via # -r requirements/pip.txt # django-celery-beat -cryptography==44.0.0 +cryptography==44.0.2 # via # -r requirements/pip.txt # fido2 # pyjwt -cssselect==1.2.0 +cssselect==1.3.0 # via # -r requirements/pip.txt # pyquery -decorator==5.2.0 +decorator==5.2.1 # via ipython -deprecated==1.2.18 - # via - # -r requirements/pip.txt - # pygithub distlib==0.3.9 # via # -r requirements/pip.txt @@ -105,7 +104,7 @@ dj-pagination==2.5.0 # via -r requirements/pip.txt dj-stripe==2.6.3 # via -r requirements/pip.txt -django==4.2.18 +django==4.2.20 # via # -r requirements/pip.txt # dj-stripe @@ -128,7 +127,7 @@ django==4.2.18 # django-timezone-field # djangorestframework # jsonfield -django-allauth[mfa,saml,socialaccount]==65.3.1 +django-allauth[mfa,saml,socialaccount]==65.5.0 # via -r requirements/pip.txt django-annoying==0.10.7 # via -r requirements/pip.txt @@ -138,7 +137,7 @@ django-cacheops==7.1 # via -r requirements/pip.txt django-celery-beat==2.7.0 # via -r requirements/pip.txt -django-cors-headers==4.6.0 +django-cors-headers==4.7.0 # via -r requirements/pip.txt django-crispy-forms==1.14.0 # via -r requirements/pip.txt @@ -191,7 +190,7 @@ djangorestframework-jsonp==1.0.2 # via -r requirements/pip.txt dnspython==2.7.0 # via -r requirements/pip.txt -docker==6.1.2 +docker==7.1.0 # via -r requirements/pip.txt docutils==0.21.2 # via -r requirements/pip.txt @@ -201,12 +200,12 @@ drf-extensions==0.7.1 # via -r requirements/pip.txt drf-flex-fields==1.0.2 # via -r requirements/pip.txt -elastic-transport==8.17.0 +elastic-transport==8.17.1 # via # -r requirements/pip.txt # elasticsearch # elasticsearch-dsl -elasticsearch==8.17.1 +elasticsearch==8.17.2 # via # -r requirements/pip.txt # elasticsearch-dsl @@ -214,13 +213,15 @@ elasticsearch-dsl==8.17.1 # via # -r requirements/pip.txt # django-elasticsearch-dsl +exceptiongroup==1.2.2 + # via ipython executing==2.2.0 # via stack-data fido2==1.2.0 # via # -r requirements/pip.txt # django-allauth -filelock==3.17.0 +filelock==3.18.0 # via # -r requirements/pip.txt # virtualenv @@ -234,7 +235,7 @@ idna==3.10 # via # -r requirements/pip.txt # requests -ipython==8.32.0 +ipython==8.34.0 # via -r requirements/deploy.in isodate==0.7.2 # via @@ -251,7 +252,7 @@ jsonfield==3.1.0 # via # -r requirements/pip.txt # dj-stripe -kombu==5.4.2 +kombu==5.5.0 # via # -r requirements/pip.txt # celery @@ -259,7 +260,7 @@ lexid==2021.1006 # via # -r requirements/pip.txt # bumpver -lxml==5.3.0 +lxml==5.3.1 # via # -r requirements/pip.txt # pyquery @@ -269,7 +270,7 @@ markdown==3.7 # via -r requirements/pip.txt matplotlib-inline==0.1.7 # via ipython -newrelic==10.5.0 +newrelic==10.7.0 # via -r requirements/deploy.in oauthlib==3.2.2 # via @@ -281,7 +282,6 @@ packaging==24.2 # via # -r requirements/pip.txt # djangorestframework-api-key - # docker # dparse # gunicorn parso==0.8.4 @@ -297,13 +297,13 @@ prompt-toolkit==3.0.50 # -r requirements/pip.txt # click-repl # ipython -psycopg[binary,pool]==3.2.5 +psycopg[binary,pool]==3.2.6 # via -r requirements/pip.txt -psycopg-binary==3.2.5 +psycopg-binary==3.2.6 # via # -r requirements/pip.txt # psycopg -psycopg-pool==3.2.5 +psycopg-pool==3.2.6 # via # -r requirements/pip.txt # psycopg @@ -321,8 +321,6 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic -pygithub==2.6.1 - # via -r requirements/pip.txt pygments==2.19.1 # via # -r requirements/pip.txt @@ -331,11 +329,6 @@ pyjwt[crypto]==2.10.1 # via # -r requirements/pip.txt # django-allauth - # pygithub -pynacl==1.5.0 - # via - # -r requirements/pip.txt - # pygithub pyquery==2.0.1 # via -r requirements/pip.txt python-crontab==3.2.0 @@ -367,12 +360,11 @@ redis==5.2.1 # django-cacheops regex==2024.11.6 # via -r requirements/pip.txt -requests==2.30.0 +requests==2.32.3 # via # -r requirements/pip.txt # django-allauth # docker - # pygithub # requests-oauthlib # requests-toolbelt # slumber @@ -385,7 +377,7 @@ requests-toolbelt==1.0.0 # via -r requirements/pip.txt rest-framework-generic-relations==2.2.0 # via -r requirements/pip.txt -s3transfer==0.11.2 +s3transfer==0.11.4 # via # -r requirements/pip.txt # boto3 @@ -424,6 +416,10 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver +tomli==2.2.1 + # via + # -r requirements/pip.txt + # dparse traitlets==5.14.3 # via # ipython @@ -431,12 +427,13 @@ traitlets==5.14.3 typing-extensions==4.12.2 # via # -r requirements/pip.txt + # asgiref # elasticsearch-dsl + # ipython # psycopg # psycopg-pool # pydantic # pydantic-core - # pygithub tzdata==2025.1 # via # -r requirements/pip.txt @@ -463,7 +460,6 @@ urllib3==2.3.0 # botocore # docker # elastic-transport - # pygithub # requests # sentry-sdk user-agents==2.2.0 @@ -474,20 +470,12 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.29.1 +virtualenv==20.29.3 # via -r requirements/pip.txt wcwidth==0.2.13 # via # -r requirements/pip.txt # prompt-toolkit -websocket-client==1.8.0 - # via - # -r requirements/pip.txt - # docker -wrapt==1.17.2 - # via - # -r requirements/pip.txt - # deprecated xmlsec==1.3.14 # via # -r requirements/pip.txt diff --git a/requirements/docker.txt b/requirements/docker.txt index 239961421e7..c8567e48c3b 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=requirements/docker.txt requirements/docker.in @@ -20,17 +20,21 @@ asgiref==3.8.1 # django-cors-headers asttokens==3.0.0 # via stack-data -attrs==25.1.0 +async-timeout==5.0.1 + # via + # -r requirements/pip.txt + # redis +attrs==25.3.0 # via wmctrl billiard==4.2.1 # via # -r requirements/pip.txt # celery -boto3==1.36.26 +boto3==1.37.13 # via # -r requirements/pip.txt # django-storages -botocore==1.36.26 +botocore==1.37.13 # via # -r requirements/pip.txt # boto3 @@ -52,7 +56,6 @@ cffi==1.17.1 # via # -r requirements/pip.txt # cryptography - # pynacl chardet==5.2.0 # via tox charset-normalizer==3.4.1 @@ -88,25 +91,21 @@ cron-descriptor==1.4.5 # via # -r requirements/pip.txt # django-celery-beat -cryptography==44.0.0 +cryptography==44.0.2 # via # -r requirements/pip.txt # fido2 # pyjwt -cssselect==1.2.0 +cssselect==1.3.0 # via # -r requirements/pip.txt # pyquery datadiff==2.2.0 # via -r requirements/docker.in -decorator==5.2.0 +decorator==5.2.1 # via # ipdb # ipython -deprecated==1.2.18 - # via - # -r requirements/pip.txt - # pygithub distlib==0.3.9 # via # -r requirements/pip.txt @@ -115,7 +114,7 @@ dj-pagination==2.5.0 # via -r requirements/pip.txt dj-stripe==2.6.3 # via -r requirements/pip.txt -django==4.2.18 +django==4.2.20 # via # -r requirements/pip.txt # dj-stripe @@ -138,7 +137,7 @@ django==4.2.18 # django-timezone-field # djangorestframework # jsonfield -django-allauth[mfa,saml,socialaccount]==65.3.1 +django-allauth[mfa,saml,socialaccount]==65.5.0 # via -r requirements/pip.txt django-annoying==0.10.7 # via -r requirements/pip.txt @@ -148,7 +147,7 @@ django-cacheops==7.1 # via -r requirements/pip.txt django-celery-beat==2.7.0 # via -r requirements/pip.txt -django-cors-headers==4.6.0 +django-cors-headers==4.7.0 # via -r requirements/pip.txt django-crispy-forms==1.14.0 # via -r requirements/pip.txt @@ -201,7 +200,7 @@ djangorestframework-jsonp==1.0.2 # via -r requirements/pip.txt dnspython==2.7.0 # via -r requirements/pip.txt -docker==6.1.2 +docker==7.1.0 # via -r requirements/pip.txt docutils==0.21.2 # via -r requirements/pip.txt @@ -211,12 +210,12 @@ drf-extensions==0.7.1 # via -r requirements/pip.txt drf-flex-fields==1.0.2 # via -r requirements/pip.txt -elastic-transport==8.17.0 +elastic-transport==8.17.1 # via # -r requirements/pip.txt # elasticsearch # elasticsearch-dsl -elasticsearch==8.17.1 +elasticsearch==8.17.2 # via # -r requirements/pip.txt # elasticsearch-dsl @@ -224,6 +223,8 @@ elasticsearch-dsl==8.17.1 # via # -r requirements/pip.txt # django-elasticsearch-dsl +exceptiongroup==1.2.2 + # via ipython executing==2.2.0 # via stack-data fancycompleter==0.9.1 @@ -232,7 +233,7 @@ fido2==1.2.0 # via # -r requirements/pip.txt # django-allauth -filelock==3.17.0 +filelock==3.18.0 # via # -r requirements/pip.txt # tox @@ -249,7 +250,7 @@ idna==3.10 # requests ipdb==0.13.13 # via -r requirements/docker.in -ipython==8.32.0 +ipython==8.34.0 # via ipdb isodate==0.7.2 # via @@ -266,7 +267,7 @@ jsonfield==3.1.0 # via # -r requirements/pip.txt # dj-stripe -kombu==5.4.2 +kombu==5.5.0 # via # -r requirements/pip.txt # celery @@ -274,7 +275,7 @@ lexid==2021.1006 # via # -r requirements/pip.txt # bumpver -lxml==5.3.0 +lxml==5.3.1 # via # -r requirements/pip.txt # pyquery @@ -298,7 +299,6 @@ packaging==24.2 # via # -r requirements/pip.txt # djangorestframework-api-key - # docker # dparse # gunicorn # pyproject-api @@ -321,13 +321,13 @@ prompt-toolkit==3.0.50 # -r requirements/pip.txt # click-repl # ipython -psycopg[binary,pool]==3.2.5 +psycopg[binary,pool]==3.2.6 # via -r requirements/pip.txt -psycopg-binary==3.2.5 +psycopg-binary==3.2.6 # via # -r requirements/pip.txt # psycopg -psycopg-pool==3.2.5 +psycopg-pool==3.2.6 # via # -r requirements/pip.txt # psycopg @@ -345,8 +345,6 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic -pygithub==2.6.1 - # via -r requirements/pip.txt pygments==2.19.1 # via # -r requirements/pip.txt @@ -357,11 +355,6 @@ pyjwt[crypto]==2.10.1 # via # -r requirements/pip.txt # django-allauth - # pygithub -pynacl==1.5.0 - # via - # -r requirements/pip.txt - # pygithub pyproject-api==1.9.0 # via tox pyquery==2.0.1 @@ -397,12 +390,11 @@ redis==5.2.1 # django-cacheops regex==2024.11.6 # via -r requirements/pip.txt -requests==2.30.0 +requests==2.32.3 # via # -r requirements/pip.txt # django-allauth # docker - # pygithub # requests-oauthlib # requests-toolbelt # slumber @@ -417,7 +409,7 @@ rest-framework-generic-relations==2.2.0 # via -r requirements/pip.txt rich==13.9.4 # via -r requirements/docker.in -s3transfer==0.11.2 +s3transfer==0.11.4 # via # -r requirements/pip.txt # boto3 @@ -451,7 +443,14 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver -tox==4.24.1 +tomli==2.2.1 + # via + # -r requirements/pip.txt + # dparse + # ipdb + # pyproject-api + # tox +tox==4.24.2 # via -r requirements/docker.in traitlets==5.14.3 # via @@ -460,12 +459,15 @@ traitlets==5.14.3 typing-extensions==4.12.2 # via # -r requirements/pip.txt + # asgiref # elasticsearch-dsl + # ipython # psycopg # psycopg-pool # pydantic # pydantic-core - # pygithub + # rich + # tox tzdata==2025.1 # via # -r requirements/pip.txt @@ -492,7 +494,6 @@ urllib3==2.3.0 # botocore # docker # elastic-transport - # pygithub # requests user-agents==2.2.0 # via -r requirements/pip.txt @@ -502,7 +503,7 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.29.1 +virtualenv==20.29.3 # via # -r requirements/pip.txt # tox @@ -510,16 +511,8 @@ wcwidth==0.2.13 # via # -r requirements/pip.txt # prompt-toolkit -websocket-client==1.8.0 - # via - # -r requirements/pip.txt - # docker wmctrl==0.5 # via pdbpp -wrapt==1.17.2 - # via - # -r requirements/pip.txt - # deprecated xmlsec==1.3.14 # via # -r requirements/pip.txt diff --git a/requirements/docs.txt b/requirements/docs.txt index d1f58aa56c2..3e8c9dcbbc9 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=requirements/docs.txt requirements/docs.in @@ -37,7 +37,9 @@ docutils==0.21.2 # sphinx-prompt # sphinx-rtd-theme # sphinx-tabs -fonttools==4.55.8 +exceptiongroup==1.2.2 + # via anyio +fonttools==4.56.0 # via matplotlib h11==0.14.0 # via uvicorn @@ -48,7 +50,7 @@ idna==3.10 # sphinx-prompt imagesize==1.4.1 # via sphinx -jinja2==3.1.5 +jinja2==3.1.6 # via # myst-parser # sphinx @@ -60,7 +62,7 @@ markdown-it-py==3.0.0 # myst-parser markupsafe==3.0.2 # via jinja2 -matplotlib==3.10.0 +matplotlib==3.10.1 # via -r requirements/docs.in mdit-py-plugins==0.4.2 # via myst-parser @@ -76,7 +78,7 @@ packaging==24.2 # via # matplotlib # sphinx -pbr==6.1.0 +pbr==6.1.1 # via sphinxcontrib-video pillow==11.1.0 # via matplotlib @@ -157,10 +159,14 @@ sphinxemoji==0.3.1 # via -r requirements/docs.in sphinxext-opengraph==0.9.1 # via -r requirements/docs.in -starlette==0.46.0 +starlette==0.46.1 # via sphinx-autobuild +tomli==2.2.1 + # via sphinx typing-extensions==4.12.2 - # via anyio + # via + # anyio + # uvicorn urllib3==2.3.0 # via # requests @@ -169,7 +175,7 @@ uvicorn==0.34.0 # via sphinx-autobuild watchfiles==1.0.4 # via sphinx-autobuild -websockets==15.0 +websockets==15.0.1 # via sphinx-autobuild # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/pip.txt b/requirements/pip.txt index e931a4d7ad0..020491bf0d5 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=requirements/pip.txt requirements/pip.in @@ -13,11 +13,13 @@ asgiref==3.8.1 # django # django-allauth # django-cors-headers +async-timeout==5.0.1 + # via redis billiard==4.2.1 # via celery -boto3==1.36.26 +boto3==1.37.13 # via django-storages -botocore==1.36.26 +botocore==1.37.13 # via # boto3 # s3transfer @@ -32,9 +34,7 @@ certifi==2025.1.31 # elastic-transport # requests cffi==1.17.1 - # via - # cryptography - # pynacl + # via cryptography charset-normalizer==3.4.1 # via requests click==8.1.8 @@ -54,21 +54,19 @@ colorama==0.4.6 # via bumpver cron-descriptor==1.4.5 # via django-celery-beat -cryptography==44.0.0 +cryptography==44.0.2 # via # fido2 # pyjwt -cssselect==1.2.0 +cssselect==1.3.0 # via pyquery -deprecated==1.2.18 - # via pygithub distlib==0.3.9 # via virtualenv dj-pagination==2.5.0 # via -r requirements/pip.in dj-stripe==2.6.3 # via -r requirements/pip.in -django==4.2.18 +django==4.2.20 # via # -r requirements/pip.in # dj-stripe @@ -91,7 +89,7 @@ django==4.2.18 # django-timezone-field # djangorestframework # jsonfield -django-allauth[mfa,saml,socialaccount]==65.3.1 +django-allauth[mfa,saml,socialaccount]==65.5.0 # via -r requirements/pip.in django-annoying==0.10.7 # via -r requirements/pip.in @@ -101,7 +99,7 @@ django-cacheops==7.1 # via -r requirements/pip.in django-celery-beat==2.7.0 # via -r requirements/pip.in -django-cors-headers==4.6.0 +django-cors-headers==4.7.0 # via -r requirements/pip.in django-crispy-forms==1.14.0 # via -r requirements/pip.in @@ -152,7 +150,7 @@ djangorestframework-jsonp==1.0.2 # via -r requirements/pip.in dnspython==2.7.0 # via -r requirements/pip.in -docker==6.1.2 +docker==7.1.0 # via -r requirements/pip.in docutils==0.21.2 # via -r requirements/pip.in @@ -162,11 +160,11 @@ drf-extensions==0.7.1 # via -r requirements/pip.in drf-flex-fields==1.0.2 # via -r requirements/pip.in -elastic-transport==8.17.0 +elastic-transport==8.17.1 # via # elasticsearch # elasticsearch-dsl -elasticsearch==8.17.1 +elasticsearch==8.17.2 # via # -r requirements/pip.in # elasticsearch-dsl @@ -176,7 +174,7 @@ elasticsearch-dsl==8.17.1 # django-elasticsearch-dsl fido2==1.2.0 # via django-allauth -filelock==3.17.0 +filelock==3.18.0 # via virtualenv funcy==2.0 # via django-cacheops @@ -194,11 +192,11 @@ jsonfield==3.1.0 # via # -r requirements/pip.in # dj-stripe -kombu==5.4.2 +kombu==5.5.0 # via celery lexid==2021.1006 # via bumpver -lxml==5.3.0 +lxml==5.3.1 # via # pyquery # python3-saml @@ -213,18 +211,17 @@ packaging==24.2 # via # -r requirements/pip.in # djangorestframework-api-key - # docker # dparse # gunicorn platformdirs==4.3.6 # via virtualenv prompt-toolkit==3.0.50 # via click-repl -psycopg[binary,pool]==3.2.5 +psycopg[binary,pool]==3.2.6 # via -r requirements/pip.in -psycopg-binary==3.2.5 +psycopg-binary==3.2.6 # via psycopg -psycopg-pool==3.2.5 +psycopg-pool==3.2.6 # via psycopg pycparser==2.22 # via cffi @@ -232,16 +229,10 @@ pydantic==2.10.6 # via -r requirements/pip.in pydantic-core==2.27.2 # via pydantic -pygithub==2.6.1 - # via -r requirements/pip.in pygments==2.19.1 # via -r requirements/pip.in pyjwt[crypto]==2.10.1 - # via - # django-allauth - # pygithub -pynacl==1.5.0 - # via pygithub + # via django-allauth pyquery==2.0.1 # via -r requirements/pip.in python-crontab==3.2.0 @@ -267,12 +258,11 @@ redis==5.2.1 # django-cacheops regex==2024.11.6 # via -r requirements/pip.in -requests==2.30.0 +requests==2.32.3 # via # -r requirements/pip.in # django-allauth # docker - # pygithub # requests-oauthlib # requests-toolbelt # slumber @@ -285,7 +275,7 @@ requests-toolbelt==1.0.0 # via -r requirements/pip.in rest-framework-generic-relations==2.2.0 # via -r requirements/pip.in -s3transfer==0.11.2 +s3transfer==0.11.4 # via boto3 selectolax==0.3.28 # via -r requirements/pip.in @@ -311,14 +301,16 @@ structlog==23.2.0 # django-structlog toml==0.10.2 # via bumpver +tomli==2.2.1 + # via dparse typing-extensions==4.12.2 # via + # asgiref # elasticsearch-dsl # psycopg # psycopg-pool # pydantic # pydantic-core - # pygithub tzdata==2025.1 # via # celery @@ -337,7 +329,6 @@ urllib3==2.3.0 # botocore # docker # elastic-transport - # pygithub # requests user-agents==2.2.0 # via -r requirements/pip.in @@ -346,16 +337,14 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.29.1 +virtualenv==20.29.3 # via -r requirements/pip.in wcwidth==0.2.13 # via prompt-toolkit -websocket-client==1.8.0 - # via docker -wrapt==1.17.2 - # via deprecated xmlsec==1.3.14 - # via python3-saml + # via + # -r requirements/pip.in + # python3-saml # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/testing.txt b/requirements/testing.txt index ae05239ca9b..1a4a65f6ad7 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=requirements/testing.txt requirements/testing.in @@ -20,17 +20,21 @@ asgiref==3.8.1 # django # django-allauth # django-cors-headers +async-timeout==5.0.1 + # via + # -r requirements/pip.txt + # redis babel==2.17.0 # via sphinx billiard==4.2.1 # via # -r requirements/pip.txt # celery -boto3==1.36.26 +boto3==1.37.13 # via # -r requirements/pip.txt # django-storages -botocore==1.36.26 +botocore==1.37.13 # via # -r requirements/pip.txt # boto3 @@ -50,7 +54,6 @@ cffi==1.17.1 # via # -r requirements/pip.txt # cryptography - # pynacl charset-normalizer==3.4.1 # via # -r requirements/pip.txt @@ -85,12 +88,12 @@ cron-descriptor==1.4.5 # via # -r requirements/pip.txt # django-celery-beat -cryptography==44.0.0 +cryptography==44.0.2 # via # -r requirements/pip.txt # fido2 # pyjwt -cssselect==1.2.0 +cssselect==1.3.0 # via # -r requirements/pip.txt # pyquery @@ -98,10 +101,6 @@ cython==3.0.12 # via sphinx defusedxml==0.7.1 # via sphinx -deprecated==1.2.18 - # via - # -r requirements/pip.txt - # pygithub distlib==0.3.9 # via # -r requirements/pip.txt @@ -110,7 +109,7 @@ dj-pagination==2.5.0 # via -r requirements/pip.txt dj-stripe==2.6.3 # via -r requirements/pip.txt -django==4.2.18 +django==4.2.20 # via # -r requirements/pip.txt # dj-stripe @@ -133,7 +132,7 @@ django==4.2.18 # django-timezone-field # djangorestframework # jsonfield -django-allauth[mfa,saml,socialaccount]==65.3.1 +django-allauth[mfa,saml,socialaccount]==65.5.0 # via -r requirements/pip.txt django-annoying==0.10.7 # via -r requirements/pip.txt @@ -143,7 +142,7 @@ django-cacheops==7.1 # via -r requirements/pip.txt django-celery-beat==2.7.0 # via -r requirements/pip.txt -django-cors-headers==4.6.0 +django-cors-headers==4.7.0 # via -r requirements/pip.txt django-crispy-forms==1.14.0 # via -r requirements/pip.txt @@ -198,7 +197,7 @@ djangorestframework-jsonp==1.0.2 # via -r requirements/pip.txt dnspython==2.7.0 # via -r requirements/pip.txt -docker==6.1.2 +docker==7.1.0 # via -r requirements/pip.txt docutils==0.21.2 # via @@ -210,12 +209,12 @@ drf-extensions==0.7.1 # via -r requirements/pip.txt drf-flex-fields==1.0.2 # via -r requirements/pip.txt -elastic-transport==8.17.0 +elastic-transport==8.17.1 # via # -r requirements/pip.txt # elasticsearch # elasticsearch-dsl -elasticsearch==8.17.1 +elasticsearch==8.17.2 # via # -r requirements/pip.txt # elasticsearch-dsl @@ -223,11 +222,13 @@ elasticsearch-dsl==8.17.1 # via # -r requirements/pip.txt # django-elasticsearch-dsl +exceptiongroup==1.2.2 + # via pytest fido2==1.2.0 # via # -r requirements/pip.txt # django-allauth -filelock==3.17.0 +filelock==3.18.0 # via # -r requirements/pip.txt # virtualenv @@ -249,7 +250,7 @@ isodate==0.7.2 # via # -r requirements/pip.txt # python3-saml -jinja2==3.1.5 +jinja2==3.1.6 # via sphinx jmespath==1.0.1 # via @@ -260,7 +261,7 @@ jsonfield==3.1.0 # via # -r requirements/pip.txt # dj-stripe -kombu==5.4.2 +kombu==5.5.0 # via # -r requirements/pip.txt # celery @@ -268,7 +269,7 @@ lexid==2021.1006 # via # -r requirements/pip.txt # bumpver -lxml==5.3.0 +lxml==5.3.1 # via # -r requirements/pip.txt # pyquery @@ -288,7 +289,6 @@ packaging==24.2 # via # -r requirements/pip.txt # djangorestframework-api-key - # docker # dparse # gunicorn # pytest @@ -303,13 +303,13 @@ prompt-toolkit==3.0.50 # via # -r requirements/pip.txt # click-repl -psycopg[binary,pool]==3.2.5 +psycopg[binary,pool]==3.2.6 # via -r requirements/pip.txt -psycopg-binary==3.2.5 +psycopg-binary==3.2.6 # via # -r requirements/pip.txt # psycopg -psycopg-pool==3.2.5 +psycopg-pool==3.2.6 # via # -r requirements/pip.txt # psycopg @@ -323,8 +323,6 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic -pygithub==2.6.1 - # via -r requirements/pip.txt pygments==2.19.1 # via # -r requirements/pip.txt @@ -333,14 +331,9 @@ pyjwt[crypto]==2.10.1 # via # -r requirements/pip.txt # django-allauth - # pygithub -pynacl==1.5.0 - # via - # -r requirements/pip.txt - # pygithub pyquery==2.0.1 # via -r requirements/pip.txt -pytest==8.3.4 +pytest==8.3.5 # via # -r requirements/testing.in # pytest-cov @@ -385,12 +378,11 @@ redis==5.2.1 # django-cacheops regex==2024.11.6 # via -r requirements/pip.txt -requests==2.30.0 +requests==2.32.3 # via # -r requirements/pip.txt # django-allauth # docker - # pygithub # requests-mock # requests-oauthlib # requests-toolbelt @@ -407,7 +399,7 @@ requests-toolbelt==1.0.0 # via -r requirements/pip.txt rest-framework-generic-relations==2.2.0 # via -r requirements/pip.txt -s3transfer==0.11.2 +s3transfer==0.11.4 # via # -r requirements/pip.txt # boto3 @@ -455,15 +447,22 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver +tomli==2.2.1 + # via + # -r requirements/pip.txt + # coverage + # dparse + # pytest + # sphinx typing-extensions==4.12.2 # via # -r requirements/pip.txt + # asgiref # elasticsearch-dsl # psycopg # psycopg-pool # pydantic # pydantic-core - # pygithub # sphinx tzdata==2025.1 # via @@ -491,7 +490,6 @@ urllib3==2.3.0 # botocore # docker # elastic-transport - # pygithub # requests user-agents==2.2.0 # via -r requirements/pip.txt @@ -501,20 +499,12 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.29.1 +virtualenv==20.29.3 # via -r requirements/pip.txt wcwidth==0.2.13 # via # -r requirements/pip.txt # prompt-toolkit -websocket-client==1.8.0 - # via - # -r requirements/pip.txt - # docker -wrapt==1.17.2 - # via - # -r requirements/pip.txt - # deprecated xmlsec==1.3.14 # via # -r requirements/pip.txt From 26ef7eb03defb90c03c4627cc66622142225c6b3 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 20 Mar 2025 14:41:56 -0500 Subject: [PATCH 78/92] Re-add pygithub --- requirements/deploy.txt | 33 ++++++++++++++++++++------------- requirements/docker.txt | 38 ++++++++++++++++++++------------------ requirements/docs.txt | 10 ++-------- requirements/pip.txt | 30 +++++++++++++++++++----------- requirements/testing.txt | 35 ++++++++++++++++++++--------------- 5 files changed, 81 insertions(+), 65 deletions(-) diff --git a/requirements/deploy.txt b/requirements/deploy.txt index d5108fddeaf..3ce4ab23288 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/deploy.txt requirements/deploy.in @@ -20,10 +20,6 @@ asgiref==3.8.1 # django-cors-headers asttokens==3.0.0 # via stack-data -async-timeout==5.0.1 - # via - # -r requirements/pip.txt - # redis billiard==4.2.1 # via # -r requirements/pip.txt @@ -53,6 +49,7 @@ cffi==1.17.1 # via # -r requirements/pip.txt # cryptography + # pynacl charset-normalizer==3.4.1 # via # -r requirements/pip.txt @@ -96,6 +93,10 @@ cssselect==1.3.0 # pyquery decorator==5.2.1 # via ipython +deprecated==1.2.18 + # via + # -r requirements/pip.txt + # pygithub distlib==0.3.9 # via # -r requirements/pip.txt @@ -213,8 +214,6 @@ elasticsearch-dsl==8.17.1 # via # -r requirements/pip.txt # django-elasticsearch-dsl -exceptiongroup==1.2.2 - # via ipython executing==2.2.0 # via stack-data fido2==1.2.0 @@ -321,6 +320,8 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic +pygithub==2.6.1 + # via -r requirements/pip.txt pygments==2.19.1 # via # -r requirements/pip.txt @@ -329,6 +330,11 @@ pyjwt[crypto]==2.10.1 # via # -r requirements/pip.txt # django-allauth + # pygithub +pynacl==1.5.0 + # via + # -r requirements/pip.txt + # pygithub pyquery==2.0.1 # via -r requirements/pip.txt python-crontab==3.2.0 @@ -365,6 +371,7 @@ requests==2.32.3 # -r requirements/pip.txt # django-allauth # docker + # pygithub # requests-oauthlib # requests-toolbelt # slumber @@ -416,10 +423,6 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver -tomli==2.2.1 - # via - # -r requirements/pip.txt - # dparse traitlets==5.14.3 # via # ipython @@ -427,13 +430,12 @@ traitlets==5.14.3 typing-extensions==4.12.2 # via # -r requirements/pip.txt - # asgiref # elasticsearch-dsl - # ipython # psycopg # psycopg-pool # pydantic # pydantic-core + # pygithub tzdata==2025.1 # via # -r requirements/pip.txt @@ -460,6 +462,7 @@ urllib3==2.3.0 # botocore # docker # elastic-transport + # pygithub # requests # sentry-sdk user-agents==2.2.0 @@ -476,6 +479,10 @@ wcwidth==0.2.13 # via # -r requirements/pip.txt # prompt-toolkit +wrapt==1.17.2 + # via + # -r requirements/pip.txt + # deprecated xmlsec==1.3.14 # via # -r requirements/pip.txt diff --git a/requirements/docker.txt b/requirements/docker.txt index c8567e48c3b..037dc688ff6 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/docker.txt requirements/docker.in @@ -20,10 +20,6 @@ asgiref==3.8.1 # django-cors-headers asttokens==3.0.0 # via stack-data -async-timeout==5.0.1 - # via - # -r requirements/pip.txt - # redis attrs==25.3.0 # via wmctrl billiard==4.2.1 @@ -56,6 +52,7 @@ cffi==1.17.1 # via # -r requirements/pip.txt # cryptography + # pynacl chardet==5.2.0 # via tox charset-normalizer==3.4.1 @@ -106,6 +103,10 @@ decorator==5.2.1 # via # ipdb # ipython +deprecated==1.2.18 + # via + # -r requirements/pip.txt + # pygithub distlib==0.3.9 # via # -r requirements/pip.txt @@ -223,8 +224,6 @@ elasticsearch-dsl==8.17.1 # via # -r requirements/pip.txt # django-elasticsearch-dsl -exceptiongroup==1.2.2 - # via ipython executing==2.2.0 # via stack-data fancycompleter==0.9.1 @@ -345,6 +344,8 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic +pygithub==2.6.1 + # via -r requirements/pip.txt pygments==2.19.1 # via # -r requirements/pip.txt @@ -355,6 +356,11 @@ pyjwt[crypto]==2.10.1 # via # -r requirements/pip.txt # django-allauth + # pygithub +pynacl==1.5.0 + # via + # -r requirements/pip.txt + # pygithub pyproject-api==1.9.0 # via tox pyquery==2.0.1 @@ -395,6 +401,7 @@ requests==2.32.3 # -r requirements/pip.txt # django-allauth # docker + # pygithub # requests-oauthlib # requests-toolbelt # slumber @@ -443,13 +450,6 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver -tomli==2.2.1 - # via - # -r requirements/pip.txt - # dparse - # ipdb - # pyproject-api - # tox tox==4.24.2 # via -r requirements/docker.in traitlets==5.14.3 @@ -459,15 +459,12 @@ traitlets==5.14.3 typing-extensions==4.12.2 # via # -r requirements/pip.txt - # asgiref # elasticsearch-dsl - # ipython # psycopg # psycopg-pool # pydantic # pydantic-core - # rich - # tox + # pygithub tzdata==2025.1 # via # -r requirements/pip.txt @@ -494,6 +491,7 @@ urllib3==2.3.0 # botocore # docker # elastic-transport + # pygithub # requests user-agents==2.2.0 # via -r requirements/pip.txt @@ -513,6 +511,10 @@ wcwidth==0.2.13 # prompt-toolkit wmctrl==0.5 # via pdbpp +wrapt==1.17.2 + # via + # -r requirements/pip.txt + # deprecated xmlsec==1.3.14 # via # -r requirements/pip.txt diff --git a/requirements/docs.txt b/requirements/docs.txt index 3e8c9dcbbc9..c7dc943fedc 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/docs.txt requirements/docs.in @@ -37,8 +37,6 @@ docutils==0.21.2 # sphinx-prompt # sphinx-rtd-theme # sphinx-tabs -exceptiongroup==1.2.2 - # via anyio fonttools==4.56.0 # via matplotlib h11==0.14.0 @@ -161,12 +159,8 @@ sphinxext-opengraph==0.9.1 # via -r requirements/docs.in starlette==0.46.1 # via sphinx-autobuild -tomli==2.2.1 - # via sphinx typing-extensions==4.12.2 - # via - # anyio - # uvicorn + # via anyio urllib3==2.3.0 # via # requests diff --git a/requirements/pip.txt b/requirements/pip.txt index 020491bf0d5..2cd9df8cded 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/pip.txt requirements/pip.in @@ -13,8 +13,6 @@ asgiref==3.8.1 # django # django-allauth # django-cors-headers -async-timeout==5.0.1 - # via redis billiard==4.2.1 # via celery boto3==1.37.13 @@ -34,7 +32,9 @@ certifi==2025.1.31 # elastic-transport # requests cffi==1.17.1 - # via cryptography + # via + # cryptography + # pynacl charset-normalizer==3.4.1 # via requests click==8.1.8 @@ -60,6 +60,8 @@ cryptography==44.0.2 # pyjwt cssselect==1.3.0 # via pyquery +deprecated==1.2.18 + # via pygithub distlib==0.3.9 # via virtualenv dj-pagination==2.5.0 @@ -229,10 +231,16 @@ pydantic==2.10.6 # via -r requirements/pip.in pydantic-core==2.27.2 # via pydantic +pygithub==2.6.1 + # via -r requirements/pip.in pygments==2.19.1 # via -r requirements/pip.in pyjwt[crypto]==2.10.1 - # via django-allauth + # via + # django-allauth + # pygithub +pynacl==1.5.0 + # via pygithub pyquery==2.0.1 # via -r requirements/pip.in python-crontab==3.2.0 @@ -263,6 +271,7 @@ requests==2.32.3 # -r requirements/pip.in # django-allauth # docker + # pygithub # requests-oauthlib # requests-toolbelt # slumber @@ -301,16 +310,14 @@ structlog==23.2.0 # django-structlog toml==0.10.2 # via bumpver -tomli==2.2.1 - # via dparse typing-extensions==4.12.2 # via - # asgiref # elasticsearch-dsl # psycopg # psycopg-pool # pydantic # pydantic-core + # pygithub tzdata==2025.1 # via # celery @@ -329,6 +336,7 @@ urllib3==2.3.0 # botocore # docker # elastic-transport + # pygithub # requests user-agents==2.2.0 # via -r requirements/pip.in @@ -341,10 +349,10 @@ virtualenv==20.29.3 # via -r requirements/pip.in wcwidth==0.2.13 # via prompt-toolkit +wrapt==1.17.2 + # via deprecated xmlsec==1.3.14 - # via - # -r requirements/pip.in - # python3-saml + # via python3-saml # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/testing.txt b/requirements/testing.txt index 1a4a65f6ad7..529348c0686 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/testing.txt requirements/testing.in @@ -20,10 +20,6 @@ asgiref==3.8.1 # django # django-allauth # django-cors-headers -async-timeout==5.0.1 - # via - # -r requirements/pip.txt - # redis babel==2.17.0 # via sphinx billiard==4.2.1 @@ -54,6 +50,7 @@ cffi==1.17.1 # via # -r requirements/pip.txt # cryptography + # pynacl charset-normalizer==3.4.1 # via # -r requirements/pip.txt @@ -101,6 +98,10 @@ cython==3.0.12 # via sphinx defusedxml==0.7.1 # via sphinx +deprecated==1.2.18 + # via + # -r requirements/pip.txt + # pygithub distlib==0.3.9 # via # -r requirements/pip.txt @@ -222,8 +223,6 @@ elasticsearch-dsl==8.17.1 # via # -r requirements/pip.txt # django-elasticsearch-dsl -exceptiongroup==1.2.2 - # via pytest fido2==1.2.0 # via # -r requirements/pip.txt @@ -323,6 +322,8 @@ pydantic-core==2.27.2 # via # -r requirements/pip.txt # pydantic +pygithub==2.6.1 + # via -r requirements/pip.txt pygments==2.19.1 # via # -r requirements/pip.txt @@ -331,6 +332,11 @@ pyjwt[crypto]==2.10.1 # via # -r requirements/pip.txt # django-allauth + # pygithub +pynacl==1.5.0 + # via + # -r requirements/pip.txt + # pygithub pyquery==2.0.1 # via -r requirements/pip.txt pytest==8.3.5 @@ -383,6 +389,7 @@ requests==2.32.3 # -r requirements/pip.txt # django-allauth # docker + # pygithub # requests-mock # requests-oauthlib # requests-toolbelt @@ -447,22 +454,15 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver -tomli==2.2.1 - # via - # -r requirements/pip.txt - # coverage - # dparse - # pytest - # sphinx typing-extensions==4.12.2 # via # -r requirements/pip.txt - # asgiref # elasticsearch-dsl # psycopg # psycopg-pool # pydantic # pydantic-core + # pygithub # sphinx tzdata==2025.1 # via @@ -490,6 +490,7 @@ urllib3==2.3.0 # botocore # docker # elastic-transport + # pygithub # requests user-agents==2.2.0 # via -r requirements/pip.txt @@ -505,6 +506,10 @@ wcwidth==0.2.13 # via # -r requirements/pip.txt # prompt-toolkit +wrapt==1.17.2 + # via + # -r requirements/pip.txt + # deprecated xmlsec==1.3.14 # via # -r requirements/pip.txt From 33893de725749457811519a42155f2817f910377 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 20 Mar 2025 18:04:40 -0500 Subject: [PATCH 79/92] Clean up and docstrings --- readthedocs/oauth/migrate.py | 73 ++++++++++++++++++++++++++++------- readthedocs/profiles/views.py | 24 +++++++++++- 2 files changed, 81 insertions(+), 16 deletions(-) diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index cd584826d37..fe29cb597c8 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -9,7 +9,8 @@ from readthedocs.integrations.models import Integration from readthedocs.oauth.constants import GITHUB from readthedocs.oauth.constants import GITHUB_APP -from readthedocs.oauth.models import GitHubAccountType, RemoteRepository +from readthedocs.oauth.models import GitHubAccountType +from readthedocs.oauth.models import RemoteRepository from readthedocs.oauth.services import GitHubAppService from readthedocs.oauth.services import GitHubService from readthedocs.projects.models import Project @@ -17,7 +18,9 @@ @dataclass class InstallationTargetGroup: - target_id: str + """Group of repositories that should be installed in the same target (user or organization).""" + + target_id: int target_type: str target_name: str repository_ids: set[str] @@ -25,6 +28,8 @@ class InstallationTargetGroup: @property def link(self): """ + Create a link to install the GitHub App on the target with the required repositories pre-selected. + See https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/migrating-oauth-apps-to-github-apps#prompt-users-to-install-your-github-app. """ repository_ids = [] @@ -39,12 +44,17 @@ def link(self): @property def installed(self): - # If we don't have any repositories, the app was already installed, - # or we don't have any repositories to install the app. + """ + Check if the app was already installed on the target. + + If we don't have any repositories left to install, the app was already installed, + or we don't have any repositories to install the app on. + """ return not bool(self.repository_ids) def get_installation_target_groups_for_user(user) -> list[InstallationTargetGroup]: + """Get all targets (accounts and organizations) that the user needs to install the GitHub App on.""" targets = {} account = user.socialaccount_set.filter(provider=GitHubProvider.id).first() @@ -52,16 +62,16 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou # that don't have an organization under the user account, # GitHub will ignore the repositories that the user doesn't own. targets[account.uid] = InstallationTargetGroup( - target_id=account.uid, + target_id=int(account.uid), target_name=account.extra_data.get("login", "unknown"), target_type=GitHubAccountType.USER, repository_ids=set(), ) - for project, has_intallation, _ in _get_projects_for_user(user): + for project, has_intallation, _ in _get_projects_missing_migration(user): remote_repository = project.remote_repository if remote_repository.organization: - target_id = remote_repository.organization.remote_id + target_id = int(remote_repository.organization.remote_id) if target_id not in targets: targets[target_id] = InstallationTargetGroup( target_id=target_id, @@ -70,7 +80,7 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou repository_ids=set(), ) else: - target_id = account.uid + target_id = int(account.uid) if not has_intallation: targets[target_id].repository_ids.add(remote_repository.remote_id) @@ -78,7 +88,13 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou return list(targets.values()) -def _get_projects_for_user(user): +def _get_projects_missing_migration(user): + """ + Get all projects where the user has admin permissions that are still connected to the old GitHub OAuth App. + + Returns a generator with the project, a boolean indicating if the GitHub App is installed on the repository, + and a boolean indicating if the user has admin permissions on the repository. + """ projects = ( AdminPermission.projects(user, admin=True) .filter(remote_repository__vcs_provider=GITHUB) @@ -107,13 +123,15 @@ def _get_projects_for_user(user): def get_valid_projects_missing_migration(user): - for project, has_installation, is_admin in _get_projects_for_user(user): + for project, has_installation, is_admin in _get_projects_missing_migration(user): if has_installation and is_admin: yield project @dataclass class MigrationTarget: + """Information about an individual project that needs to be migrated.""" + project: Project has_installation: bool is_admin: bool @@ -122,6 +140,8 @@ class MigrationTarget: @property def installation_link(self): """ + Create a link to install the GitHub App on the target repository. + See https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/migrating-oauth-apps-to-github-apps """ base_url = ( @@ -131,24 +151,29 @@ def installation_link(self): @property def can_be_migrated(self): + """ + Check if the project can be migrated. + + The project can be migrated if the user is an admin on the repository and the GitHub App is installed. + """ return self.is_admin and self.has_installation def get_migration_targets(user): + """Get all projects that the user needs to migrate to the GitHub App.""" targets = [] # NOTE: there are some users that have more than one GH account connected. - # so this isn't 100% accurate. + # They will need to migrate each account at a time. gh_account = user.socialaccount_set.filter(provider=GITHUB).first() if not gh_account: return targets - for project, has_installation, is_admin in _get_projects_for_user(user): + for project, has_installation, is_admin in _get_projects_missing_migration(user): remote_repository = project.remote_repository - if remote_repository.organization: - target_id = remote_repository.organization.remote_id + target_id = int(remote_repository.organization.remote_id) else: - target_id = gh_account.uid + target_id = int(gh_account.uid) targets.append( MigrationTarget( project=project, @@ -161,12 +186,19 @@ def get_migration_targets(user): def get_old_app_link(): + """ + Get the link to the old GitHub OAuth App settings page. + + Useful so users can revoke the old app. + """ client_id = settings.SOCIALACCOUNT_PROVIDERS["github"]["APPS"][0]["client_id"] return f"https://github.com/settings/connections/applications/{client_id}" @dataclass class MigrationResult: + """Result of a migration operation.""" + webhook_removed: bool ssh_key_removed: bool @@ -176,6 +208,17 @@ class MigrationError(Exception): def migrate_project_to_github_app(project, user) -> MigrationResult: + """ + Migrate a project to the new GitHub App. + + This will remove the webhook and SSH key from the old GitHub OAuth App and + connect the project to the new GitHub App. + + Returns a MigrationResult with the status of the migration. + Raises a MigrationError if the project can't be migrated, + this should never happen as we don't allow migrating projects + that can't be migrated from the UI. + """ # No remote repository, nothing to migrate. if not project.remote_repository: raise MigrationError("Project isn't connected to a repository") diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index ad9c4bbdf04..72d20a5de96 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -306,6 +306,21 @@ class MigrationSteps(StrEnum): class MigrateToGitHubAppView(PrivateViewMixin, TemplateView): + """ + View to help users migrate their account to the new GitHub App. + + This view will guide the user through the process of migrating their account + and projects to the new GitHub App. + + A get request will show the overview of the migration process, + and each step to follow. A post request will migrate a single project + if the project slug is provided in the request, otherwise, it will migrate + all projects that can be migrated. + + In case we weren't able to remove the webhook or SSH key from the old GitHub App, + we create a notification for the user, so they can manually remove it. + """ + template_name = "profiles/private/migrate_to_gh_app.html" def get(self, request, *args, **kwargs): @@ -329,7 +344,9 @@ def get_context_data(self, **kwargs): user = self.request.user - context["has_multiple_github_accounts"] = user.socialaccount_set.filter(provider=GitHubProvider.id).count() > 1 + context["has_multiple_github_accounts"] = ( + user.socialaccount_set.filter(provider=GitHubProvider.id).count() > 1 + ) context["step_connect_completed"] = self._has_new_account_for_old_account() context["installation_target_groups"] = get_installation_target_groups_for_user(user) context["gh_app_name"] = settings.GITHUB_APP_NAME @@ -359,6 +376,11 @@ def _is_access_to_old_github_account_revoked(self): return False def _has_new_account_for_old_account(self): + """ + Check if the user has connected his account to the new GitHub App. + + The new connected account must the same as the old one. + """ old_account = self._get_old_github_account() return self.request.user.socialaccount_set.filter( provider=GitHubAppProvider.id, From 8f23fa4b2b1cb8b1b201b5faf5bb13c79b171bc7 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 24 Mar 2025 16:39:59 -0500 Subject: [PATCH 80/92] Docs about local dev --- docs/dev/install.rst | 22 +++++++++++++++++++++- docs/dev/settings.rst | 10 ++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/dev/install.rst b/docs/dev/install.rst index 8e5aed50d28..07c84c2d1a8 100644 --- a/docs/dev/install.rst +++ b/docs/dev/install.rst @@ -262,10 +262,30 @@ For others, the webhook will simply fail to connect when there are new commits t * Configure the applications on GitHub, Bitbucket, and GitLab. For each of these, the callback URI is ``http://devthedocs.org/accounts//login/callback/`` - where ```` is one of ``github``, ``gitlab``, or ``bitbucket_oauth2``. + where ```` is one of ``github``, ``githubapp``, ``gitlab``, or ``bitbucket_oauth2``. When setup, you will be given a "Client ID" (also called an "Application ID" or just "Key") and a "Secret". * Take the "Client ID" and "Secret" for each service and set them as :ref:`environment variables `. +Configuring GitHub App +~~~~~~~~~~~~~~~~~~~~~~ + +- Create a new GitHub app from https://github.com/settings/apps/new. +- Callback URL should be http://dev.readthedocs.org/accounts/githubapp/login/callback/. +- Keep marked "Expire user authorization tokens" +- Activate the webhook, and set the URL to one provided by a service like `Webhook.site `__ to forward all incoming webhooks to your local development instance. + You should forward all events to ``http://dev.readthedocs.org/webhook/githubapp/``. +- In permissions, select the following: + - Repository permissions: Commit statuses (read and write, so we can create commit statuses), + Contents (read only, so we can clone repos with a token), + Metadata (read only, so we read the repo collaborators), + Pull requests (read and write, so we can post a comment on PRs in the future). + - Organization permissions: Members (read only so we can read the organization members). + - Account permissions: Email addresses (read only, so allauth can fetch all verified emails). +- Subscribe to the following events: Installation target, Member, Organization, Membership, Pull request, Push, and Repository. +- Copy the "Client ID" and "Client Secret" and set them as :ref:`environment variables `. +- Generate a webhook secret and a private key from the GitHub App settings, + and set them as :ref:`environment variables `. + Troubleshooting --------------- diff --git a/docs/dev/settings.rst b/docs/dev/settings.rst index 2db4844c16c..175b11e0031 100644 --- a/docs/dev/settings.rst +++ b/docs/dev/settings.rst @@ -167,6 +167,16 @@ providers using the following environment variables: .. envvar:: RTD_SOCIALACCOUNT_PROVIDERS_GOOGLE_CLIENT_ID .. envvar:: RTD_SOCIALACCOUNT_PROVIDERS_GOOGLE_SECRET +GitHub App +~~~~~~~~~~ + +You can use the following environment variables to set the settings used by the GitHub App: + +.. envvar:: RTD_GITHUB_APP_ID +.. envvar:: RTD_GITHUB_APP_NAME +.. envvar:: RTD_GITHUB_PRIVATE_KEY +.. envvar:: RTD_GITHUB_APP_WEBHOOK_SECRET + Stripe secrets ~~~~~~~~~~~~~~ From 192f3ac1fae2e02de39d3eddd48fd37f45c4c434 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 25 Mar 2025 14:55:37 -0500 Subject: [PATCH 81/92] Add tests for migration page --- readthedocs/oauth/migrate.py | 8 +- readthedocs/profiles/tests/__init__.py | 0 readthedocs/profiles/tests/test_views.py | 684 +++++++++++++++++++++++ readthedocs/profiles/views.py | 2 +- 4 files changed, 689 insertions(+), 5 deletions(-) create mode 100644 readthedocs/profiles/tests/__init__.py create mode 100644 readthedocs/profiles/tests/test_views.py diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index fe29cb597c8..252eb6c98b1 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -21,9 +21,9 @@ class InstallationTargetGroup: """Group of repositories that should be installed in the same target (user or organization).""" target_id: int - target_type: str + target_type: GitHubAccountType target_name: str - repository_ids: set[str] + repository_ids: set[int] @property def link(self): @@ -61,7 +61,7 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou # Since we don't save the ID of the owner of each repository, we group all repositories # that don't have an organization under the user account, # GitHub will ignore the repositories that the user doesn't own. - targets[account.uid] = InstallationTargetGroup( + targets[int(account.uid)] = InstallationTargetGroup( target_id=int(account.uid), target_name=account.extra_data.get("login", "unknown"), target_type=GitHubAccountType.USER, @@ -83,7 +83,7 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou target_id = int(account.uid) if not has_intallation: - targets[target_id].repository_ids.add(remote_repository.remote_id) + targets[target_id].repository_ids.add(int(remote_repository.remote_id)) return list(targets.values()) diff --git a/readthedocs/profiles/tests/__init__.py b/readthedocs/profiles/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/profiles/tests/test_views.py b/readthedocs/profiles/tests/test_views.py new file mode 100644 index 00000000000..be62de0867a --- /dev/null +++ b/readthedocs/profiles/tests/test_views.py @@ -0,0 +1,684 @@ +from unittest import mock + +import pytest +import requests_mock +from allauth.socialaccount.models import SocialAccount, SocialToken +from allauth.socialaccount.providers.github.provider import GitHubProvider +from django.conf import settings +from django.contrib.auth.models import User +from django.test import TestCase, override_settings +from django.urls import reverse +from django_dynamic_fixture import get + +from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider +from readthedocs.notifications.models import Notification +from readthedocs.oauth.constants import GITHUB, GITHUB_APP +from readthedocs.oauth.migrate import InstallationTargetGroup, MigrationTarget +from readthedocs.oauth.models import ( + GitHubAccountType, + GitHubAppInstallation, + RemoteOrganization, + RemoteOrganizationRelation, + RemoteRepository, + RemoteRepositoryRelation, +) +from readthedocs.oauth.notifications import ( + MESSAGE_OAUTH_DEPLOY_KEY_NOT_REMOVED, + MESSAGE_OAUTH_WEBHOOK_NOT_REMOVED, +) +from readthedocs.oauth.services.github import GitHubService +from readthedocs.projects.models import Project + + +@pytest.mark.skipif( + not settings.RTD_EXT_THEME_ENABLED, reason="Not applicable for the old theme" +) +@override_settings(GITHUB_APP_NAME="readthedocs") +class TestMigrateToGitHubAppView(TestCase): + def setUp(self): + self.user = get(User) + self.social_account_github = get( + SocialAccount, + provider=GitHubProvider.id, + user=self.user, + uid="1234", + extra_data={"login": "user"}, + ) + get( + SocialToken, + account=self.social_account_github, + ) + self.social_account_github_app = get( + SocialAccount, + provider=GitHubAppProvider.id, + user=self.user, + uid="1234", + extra_data={"login": "user"}, + ) + self.github_app_installation = get( + GitHubAppInstallation, + installation_id=1111, + target_id=int(self.social_account_github_app.uid), + target_type=GitHubAccountType.USER, + ) + + # Project with remote repository where the user is admin. + self.remote_repository_a = get( + RemoteRepository, + name="repo-a", + full_name="user/repo-a", + html_url="https://github.com/user/repo-a", + remote_id="1111", + vcs_provider=GITHUB, + ) + get( + RemoteRepositoryRelation, + user=self.user, + account=self.social_account_github, + remote_repository=self.remote_repository_a, + admin=True, + ) + self.project_with_remote_repository = get( + Project, + users=[self.user], + remote_repository=self.remote_repository_a, + ) + + # Project with remote repository where the user is not admin. + self.remote_repository_b = get( + RemoteRepository, + name="repo-b", + full_name="user/repo-b", + html_url="https://github.com/user/repo-b", + remote_id="2222", + vcs_provider=GITHUB, + ) + get( + RemoteRepositoryRelation, + user=self.user, + account=self.social_account_github, + remote_repository=self.remote_repository_b, + admin=False, + ) + self.project_with_remote_repository_no_admin = get( + Project, + users=[self.user], + remote_repository=self.remote_repository_b, + ) + + # Project with remote repository where the user doesn't have permissions at all. + self.remote_repository_c = get( + RemoteRepository, + name="repo-c", + full_name="user2/repo-c", + html_url="https://github.com/user2/repo-c", + remote_id="3333", + vcs_provider=GITHUB, + ) + get( + RemoteRepositoryRelation, + user=self.user, + account=self.social_account_github, + remote_repository=self.remote_repository_c, + admin=False, + ) + self.project_with_remote_repository_no_member = get( + Project, + users=[self.user], + remote_repository=self.remote_repository_c, + ) + + # Project connected to a remote repository that belongs to an organization. + self.remote_organization = get( + RemoteOrganization, + slug="org", + name="Organization", + remote_id="9999", + vcs_provider=GITHUB, + ) + get( + RemoteOrganizationRelation, + user=self.user, + account=self.social_account_github, + ) + self.remote_repository_d = get( + RemoteRepository, + name="repo-d", + full_name="org/repo-d", + html_url="https://github.com/org/repo-d", + remote_id="4444", + organization=self.remote_organization, + ) + get( + RemoteRepositoryRelation, + user=self.user, + account=self.social_account_github, + remote_repository=self.remote_repository_d, + admin=True, + ) + self.project_with_remote_organization = get( + Project, + users=[self.user], + remote_repository=self.remote_repository_d, + ) + self.github_app_organization_installation = get( + GitHubAppInstallation, + installation_id=2222, + target_id=int(self.remote_organization.remote_id), + target_type=GitHubAccountType.ORGANIZATION, + ) + + # Project without a remote repository. + self.project_without_remote_repository = get( + Project, + users=[self.user], + repo="https://github.com/user/repo-e", + ) + + self.url = reverse("migrate_to_github_app") + self.client.force_login(self.user) + + def _create_github_app_remote_repository(self, remote_repository): + new_remote_repository = get( + RemoteRepository, + name=remote_repository.name, + full_name=remote_repository.full_name, + html_url=remote_repository.html_url, + remote_id=remote_repository.remote_id, + vcs_provider=GITHUB_APP, + github_app_installation=self.github_app_installation, + ) + if remote_repository.organization: + new_remote_repository.organization = get( + RemoteOrganization, + slug=remote_repository.organization.slug, + name=remote_repository.organization.name, + remote_id=remote_repository.organization.remote_id, + vcs_provider=GITHUB_APP, + ) + new_remote_repository.github_app_installation = ( + self.github_app_organization_installation + ) + new_remote_repository.save() + for relation in remote_repository.remote_repository_relations.all(): + github_app_account = relation.user.socialaccount_set.get( + provider=GitHubAppProvider.id + ) + get( + RemoteRepositoryRelation, + user=relation.user, + account=github_app_account, + remote_repository=new_remote_repository, + admin=relation.admin, + ) + return new_remote_repository + + def test_user_without_github_account(self): + self.user.socialaccount_set.all().delete() + response = self.client.get(self.url) + assert response.status_code == 302 + response = self.client.get(reverse("projects_dashboard")) + content = response.content.decode() + print(content) + assert ( + "You don\\u0026#x27\\u003Bt have any GitHub account connected." in content + ) + + def test_user_without_github_account_but_with_github_app_account(self): + self.user.socialaccount_set.exclude(provider=GitHubAppProvider.id).delete() + response = self.client.get(self.url) + assert response.status_code == 302 + response = self.client.get(reverse("projects_dashboard")) + assert ( + "You have already migrated your account to the new GitHub App" + in response.content.decode() + ) + + @requests_mock.Mocker(kw="request") + def test_migration_page_initial_state(self, request): + request.get("https://api.github.com/user", status_code=200) + + self.user.socialaccount_set.filter(provider=GitHubAppProvider.id).delete() + response = self.client.get(self.url) + assert response.status_code == 200 + context = response.context + + assert context["step"] == "overview" + assert context["has_multiple_github_accounts"] is False + assert context["step_connect_completed"] is False + assert context["installation_target_groups"] == [ + InstallationTargetGroup( + target_id=int(self.social_account_github.uid), + target_type=GitHubAccountType.USER, + target_name="user", + repository_ids={1111, 2222, 3333}, + ), + InstallationTargetGroup( + target_id=int(self.remote_organization.remote_id), + target_type=GitHubAccountType.ORGANIZATION, + target_name="org", + repository_ids={4444}, + ), + ] + assert context["github_app_name"] == "readthedocs" + assert context["migration_targets"] == [ + MigrationTarget( + project=self.project_with_remote_repository, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_admin, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_member, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_organization, + has_installation=False, + is_admin=False, + target_id=int(self.remote_organization.remote_id), + ), + ] + assert list(context["migrated_projects"]) == [] + assert ( + context["old_application_link"] + == "https://github.com/settings/connections/applications/123" + ) + assert context["step_revoke_completed"] is False + assert context["old_github_account"] == self.social_account_github + + @requests_mock.Mocker(kw="request") + def test_migration_page_step_connect_done(self, request): + request.get("https://api.github.com/user", status_code=200) + response = self.client.get(self.url) + assert response.status_code == 200 + context = response.context + + assert context["step"] == "overview" + assert context["has_multiple_github_accounts"] is False + assert context["step_connect_completed"] is True + assert context["installation_target_groups"] == [ + InstallationTargetGroup( + target_id=int(self.social_account_github.uid), + target_type=GitHubAccountType.USER, + target_name="user", + repository_ids={1111, 2222, 3333}, + ), + InstallationTargetGroup( + target_id=int(self.remote_organization.remote_id), + target_type=GitHubAccountType.ORGANIZATION, + target_name="org", + repository_ids={4444}, + ), + ] + assert context["github_app_name"] == "readthedocs" + assert context["migration_targets"] == [ + MigrationTarget( + project=self.project_with_remote_repository, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_admin, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_member, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_organization, + has_installation=False, + is_admin=False, + target_id=int(self.remote_organization.remote_id), + ), + ] + assert list(context["migrated_projects"]) == [] + assert ( + context["old_application_link"] + == "https://github.com/settings/connections/applications/123" + ) + assert context["step_revoke_completed"] is False + assert context["old_github_account"] == self.social_account_github + + @requests_mock.Mocker(kw="request") + def test_migration_page_step_install_done(self, request): + request.get("https://api.github.com/user", status_code=200) + + self._create_github_app_remote_repository(self.remote_repository_a) + self._create_github_app_remote_repository(self.remote_repository_b) + self._create_github_app_remote_repository(self.remote_repository_d) + + response = self.client.get(self.url) + assert response.status_code == 200 + context = response.context + + assert context["step"] == "overview" + assert context["has_multiple_github_accounts"] is False + assert context["step_connect_completed"] is True + assert context["installation_target_groups"] == [ + InstallationTargetGroup( + target_id=int(self.social_account_github.uid), + target_type=GitHubAccountType.USER, + target_name="user", + repository_ids={3333}, + ), + InstallationTargetGroup( + target_id=int(self.remote_organization.remote_id), + target_type=GitHubAccountType.ORGANIZATION, + target_name="org", + repository_ids=set(), + ), + ] + assert context["github_app_name"] == "readthedocs" + assert context["migration_targets"] == [ + MigrationTarget( + project=self.project_with_remote_repository, + has_installation=True, + is_admin=True, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_admin, + has_installation=True, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_member, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_organization, + has_installation=True, + is_admin=True, + target_id=int(self.remote_organization.remote_id), + ), + ] + assert list(context["migrated_projects"]) == [] + assert ( + context["old_application_link"] + == "https://github.com/settings/connections/applications/123" + ) + assert context["step_revoke_completed"] is False + assert context["old_github_account"] == self.social_account_github + + @requests_mock.Mocker(kw="request") + @mock.patch.object(GitHubService, "remove_webhook") + @mock.patch.object(GitHubService, "remove_ssh_key") + def test_migration_page_step_migrate_one_project( + self, remove_ssh_key, remove_webhook, request + ): + request.get("https://api.github.com/user", status_code=200) + + remove_ssh_key.return_value = True + remove_webhook.return_value = True + + self._create_github_app_remote_repository(self.remote_repository_a) + self._create_github_app_remote_repository(self.remote_repository_b) + self._create_github_app_remote_repository(self.remote_repository_d) + + response = self.client.post( + self.url, data={"project": self.project_with_remote_repository.slug} + ) + assert response.status_code == 302 + response = self.client.get(self.url) + context = response.context + + assert context["step"] == "overview" + assert context["has_multiple_github_accounts"] is False + assert context["step_connect_completed"] is True + assert context["installation_target_groups"] == [ + InstallationTargetGroup( + target_id=int(self.social_account_github.uid), + target_type=GitHubAccountType.USER, + target_name="user", + repository_ids={3333}, + ), + InstallationTargetGroup( + target_id=int(self.remote_organization.remote_id), + target_type=GitHubAccountType.ORGANIZATION, + target_name="org", + repository_ids=set(), + ), + ] + assert context["github_app_name"] == "readthedocs" + assert context["migration_targets"] == [ + MigrationTarget( + project=self.project_with_remote_repository_no_admin, + has_installation=True, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_member, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_organization, + has_installation=True, + is_admin=True, + target_id=int(self.remote_organization.remote_id), + ), + ] + assert list(context["migrated_projects"]) == [ + self.project_with_remote_repository, + ] + assert ( + context["old_application_link"] + == "https://github.com/settings/connections/applications/123" + ) + assert context["step_revoke_completed"] is False + assert context["old_github_account"] == self.social_account_github + + @requests_mock.Mocker(kw="request") + @mock.patch.object(GitHubService, "remove_webhook") + @mock.patch.object(GitHubService, "remove_ssh_key") + def test_migration_page_step_migrate_all_projects( + self, remove_ssh_key, remove_webhook, request + ): + request.get("https://api.github.com/user", status_code=200) + + remove_ssh_key.return_value = True + remove_webhook.return_value = True + + self._create_github_app_remote_repository(self.remote_repository_a) + self._create_github_app_remote_repository(self.remote_repository_b) + self._create_github_app_remote_repository(self.remote_repository_d) + + response = self.client.post(self.url) + assert response.status_code == 302 + response = self.client.get(self.url) + context = response.context + + assert context["step"] == "overview" + assert context["has_multiple_github_accounts"] is False + assert context["step_connect_completed"] is True + assert context["installation_target_groups"] == [ + InstallationTargetGroup( + target_id=int(self.social_account_github.uid), + target_type=GitHubAccountType.USER, + target_name="user", + repository_ids={3333}, + ), + ] + assert context["github_app_name"] == "readthedocs" + assert context["migration_targets"] == [ + MigrationTarget( + project=self.project_with_remote_repository_no_admin, + has_installation=True, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_member, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + ] + assert list(context["migrated_projects"]) == [ + self.project_with_remote_repository, + self.project_with_remote_organization, + ] + assert ( + context["old_application_link"] + == "https://github.com/settings/connections/applications/123" + ) + assert context["step_revoke_completed"] is False + assert context["old_github_account"] == self.social_account_github + + @requests_mock.Mocker(kw="request") + @mock.patch.object(GitHubService, "remove_webhook") + @mock.patch.object(GitHubService, "remove_ssh_key") + def test_migration_page_step_migrate_one_project_with_errors( + self, remove_ssh_key, remove_webhook, request + ): + request.get("https://api.github.com/user", status_code=200) + + remove_ssh_key.return_value = False + remove_webhook.return_value = False + + self._create_github_app_remote_repository(self.remote_repository_a) + self._create_github_app_remote_repository(self.remote_repository_b) + self._create_github_app_remote_repository(self.remote_repository_d) + + response = self.client.post( + self.url, data={"project": self.project_with_remote_repository.slug} + ) + assert response.status_code == 302 + response = self.client.get(self.url) + context = response.context + + assert context["step"] == "overview" + assert context["has_multiple_github_accounts"] is False + assert context["step_connect_completed"] is True + assert context["installation_target_groups"] == [ + InstallationTargetGroup( + target_id=int(self.social_account_github.uid), + target_type=GitHubAccountType.USER, + target_name="user", + repository_ids={3333}, + ), + InstallationTargetGroup( + target_id=int(self.remote_organization.remote_id), + target_type=GitHubAccountType.ORGANIZATION, + target_name="org", + repository_ids=set(), + ), + ] + assert context["github_app_name"] == "readthedocs" + assert context["migration_targets"] == [ + MigrationTarget( + project=self.project_with_remote_repository_no_admin, + has_installation=True, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_member, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_organization, + has_installation=True, + is_admin=True, + target_id=int(self.remote_organization.remote_id), + ), + ] + assert list(context["migrated_projects"]) == [ + self.project_with_remote_repository, + ] + assert ( + context["old_application_link"] + == "https://github.com/settings/connections/applications/123" + ) + assert context["step_revoke_completed"] is False + assert context["old_github_account"] == self.social_account_github + + notifications = Notification.objects.for_user(self.user, self.user) + assert notifications.count() == 2 + assert notifications.filter( + message_id=MESSAGE_OAUTH_WEBHOOK_NOT_REMOVED + ).exists() + assert notifications.filter( + message_id=MESSAGE_OAUTH_DEPLOY_KEY_NOT_REMOVED + ).exists() + + @requests_mock.Mocker(kw="request") + def test_migration_page_step_revoke_done(self, request): + request.get("https://api.github.com/user", status_code=401) + response = self.client.get(self.url) + assert response.status_code == 200 + context = response.context + + assert context["step"] == "overview" + assert context["has_multiple_github_accounts"] is False + assert context["step_connect_completed"] is True + assert context["installation_target_groups"] == [ + InstallationTargetGroup( + target_id=int(self.social_account_github.uid), + target_type=GitHubAccountType.USER, + target_name="user", + repository_ids={1111, 2222, 3333}, + ), + InstallationTargetGroup( + target_id=int(self.remote_organization.remote_id), + target_type=GitHubAccountType.ORGANIZATION, + target_name="org", + repository_ids={4444}, + ), + ] + assert context["github_app_name"] == "readthedocs" + assert context["migration_targets"] == [ + MigrationTarget( + project=self.project_with_remote_repository, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_admin, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_repository_no_member, + has_installation=False, + is_admin=False, + target_id=int(self.social_account_github.uid), + ), + MigrationTarget( + project=self.project_with_remote_organization, + has_installation=False, + is_admin=False, + target_id=int(self.remote_organization.remote_id), + ), + ] + assert list(context["migrated_projects"]) == [] + assert ( + context["old_application_link"] + == "https://github.com/settings/connections/applications/123" + ) + assert context["step_revoke_completed"] is True + assert context["old_github_account"] == self.social_account_github diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index 72d20a5de96..937e99fd214 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -349,7 +349,7 @@ def get_context_data(self, **kwargs): ) context["step_connect_completed"] = self._has_new_account_for_old_account() context["installation_target_groups"] = get_installation_target_groups_for_user(user) - context["gh_app_name"] = settings.GITHUB_APP_NAME + context["github_app_name"] = settings.GITHUB_APP_NAME context["migration_targets"] = get_migration_targets(user) context["migrated_projects"] = ( AdminPermission.projects(user, admin=True) From 08f9d2c11c0c63fa1e1c9f6e544a6091713c748f Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 25 Mar 2025 15:21:41 -0500 Subject: [PATCH 82/92] Fix merge conflict --- readthedocs/core/views/hooks.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index baa71dc4aed..5f50408df09 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -81,33 +81,6 @@ def build_versions_from_names(project, versions_info: list[VersionInfo]): return to_build, not_building -def build_versions_from_names(project, version_names: list[tuple[str, str]]): - """ - Build the branches or tags from the project. - - :param project: Project instance - :param version_names: A list of tuples with the version name and type. - :returns: A tuple with the versions that were built and the versions that were not built. - """ - to_build = set() - not_building = set() - for version_name, version_type in version_names: - for version in project.versions_from_name(version_name, version_type): - log.debug( - "Processing.", - project_slug=project.slug, - version_slug=version.slug, - ) - if version.slug in to_build: - continue - triggered = _build_version(project, version) - if triggered: - to_build.add(triggered) - else: - not_building.add(version.slug) - return to_build, not_building - - def trigger_sync_versions(project): """ Sync the versions of a repo using its latest version. From ce6c2da2fc0a58c574b2274446e8f640778953c2 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 25 Mar 2025 15:30:41 -0500 Subject: [PATCH 83/92] Match changes --- readthedocs/oauth/views.py | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/readthedocs/oauth/views.py b/readthedocs/oauth/views.py index 27682d897e2..825b8058cbd 100644 --- a/readthedocs/oauth/views.py +++ b/readthedocs/oauth/views.py @@ -11,8 +11,7 @@ from readthedocs.api.v2.views.integrations import GITHUB_SIGNATURE_HEADER from readthedocs.api.v2.views.integrations import ExternalVersionData from readthedocs.api.v2.views.integrations import WebhookMixin -from readthedocs.builds.constants import BRANCH -from readthedocs.builds.constants import TAG +from readthedocs.core.views.hooks import VersionInfo from readthedocs.core.views.hooks import build_external_version from readthedocs.core.views.hooks import build_versions_from_names from readthedocs.core.views.hooks import close_external_version @@ -21,6 +20,7 @@ from readthedocs.oauth.models import GitHubAppInstallation from readthedocs.oauth.services.githubapp import get_gh_app_client from readthedocs.projects.models import Project +from readthedocs.vcs_support.backends.git import parse_version_from_ref log = structlog.get_logger(__name__) @@ -341,28 +341,9 @@ def _handle_push_event(self): # If this is a push to an existing branch or tag, # we need to build the version if active. - version_name, version_type = self._parse_version_from_ref(data["ref"]) + version_name, version_type = parse_version_from_ref(data["ref"]) for project in self._get_projects(): - build_versions_from_names(project, [(version_name, version_type)]) - - def _parse_version_from_ref(self, ref: str): - """ - Parse the version name and type from a GitHub ref. - - The ref can be a branch or a tag. - - :param ref: The ref to parse (e.g. refs/heads/main, refs/tags/v1.0.0). - :returns: A tuple with the version name and type. - """ - heads_prefix = "refs/heads/" - tags_prefix = "refs/tags/" - if ref.startswith(heads_prefix): - return ref.removeprefix(heads_prefix), BRANCH - if ref.startswith(tags_prefix): - return ref.removeprefix(tags_prefix), TAG - - # NOTE: this should never happen. - raise ValidationError(f"Invalid ref: {ref}") + build_versions_from_names(project, [VersionInfo(name=version_name, type=version_type)]) def _handle_pull_request_event(self): """ From c4891373f767bc846819e3039b0a245d10c1038e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 27 Mar 2025 14:17:36 -0500 Subject: [PATCH 84/92] Decouple migration from current GH account --- readthedocs/oauth/migrate.py | 188 +++++++++++++++++++++-------------- 1 file changed, 111 insertions(+), 77 deletions(-) diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index 252eb6c98b1..21fb82acbb7 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -2,9 +2,11 @@ from dataclasses import dataclass +from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.providers.github.provider import GitHubProvider from django.conf import settings +from readthedocs.allauth.providers.githubapp.provider import GitHubAppProvider from readthedocs.core.permissions import AdminPermission from readthedocs.integrations.models import Integration from readthedocs.oauth.constants import GITHUB @@ -16,6 +18,13 @@ from readthedocs.projects.models import Project +@dataclass +class GitHubAccountTarget: + login: str + id: int + type: GitHubAccountType + + @dataclass class InstallationTargetGroup: """Group of repositories that should be installed in the same target (user or organization).""" @@ -53,41 +62,117 @@ def installed(self): return not bool(self.repository_ids) +@dataclass +class MigrationTarget: + """Information about an individual project that needs to be migrated.""" + + project: Project + has_installation: bool + is_admin: bool + target_id: int + + @property + def installation_link(self): + """ + Create a link to install the GitHub App on the target repository. + + See https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/migrating-oauth-apps-to-github-apps + """ + base_url = ( + f"https://github.com/apps/{settings.GITHUB_APP_NAME}/installations/new/permissions" + ) + return f"{base_url}?suggested_target_id={self.target_id}&repository_ids[]={self.project.remote_repository.remote_id}" + + @property + def can_be_migrated(self): + """ + Check if the project can be migrated. + + The project can be migrated if the user is an admin on the repository and the GitHub App is installed. + """ + return self.is_admin and self.has_installation + + +@dataclass +class MigrationResult: + """Result of a migration operation.""" + + webhook_removed: bool + ssh_key_removed: bool + + +class MigrationError(Exception): + """Error raised when a migration operation fails.""" + + pass + + def get_installation_target_groups_for_user(user) -> list[InstallationTargetGroup]: """Get all targets (accounts and organizations) that the user needs to install the GitHub App on.""" - targets = {} - account = user.socialaccount_set.filter(provider=GitHubProvider.id).first() - # Since we don't save the ID of the owner of each repository, we group all repositories - # that don't have an organization under the user account, + # that we aren't able to identify the owner into the user's account. # GitHub will ignore the repositories that the user doesn't own. - targets[int(account.uid)] = InstallationTargetGroup( - target_id=int(account.uid), - target_name=account.extra_data.get("login", "unknown"), - target_type=GitHubAccountType.USER, - repository_ids=set(), - ) + default_target_account = _get_default_github_account_target(user) + targets = {} for project, has_intallation, _ in _get_projects_missing_migration(user): remote_repository = project.remote_repository - if remote_repository.organization: - target_id = int(remote_repository.organization.remote_id) - if target_id not in targets: - targets[target_id] = InstallationTargetGroup( - target_id=target_id, - target_name=remote_repository.organization.slug, - target_type=GitHubAccountType.ORGANIZATION, - repository_ids=set(), - ) - else: - target_id = int(account.uid) - + target_account = _get_github_account_target(remote_repository) or default_target_account + if target_account.id not in targets: + targets[target_account.id] = InstallationTargetGroup( + target_id=target_account.id, + target_name=target_account.login, + target_type=target_account.type, + repository_ids=set(), + ) if not has_intallation: - targets[target_id].repository_ids.add(int(remote_repository.remote_id)) + targets[target_account.id].repository_ids.add(int(remote_repository.remote_id)) return list(targets.values()) +def _get_default_github_account_target(user): + # NOTE: there are some users that have more than one GH account connected. + # They will need to migrate each account at a time. + account = user.socialaccount_set.filter(provider=GitHubProvider.id).first() + if not account: + account = user.socialaccount_set.filter(provider=GitHubAppProvider.id).first() + + return GitHubAccountTarget( + login=account.extra_data.get("login", "ghost"), + id=int(account.uid), + type=GitHubAccountType.USER, + ) + + +def _get_github_account_target(remote_repository): + """ + Get the GitHub account target for a repository. + + This will return the account that owns the repository, if we can identify it. + For repositories owned by organizations, we return the organization account, + for repositories owned by users, we try to guess the account based on the repository owner + (as we don't save the owner ID in the repository). + """ + if remote_repository.organization: + return GitHubAccountTarget( + login=remote_repository.organization.slug, + id=int(remote_repository.organization.remote_id), + type=GitHubAccountType.ORGANIZATION, + ) + login = remote_repository.full_name.split("/", 1)[0] + account = SocialAccount.objects.filter( + provider__in=[GitHubProvider.id, GitHubAppProvider.id], extra_data__login=login + ).first() + if account: + return GitHubAccountTarget( + login=login, + id=int(account.uid), + type=GitHubAccountType.USER, + ) + return None + + def _get_projects_missing_migration(user): """ Get all projects where the user has admin permissions that are still connected to the old GitHub OAuth App. @@ -128,58 +213,19 @@ def get_valid_projects_missing_migration(user): yield project -@dataclass -class MigrationTarget: - """Information about an individual project that needs to be migrated.""" - - project: Project - has_installation: bool - is_admin: bool - target_id: int - - @property - def installation_link(self): - """ - Create a link to install the GitHub App on the target repository. - - See https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/migrating-oauth-apps-to-github-apps - """ - base_url = ( - f"https://github.com/apps/{settings.GITHUB_APP_NAME}/installations/new/permissions" - ) - return f"{base_url}?suggested_target_id={self.target_id}&repository_ids[]={self.project.remote_repository.remote_id}" - - @property - def can_be_migrated(self): - """ - Check if the project can be migrated. - - The project can be migrated if the user is an admin on the repository and the GitHub App is installed. - """ - return self.is_admin and self.has_installation - - def get_migration_targets(user): """Get all projects that the user needs to migrate to the GitHub App.""" targets = [] - # NOTE: there are some users that have more than one GH account connected. - # They will need to migrate each account at a time. - gh_account = user.socialaccount_set.filter(provider=GITHUB).first() - if not gh_account: - return targets - + default_target_account = _get_default_github_account_target(user) for project, has_installation, is_admin in _get_projects_missing_migration(user): remote_repository = project.remote_repository - if remote_repository.organization: - target_id = int(remote_repository.organization.remote_id) - else: - target_id = int(gh_account.uid) + target_account = _get_github_account_target(remote_repository) or default_target_account targets.append( MigrationTarget( project=project, has_installation=has_installation, is_admin=is_admin, - target_id=target_id, + target_id=target_account.id, ) ) return targets @@ -195,18 +241,6 @@ def get_old_app_link(): return f"https://github.com/settings/connections/applications/{client_id}" -@dataclass -class MigrationResult: - """Result of a migration operation.""" - - webhook_removed: bool - ssh_key_removed: bool - - -class MigrationError(Exception): - pass - - def migrate_project_to_github_app(project, user) -> MigrationResult: """ Migrate a project to the new GitHub App. From 3a88a621358d126169b4048767f204cd619dafc0 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 27 Mar 2025 14:47:55 -0500 Subject: [PATCH 85/92] More decoupling --- readthedocs/oauth/migrate.py | 23 +++++++++++++++++++++++ readthedocs/profiles/views.py | 22 ++++++++++------------ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index 21fb82acbb7..604da3ff1c6 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -128,6 +128,19 @@ def get_installation_target_groups_for_user(user) -> list[InstallationTargetGrou if not has_intallation: targets[target_account.id].repository_ids.add(int(remote_repository.remote_id)) + # Include accounts that have already migrated projects, + # so they are shown as "Installed" in the UI. + for project in get_migrated_projects(user): + remote_repository = project.remote_repository + target_account = _get_github_account_target(remote_repository) or default_target_account + if target_account.id not in targets: + targets[target_account.id] = InstallationTargetGroup( + target_id=target_account.id, + target_name=target_account.login, + target_type=GitHubAccountType.USER, + repository_ids=set(), + ) + return list(targets.values()) @@ -207,6 +220,16 @@ def _get_projects_missing_migration(user): yield project, has_installation, is_admin +def get_migrated_projects(user): + return ( + AdminPermission.projects(user, admin=True) + .filter(remote_repository__vcs_provider=GITHUB_APP) + .select_related( + "remote_repository", + ) + ) + + def get_valid_projects_missing_migration(user): for project, has_installation, is_admin in _get_projects_missing_migration(user): if has_installation and is_admin: diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index 937e99fd214..006c3c19972 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -38,8 +38,8 @@ from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.notifications.models import Notification from readthedocs.oauth.clients import get_oauth2_client -from readthedocs.oauth.constants import GITHUB_APP from readthedocs.oauth.migrate import get_installation_target_groups_for_user +from readthedocs.oauth.migrate import get_migrated_projects from readthedocs.oauth.migrate import get_migration_targets from readthedocs.oauth.migrate import get_old_app_link from readthedocs.oauth.migrate import get_valid_projects_missing_migration @@ -351,13 +351,7 @@ def get_context_data(self, **kwargs): context["installation_target_groups"] = get_installation_target_groups_for_user(user) context["github_app_name"] = settings.GITHUB_APP_NAME context["migration_targets"] = get_migration_targets(user) - context["migrated_projects"] = ( - AdminPermission.projects(user, admin=True) - .filter(remote_repository__vcs_provider=GITHUB_APP) - .select_related( - "remote_repository", - ) - ) + context["migrated_projects"] = get_migrated_projects(user) context["old_application_link"] = get_old_app_link() context["step_revoke_completed"] = self._is_access_to_old_github_account_revoked() context["old_github_account"] = self._get_old_github_account() @@ -365,6 +359,8 @@ def get_context_data(self, **kwargs): def _is_access_to_old_github_account_revoked(self): old_account = self._get_old_github_account() + if not old_account: + return True client = get_oauth2_client(old_account) if client is None: return True @@ -381,11 +377,13 @@ def _has_new_account_for_old_account(self): The new connected account must the same as the old one. """ - old_account = self._get_old_github_account() - return self.request.user.socialaccount_set.filter( + query = self.request.user.socialaccount_set.filter( provider=GitHubAppProvider.id, - uid=old_account.uid, - ).exists() + ) + old_account = self._get_old_github_account() + if old_account: + query.filter(uid=old_account.uid) + return query.exists() def _get_new_github_account(self): return self.request.user.socialaccount_set.filter(provider=GitHubAppProvider.id).first() From 928984b80147c81d4e89c2caef03a52cad1fdff9 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 27 Mar 2025 18:59:04 -0500 Subject: [PATCH 86/92] Very WIP docs --- docs/user/guides/connecting-git-account.rst | 6 +- docs/user/guides/pull-requests.rst | 27 +++--- docs/user/intro/add-project.rst | 1 + docs/user/reference/git-integration.rst | 96 +++++++++++++++++++-- docs/user/tutorial/index.rst | 26 ++---- 5 files changed, 112 insertions(+), 44 deletions(-) diff --git a/docs/user/guides/connecting-git-account.rst b/docs/user/guides/connecting-git-account.rst index 9410ffe2929..c2a5648529f 100644 --- a/docs/user/guides/connecting-git-account.rst +++ b/docs/user/guides/connecting-git-account.rst @@ -40,7 +40,7 @@ You will now see the account appear in the list of connected services. :align: center :alt: Screenshot of Read the Docs "Connected Services" page with multiple services connected - Connected Services [#f1]_ [#f2]_ shows the list of Git providers that + Connected Services [#f1]_ [#f2]_ shows the list of Git providers that are connected to your Read the Docs account. Now your connection is ready and you will be able to import and configure Git repositories with just a few clicks. @@ -57,11 +57,11 @@ You may at any time delete the connection from Read the Docs. Delete the connection makes Read the Docs forget the immediate access, but you should also disable our OAuth Application from your Git provider. -* On GitHub, navigate to `Authorized OAuth Apps`_. +* On GitHub, navigate to `Authorized GitHub Apps`_. * On Bitbucket, navigate to `Application Authorizations`_. * On GitLab, navigat to `Applications`_ -.. _Authorized OAuth Apps: https://github.com/settings/applications +.. _Authorized GitHub Apps: https://github.com/settings/apps/authorizations .. _Application Authorizations: https://bitbucket.org/account/settings/app-authorizations/ .. _Applications: https://gitlab.com/-/profile/applications diff --git a/docs/user/guides/pull-requests.rst b/docs/user/guides/pull-requests.rst index 97d57f55c32..f5df23997b5 100644 --- a/docs/user/guides/pull-requests.rst +++ b/docs/user/guides/pull-requests.rst @@ -4,15 +4,15 @@ How to configure pull request builds In this section, you can learn how to configure :doc:`pull request builds `. To enable pull request builds for your project, -your Read the Docs account needs to be connected to an account with a supported Git provider. +your Read the Docs project needs to be connected to a repository in a supported Git provider. See `Limitations`_ for more information. -If your account is already connected: +If your project is already connected: #. Go to your project dashboard -#. Go to :guilabel:`Admin`, then :guilabel:`Settings` +#. Go to :guilabel:`Settings`, then :guilabel:`Pull request builds` #. Enable the :guilabel:`Build pull requests for this project` option -#. Click on :guilabel:`Save` +#. Click on :guilabel:`Update` .. tip:: @@ -44,9 +44,9 @@ while private previews are only available to users with access to the Read the D To change the privacy level: #. Go to your project dashboard -#. Go to :guilabel:`Admin`, then :guilabel:`Settings` -#. Select your option in :guilabel:`Privacy level of builds from pull requests` -#. Click on :guilabel:`Save` +#. Go to :guilabel:`Settings`, then :guilabel:`Pull request builds` +#. Select your option in :guilabel:`Privacy level of builds of Pull Requests` +#. Click on :guilabel:`Update` Privacy levels work the same way as :ref:`normal versions `. @@ -54,8 +54,7 @@ Limitations ----------- - Pull requests are only available for **GitHub** and **GitLab** currently. Bitbucket is not yet supported. -- To enable this feature, your Read the Docs account needs to be connected to an - account with your Git provider. +- To enable this feature, your Read the Docs project needs to be connected to a repository in a supported Git provider. - Builds from pull requests have the same memory and time limitations :doc:`as regular builds `. - Additional formats like PDF aren't built in order to reduce build time. @@ -66,7 +65,10 @@ Troubleshooting --------------- No new builds are started when I open a pull request - The most common cause is that your repository's webhook is not configured to + The most common cause when using GitHub is that your Read the Docs project is not + connected to the corresponding repository on GitHub. + + The most common cause for GitLab and Bitbucket is that your repository's webhook is not configured to send Read the Docs pull request events. You'll need to re-sync your project's webhook integration to reconfigure the Read the Docs webhook. @@ -85,11 +87,6 @@ Build status is not being reported to your Git provider being updated with your Git provider, then your connected account may have out dated or insufficient permissions. - Make sure that you have granted access to the Read the Docs `GitHub OAuth App`_ for - your personal or organization GitHub account. - .. seealso:: - :ref:`guides/setup/git-repo-manual:Debugging webhooks` - :ref:`github-permission-troubleshooting` - -.. _GitHub OAuth App: https://github.com/settings/applications diff --git a/docs/user/intro/add-project.rst b/docs/user/intro/add-project.rst index 8545a9053ec..627f3995bd5 100644 --- a/docs/user/intro/add-project.rst +++ b/docs/user/intro/add-project.rst @@ -16,6 +16,7 @@ Automatically add your project #. Go to your :term:`dashboard`. #. Click on :guilabel:`Add project`. #. Type the name of the repository you want to add and click on it. + If you are using GitHub, make sure you have installed the :doc:`Read the Docs GitHub App ` in your repository. #. Click on :guilabel:`Continue`. #. Edit any of the pre-filled fields with information of the repository. #. Click on :guilabel:`Next`. diff --git a/docs/user/reference/git-integration.rst b/docs/user/reference/git-integration.rst index d2863677c80..2353d8474e0 100644 --- a/docs/user/reference/git-integration.rst +++ b/docs/user/reference/git-integration.rst @@ -46,10 +46,17 @@ you can follow the :doc:`/intro/add-project` guide to actually add your project How automatic configuration works --------------------------------- -When your Read the Docs account is connected to |git_providers_or| and you :doc:`add a new Read the Docs project `: +When you Read the Docs account is connected to GitHub, and you :doc:`add a new Read the Docs project `: + +* Read the Docs automatically connects your project with the GitHub repository, + and subscribes to the repository's events. +* Read the Docs makes use of its GitHub App to interact with your repository. + +When your Read the Docs account is connected to GitLab or Bitbucket, and you :doc:`add a new Read the Docs project `: * Read the Docs automatically creates a Read the Docs Integration that matches your Git provider. * Read the Docs creates an incoming webhook with your Git provider, which is automatically added to your Git repository's settings using the account connection. +* Read the Docs creates a deploy key for your Git repository, which is automatically added to your Git repository (when importing private repositories on |com_brand|). After project creation, you can continue to configure the project. @@ -64,7 +71,12 @@ including the ones that were automatically created. Read the Docs incoming webhook ------------------------------ -Accounts with |git_providers_and| integration automatically have Read the Docs' incoming :term:`webhook` configured on all Git repositories that are imported. +.. note:: + + When using GitHub, Read the Docs uses a GitHub App that subscribes to all required events, + so you don't need to create a webhook manually. + +Accounts with GitLab and Bitbucket integrations automatically have Read the Docs' incoming :term:`webhook` configured on all repositories that are imported. Other setups can set up the webhook through :doc:`manual configuration `. When an incoming webhook notification is received, @@ -97,16 +109,22 @@ Read the Docs uses `OAuth`_ to connect to your account at |git_providers_or|. You are asked to grant permissions for Read the Docs to perform a number of actions on your behalf. At the same time, we use this process for authentication (login) -since we trust that |git_providers_or| have verified your user account and email address. +since we trust that the user who connects the account is the owner of the account. By granting Read the Docs the requested permissions, we are issued a secret OAuth token from your Git provider. -Using the secret token, -we can automatically configure repositories during :doc:`project creation `. +In the case of GitLab and Bitbucket, we can use the secret token +to automatically configure repositories during :doc:`project creation `. We also use the token to send back build statuses and preview URLs for :doc:`pull requests `. .. _OAuth: https://en.wikipedia.org/wiki/OAuth +.. note:: + + For GitHub we use a GitHub App to interact with your repositories. + This means, that you need to install the Read the Docs GitHub App in your repository + to enable the automatic configuration. + .. note:: Access granted to Read the Docs can always be revoked. @@ -115,6 +133,12 @@ We also use the token to send back build statuses and preview URLs for :doc:`pul Git provider integrations ------------------------- +.. note:: + + When using GitHub, Read the Docs uses a GitHub App to interact with your repositories. + If the original user who connected the repository to Read the Docs loses access to the project or repository, + the GitHub App will still have access to the repository, and the integrations will continue to work. + If your project is using :doc:`Organizations ` (|com_brand|) or :term:`maintainers ` (|org_brand|), then you need to be aware of *who* is setting up the integration for the project. @@ -136,6 +160,42 @@ so that you can log in to Read the Docs with your connected account credentials. .. tab:: GitHub + Read the Docs requests the following permissions when connecting your Read the Docs account to GitHub. + + Account email addresses (read only) + We ask for this so we can verify your email address and create a Read the Docs account. + + When installing the Read the Docs GitHub App in a repository, you will be asked to grant the following permissions: + + - Repository permissions: + + Commit statuses (read and write) + This allows Read the Docs to report the status of the build to GitHub. + Contents (read only) + This allows Read the Docs to clone the repository and build the documentation. + Metadata (read only) + This allows Read the Docs to read the repository collaborators and the permissions they have on the repository. + This is used to determine if the user can connect the repository to a Read the Docs project. + Pull requests (read and write) + This allows Read the Docs to subscribe to pull request events, + and to create a comment on the pull request with information about the build. + + - Organization permissions + + Members (read only) + This allows Read the Docs to read the organization members. + + + .. tab:: GitHub (old OAuth app integration) + + .. note:: + + Read the Docs used to use a GitHub OAuth application for integration, + which has been replaced by a `GitHub App `__. + If you haven't migrated your projects to the new GitHub App, + we will still use the OAuth application to interact with your repositories, + but we recommend migrating to the GitHub App for a better experience and more granular permissions. + Read the Docs requests the following permissions (more precisely, `OAuth scopes`_) when connecting your Read the Docs account to GitHub. @@ -197,6 +257,32 @@ so that you can log in to Read the Docs with your connected account credentials. * API access (``api``) which is needed to create webhooks in GitLab +GitHub App +---------- + +.. note:: + + Read the Docs used to use a GitHub OAuth application for integration, + which has been replaced by a `GitHub App `__. + If you haven't migrated your projects to the new GitHub App, + we will still use the OAuth application similar to the other Git providers to interact with your repositories, + we recommend migrating to the GitHub App for a better experience and more granular permissions. + +When using GitHub, Read the Docs uses a GitHub App to interact with your repositories. +This has the following benefits over using an OAuth application (like the other Git providers): + +- More control over which repositories Read the Docs can access. + You don't need to grant access to all your repositories in order to create an account or import a single repository. +- No need to create webhooks on your repositories. + The GitHub App subscribes to all required events when you install it. +- No need to create a deploy key on your repository (|com_brand| only). + The GitHub App can clone your private repositories using a temporal token. +- If the original user who connected the repository to Read the Docs loses access to the project or repository, + the GitHub App will still have access to the repository. +- You can revoke access to the GitHub App at any time from your GitHub settings. +- Never out of sync with changes on your repository. + The GitHub App subscribes to all required events and will always keep your project up to date with your repository. + .. _github-permission-troubleshooting: GitHub permission troubleshooting diff --git a/docs/user/tutorial/index.rst b/docs/user/tutorial/index.rst index 8cd20098511..504d6b87678 100644 --- a/docs/user/tutorial/index.rst +++ b/docs/user/tutorial/index.rst @@ -70,14 +70,6 @@ On the authorization page, click the green :guilabel:`Authorize readthedocs` but GitHub authorization page -.. note:: - - Read the Docs needs elevated permissions to perform certain operations - that ensure that the workflow is as smooth as possible, - like installing :term:`webhooks `. - If you want to learn more, - check out :ref:`reference/git-integration:permissions for connected accounts`. - After that, you will be redirected to Read the Docs to confirm your e-mail and username. Click the :guilabel:`Sign Up »` button to create your account and open your :term:`dashboard`. @@ -95,16 +87,11 @@ Importing the project to Read the Docs To import your GitHub project to Read the Docs: -#. Click the :guilabel:`Import a Project` button on your `dashboard `_. +#. Click the :guilabel:`Add project` button on your `dashboard `_. -#. Click the |:heavy_plus_sign:| button to the right of your ``rtd-tutorial`` project. If the list of repositories is empty, click the |:arrows_counterclockwise:| button. +#. Click on :guilabel:`Install GitHub App on repository`, and choose your account and select the repository you created in the previous step. - .. figure:: /_static/images/tutorial/rtd-import-projects.gif - :width: 80% - :align: center - :alt: Import projects workflow - - Import projects workflow +#. Type the repository name in the search box, and select the repository from the list, and click on :guilabel:`Continue`. #. Enter some details about your Read the Docs project: @@ -113,9 +100,6 @@ To import your GitHub project to Read the Docs: so it is better if you prepend your username, for example ``{username}-rtd-tutorial``. - Repository URL - The URL that contains the documentation source. Leave the automatically filled value. - Default branch Name of the default branch of the project, leave it as ``main``. @@ -175,7 +159,7 @@ Configuring the project To update the project description and configure the notification settings: -#. Navigate back to the :term:`project page` and click the :guilabel:`⚙ Admin` button,to open the Settings page. +#. Navigate back to the :term:`project page` and click the :guilabel:`⚙ Settings` button, to open the settings page. #. Update the project description by adding the following text: @@ -194,7 +178,7 @@ and show you a preview of the documentation with those changes. To trigger builds from pull requests: -#. Click the :guilabel:`Settings` link on the left under the :guilabel:`⚙ Admin` menu, check the "Build pull requests for this project" checkbox, and click the :guilabel:`Save` button at the bottom of the page. +#. Click the :guilabel:`Pull request builds` link on the left under the :guilabel:`⚙ Settings` menu, check the "Build pull requests for this project" checkbox, and click the :guilabel:`Update` button at the bottom of the page. #. Make some changes to your documentation: From 6227ea3150f26f8644b93729dd78877331cef3a1 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 31 Mar 2025 13:25:34 -0500 Subject: [PATCH 87/92] Update docs --- docs/user/guides/private-submodules.rst | 16 ++- docs/user/guides/setup/git-repo-manual.rst | 8 +- docs/user/reference/git-integration.rst | 116 ++++++++++++--------- 3 files changed, 76 insertions(+), 64 deletions(-) diff --git a/docs/user/guides/private-submodules.rst b/docs/user/guides/private-submodules.rst index 0bc67d69310..eb9ea167a4f 100644 --- a/docs/user/guides/private-submodules.rst +++ b/docs/user/guides/private-submodules.rst @@ -8,9 +8,12 @@ How to use private Git submodules If you are using private Git repositories and they also contain private Git submodules, you need to follow a few special steps. -Read the Docs uses SSH keys (with read only permissions) in order to clone private repositories. -A SSH key is automatically generated and added to your main repository, but not to your submodules. -In order to give Read the Docs access to clone your submodules you'll need to add the public SSH key to each repository of your submodules. +Read the Docs uses SSH keys (with read only permissions) for GitLab and Bitbucket in order to clone private repositories, +this key is added to your main repository, but not to your submodules. +For GitHub we make use of a temporal token generated using our :ref:`GitHub App `. + +When a project is created, a SSH key is automatically generated. +You can use this SSH key to give Read the Docs access to clone your private submodules. .. note:: @@ -33,13 +36,6 @@ Since GitHub doesn't allow you to reuse a deploy key across different repositori you'll need to use `machine users `__ to give read access to several repositories using only one SSH key. -#. Remove the SSH deploy key that was added to the main repository on GitHub - - #. Go to your project on GitHub - #. Click on :guilabel:`Settings` - #. Click on :guilabel:`Deploy Keys` - #. Delete the key added by ``Read the Docs Commercial (readthedocs.com)`` - #. Create a GitHub user and give it read only permissions to all the necessary repositories. You can do this by adding the account as: diff --git a/docs/user/guides/setup/git-repo-manual.rst b/docs/user/guides/setup/git-repo-manual.rst index b4df8b6b508..bfa454759ef 100644 --- a/docs/user/guides/setup/git-repo-manual.rst +++ b/docs/user/guides/setup/git-repo-manual.rst @@ -205,13 +205,7 @@ Webhook activation failed. Make sure you have the necessary permissions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you find this error, -make sure your user has permissions over the repository. -In case of GitHub, -check that you have granted access to the Read the Docs `OAuth App`_ to your organization. -A similar workflow is required for other supported providers. - -.. _OAuth App: https://github.com/settings/applications - +make sure your user has admin permissions over the repository. My project isn't automatically building ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/user/reference/git-integration.rst b/docs/user/reference/git-integration.rst index 2353d8474e0..c563c56b160 100644 --- a/docs/user/reference/git-integration.rst +++ b/docs/user/reference/git-integration.rst @@ -12,8 +12,9 @@ Connecting your account provides the following features: See: :doc:`/intro/add-project`. ⚙️ Automatic configuration - Have your Git repository automatically configured with your Read the Docs :term:`webhook`, - which allows Read the Docs to build your docs on every change to your repository. + Have Read the Docs subscribe to your repository's events, + allowing us to build your docs on every change to your repository, + and keep in sync with your tags and branches. 🚥️ Commit status See your documentation build status as a commit status indicator on :doc:`pull request builds `. @@ -50,7 +51,7 @@ When you Read the Docs account is connected to GitHub, and you :doc:`add a new R * Read the Docs automatically connects your project with the GitHub repository, and subscribes to the repository's events. -* Read the Docs makes use of its GitHub App to interact with your repository. +* Read the Docs makes use of its :ref:`GitHub App ` to interact with your repository. When your Read the Docs account is connected to GitLab or Bitbucket, and you :doc:`add a new Read the Docs project `: @@ -73,8 +74,8 @@ Read the Docs incoming webhook .. note:: - When using GitHub, Read the Docs uses a GitHub App that subscribes to all required events, - so you don't need to create a webhook manually. + When using GitHub, Read the Docs uses a GitHub App that subscribes to all required events. + You don't need to create a webhook on your repository. Accounts with GitLab and Bitbucket integrations automatically have Read the Docs' incoming :term:`webhook` configured on all repositories that are imported. Other setups can set up the webhook through :doc:`manual configuration `. @@ -109,26 +110,20 @@ Read the Docs uses `OAuth`_ to connect to your account at |git_providers_or|. You are asked to grant permissions for Read the Docs to perform a number of actions on your behalf. At the same time, we use this process for authentication (login) -since we trust that the user who connects the account is the owner of the account. +since we trust that the user who connects the account is the owner of Git provider account. By granting Read the Docs the requested permissions, we are issued a secret OAuth token from your Git provider. In the case of GitLab and Bitbucket, we can use the secret token -to automatically configure repositories during :doc:`project creation `. -We also use the token to send back build statuses and preview URLs for :doc:`pull requests `. +to automatically configure repositories during :doc:`project creation `, +for GitHub, you need to install our :ref:`GitHub App ` in the repository you want to import. .. _OAuth: https://en.wikipedia.org/wiki/OAuth .. note:: - For GitHub we use a GitHub App to interact with your repositories. - This means, that you need to install the Read the Docs GitHub App in your repository - to enable the automatic configuration. - -.. note:: - - Access granted to Read the Docs can always be revoked. - This is a function offered by all Git providers. + Access granted to Read the Docs can always be revoked. + This is a function offered by all Git providers. Git provider integrations ------------------------- @@ -167,21 +162,19 @@ so that you can log in to Read the Docs with your connected account credentials. When installing the Read the Docs GitHub App in a repository, you will be asked to grant the following permissions: - - Repository permissions: - + Repository permissions Commit statuses (read and write) This allows Read the Docs to report the status of the build to GitHub. Contents (read only) This allows Read the Docs to clone the repository and build the documentation. Metadata (read only) This allows Read the Docs to read the repository collaborators and the permissions they have on the repository. - This is used to determine if the user can connect the repository to a Read the Docs project. + This is used to determine if the user can connect a repository to a Read the Docs project. Pull requests (read and write) This allows Read the Docs to subscribe to pull request events, and to create a comment on the pull request with information about the build. - - Organization permissions - + Organization permissions Members (read only) This allows Read the Docs to read the organization members. @@ -260,13 +253,19 @@ so that you can log in to Read the Docs with your connected account credentials. GitHub App ---------- -.. note:: +Read the Docs used to use a GitHub OAuth application for integration, +which has been replaced by a `GitHub App `__. +If you haven't migrated your projects to the new GitHub App, +we will still use the OAuth application similar to the other Git providers to interact with your repositories, +we recommend migrating to the GitHub App for a better experience and more granular permissions. + +We have two GitHub Apps, one for each of our platforms: + +- `Read the Docs Community `__. +- `Read the Docs for Business `__. - Read the Docs used to use a GitHub OAuth application for integration, - which has been replaced by a `GitHub App `__. - If you haven't migrated your projects to the new GitHub App, - we will still use the OAuth application similar to the other Git providers to interact with your repositories, - we recommend migrating to the GitHub App for a better experience and more granular permissions. +Features +~~~~~~~~ When using GitHub, Read the Docs uses a GitHub App to interact with your repositories. This has the following benefits over using an OAuth application (like the other Git providers): @@ -283,35 +282,58 @@ This has the following benefits over using an OAuth application (like the other - Never out of sync with changes on your repository. The GitHub App subscribes to all required events and will always keep your project up to date with your repository. -.. _github-permission-troubleshooting: +Revoking access +~~~~~~~~~~~~~~~ -GitHub permission troubleshooting ---------------------------------- +You can revoke access to the Read the Docs GitHub App at any time from your GitHub settings. -**Repositories not in your list to import**. +- `Read the Docs Community `__. +- `Read the Docs for Business `__. -Many organizations require approval for each OAuth application that is used, -or you might have disabled it in the past for your personal account. -This can happen at the personal or organization level, -depending on where the project you are trying to access has permissions from. +There are three ways to revoke access to the Read the Docs GitHub App: -.. tabs:: +Revoke access to one or more repositories: + Remove the repositories from the list of repositories that the GitHub App has access to. +Suspend the GitHub App: + This will suspend the GitHub App and revoke access to all repositories. + The installation and configuration will still be available, + and you can re-enable the GitHub App at any time. +Uninstall the GitHub App: + This will uninstall the GitHub App and revoke access to all repositories. + The installation and configuration will be removed, + and you will need to re-install the GitHub App and reconfigure it to use it again. + +.. warning:: + + If you revoke access to the GitHub App with any of the above methods, + all projects linked to that repository will stop working, + but the projects and its documentation will still be available. + If you grant access to the repository again, + you will need to manually connect your project to the repository. + +.. _github-permission-troubleshooting: - .. tab:: Personal Account +Troubleshooting +~~~~~~~~~~~~~~~ - You need to make sure that you have granted access to the Read the Docs `OAuth App`_ to your **personal GitHub account**. - If you do not see Read the Docs in the `OAuth App`_ settings, you might need to disconnect and reconnect the GitHub service. +**Repository not in the list to import** - .. seealso:: GitHub docs on `requesting access to your personal OAuth`_ for step-by-step instructions. +Make sure you have installed the corresponding GitHub App in your GitHub account or organization, +and have granted access to the repository you want to import. - .. _OAuth App: https://github.com/settings/applications - .. _requesting access to your personal OAuth: https://docs.github.com/en/organizations/restricting-access-to-your-organizations-data/approving-oauth-apps-for-your-organization +- `Read the Docs Community `__. +- `Read the Docs for Business `__. - .. tab:: Organization Account +If you still can't see the repository in the list, +you may need to wait a couple of minutes and refresh the page, +or click on the "Refresh your repositories" button in the import page. - You need to make sure that you have granted access to the Read the Docs OAuth App to your **organization GitHub account**. - If you don't see "Read the Docs" listed, then you might need to connect GitHub to your social accounts as noted above. +**Repository is in the list, but can't be imported** - .. seealso:: GitHub doc on `requesting access to your organization OAuth`_ for step-by-step instructions. +Make sure you have admin access to the repository you are trying to import. +If you are using |org_brand|, make sure your project is public, +or use |com_brand| to import private repositories. - .. _requesting access to your organization OAuth: https://docs.github.com/en/github/setting-up-and-managing-your-github-user-account/managing-your-membership-in-organizations/requesting-organization-approval-for-oauth-apps +If you still can't import the repository, +you may need to wait a couple of minutes and refresh the page, +or click on the "Refresh your repositories" button in the import page. From c962205fe3412df6954688bfc07186c276066f87 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 Apr 2025 14:06:54 -0500 Subject: [PATCH 88/92] Add old_github_accounts so we can use it as a listing --- readthedocs/profiles/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index 006c3c19972..93ef7edbc57 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -355,6 +355,8 @@ def get_context_data(self, **kwargs): context["old_application_link"] = get_old_app_link() context["step_revoke_completed"] = self._is_access_to_old_github_account_revoked() context["old_github_account"] = self._get_old_github_account() + # NOTE: this is a done, so the template can display this single element in a list. + context["old_github_accounts"] = [context["old_github_account"]] return context def _is_access_to_old_github_account_revoked(self): From 6c5eefbbc3e14c1a996f2dc4f28df93ea302b33b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 Apr 2025 17:54:20 -0500 Subject: [PATCH 89/92] Add type --- readthedocs/oauth/migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/migrate.py b/readthedocs/oauth/migrate.py index 604da3ff1c6..addb0dd262c 100644 --- a/readthedocs/oauth/migrate.py +++ b/readthedocs/oauth/migrate.py @@ -236,7 +236,7 @@ def get_valid_projects_missing_migration(user): yield project -def get_migration_targets(user): +def get_migration_targets(user) -> list[MigrationTarget]: """Get all projects that the user needs to migrate to the GitHub App.""" targets = [] default_target_account = _get_default_github_account_target(user) From f2d5498c2e267c4847ac6185f24420c835414f10 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 15 Apr 2025 12:53:49 -0500 Subject: [PATCH 90/92] Fix merge conflicts --- readthedocs/oauth/services/base.py | 2 +- requirements/docker.txt | 3 --- requirements/docs.txt | 14 -------------- requirements/pip.txt | 1 - requirements/testing.txt | 11 ----------- 5 files changed, 1 insertion(+), 30 deletions(-) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index f2c3e13b557..e336827b4c8 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -33,6 +33,7 @@ class Service: vcs_provider_slug: str allauth_provider = type[OAuth2Provider] + url_pattern: re.Pattern | None = None default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL @@ -269,7 +270,6 @@ def sync(self): # Delete RemoteOrganization where the user doesn't have access anymore organization_remote_ids = [o.remote_id for o in remote_organizations if o is not None] - ( self.user.remote_organization_relations.filter( account=self.account, diff --git a/requirements/docker.txt b/requirements/docker.txt index 652fa5727a2..adc4065f1d3 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -460,9 +460,6 @@ typing-extensions==4.13.2 # psycopg-pool # pydantic # pydantic-core - # rich - # tox -tzdata==2025.1 # typing-inspection typing-inspection==0.4.0 # via diff --git a/requirements/docs.txt b/requirements/docs.txt index 5b8b7164c0c..0335711d509 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -161,23 +161,9 @@ sphinxext-opengraph==0.10.0 # via -r requirements/docs.in starlette==0.46.1 # via sphinx-autobuild -<<<<<<< HEAD -typing-extensions==4.12.2 - # via anyio -urllib3==2.3.0 -||||||| 551e5e872 -tomli==2.2.1 - # via sphinx -typing-extensions==4.12.2 - # via - # anyio - # uvicorn -urllib3==2.3.0 -======= typing-extensions==4.13.2 # via anyio urllib3==2.4.0 ->>>>>>> main # via # requests # sphinx-prompt diff --git a/requirements/pip.txt b/requirements/pip.txt index bf0cfbc96bf..0eeedb75a4c 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -312,7 +312,6 @@ typing-extensions==4.13.2 # psycopg-pool # pydantic # pydantic-core -tzdata==2025.1 # typing-inspection typing-inspection==0.4.0 # via pydantic diff --git a/requirements/testing.txt b/requirements/testing.txt index ccf547403bd..85c20eca5b1 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -456,17 +456,6 @@ toml==0.10.2 # via # -r requirements/pip.txt # bumpver -tomli==2.2.1 - # via - # -r requirements/pip.txt - # coverage - # dparse - # pytest - # sphinx -typing-extensions==4.12.2 - # via - # -r requirements/pip.txt - # asgiref typing-extensions==4.13.2 # via # -r requirements/pip.txt From 34d7ed63b5ebaa22470a5d067e805acb378032f9 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 15 Apr 2025 12:55:40 -0500 Subject: [PATCH 91/92] Format --- docs/user/reference/git-integration.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user/reference/git-integration.rst b/docs/user/reference/git-integration.rst index c563c56b160..bb025fd5fc5 100644 --- a/docs/user/reference/git-integration.rst +++ b/docs/user/reference/git-integration.rst @@ -159,7 +159,7 @@ so that you can log in to Read the Docs with your connected account credentials. Account email addresses (read only) We ask for this so we can verify your email address and create a Read the Docs account. - + When installing the Read the Docs GitHub App in a repository, you will be asked to grant the following permissions: Repository permissions @@ -173,11 +173,11 @@ so that you can log in to Read the Docs with your connected account credentials. Pull requests (read and write) This allows Read the Docs to subscribe to pull request events, and to create a comment on the pull request with information about the build. - + Organization permissions Members (read only) This allows Read the Docs to read the organization members. - + .. tab:: GitHub (old OAuth app integration) From 225c640fcd1ba6cc64bfad3010249112657f1757 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 15 Apr 2025 12:57:23 -0500 Subject: [PATCH 92/92] Fix reference --- docs/user/intro/add-project.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/intro/add-project.rst b/docs/user/intro/add-project.rst index 627f3995bd5..7c5fa57b4b4 100644 --- a/docs/user/intro/add-project.rst +++ b/docs/user/intro/add-project.rst @@ -16,7 +16,7 @@ Automatically add your project #. Go to your :term:`dashboard`. #. Click on :guilabel:`Add project`. #. Type the name of the repository you want to add and click on it. - If you are using GitHub, make sure you have installed the :doc:`Read the Docs GitHub App ` in your repository. + If you are using GitHub, make sure you have installed the :ref:`Read the Docs GitHub App ` in your repository. #. Click on :guilabel:`Continue`. #. Edit any of the pre-filled fields with information of the repository. #. Click on :guilabel:`Next`.