From 6b418f850f86d99caa90076a7040ba07409f8613 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 19 Oct 2021 12:01:34 +0200 Subject: [PATCH 1/4] SSO: re-sync VCS accounts for SSO organization daily Created a Celery scheduled task to be called every day and re-sync all the `RemoteRepository` and `RemoteRepositoryRelation` for all the users belonging to an organization with AllAuth SSO enabled. --- readthedocs/oauth/tasks.py | 25 +++++++++++++++++++++++++ readthedocs/settings/base.py | 5 +++++ 2 files changed, 30 insertions(+) diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index 518ad2b1abb..9a9058f2e98 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -6,6 +6,7 @@ from django.contrib.auth.models import User from django.utils.translation import ugettext_lazy as _ +from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.tasks import PublicTask, user_id_matches from readthedocs.oauth.notifications import ( AttachWebhookNotification, @@ -49,6 +50,30 @@ def sync_remote_repositories(user_id): ) +@app.task(queue='web') +def sync_remote_repositories_organizations(): + """ + Re-sync users member of organizations with SSO enabled. + + It will trigger one `sync_remote_repositories` task per user. + """ + query = ( + SSOIntegration.objects + .filter(provider=SSOIntegration.PROVIDER_ALLAUTH) + .values_list('organization', flat=True) + ) + log.info('Triggering scheduled SSO re-sync for all organizations. count=%s', query.count()) + for organization in query: + members = AdminPermission.members(organization) + log.info( + 'Triggering scheduled SSO re-sync for organization. organization=%s users=%s', + organization.slug, + members.count(), + ) + for user in members: + sync_remote_repositories.delay(user.pk) + + @app.task(queue='web') def attach_webhook(project_pk, user_pk, integration=None): """ diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index ea6e4584340..fdba0bfa610 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -422,6 +422,11 @@ def TEMPLATES(self): 'schedule': crontab(minute=0, hour=1), 'options': {'queue': 'web'}, }, + 'every-day-resync-sso-organization-users': { + 'task': 'readthedocs.oauth.tasks.sync_remote_repositories_organizations', + 'schedule': crontab(minute=0, hour=4), + 'options': {'queue': 'web'}, + }, 'hourly-archive-builds': { 'task': 'readthedocs.builds.tasks.archive_builds', 'schedule': crontab(minute=30), From 4271c694f1a625caba4c9310a780eaf49bbbce26 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 21 Oct 2021 11:02:17 +0200 Subject: [PATCH 2/4] Delay each task to re-sync users by 5 seconds between each --- readthedocs/oauth/tasks.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index 9a9058f2e98..9673831ef9b 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -63,6 +63,7 @@ def sync_remote_repositories_organizations(): .values_list('organization', flat=True) ) log.info('Triggering scheduled SSO re-sync for all organizations. count=%s', query.count()) + n_task = -1 for organization in query: members = AdminPermission.members(organization) log.info( @@ -71,7 +72,12 @@ def sync_remote_repositories_organizations(): members.count(), ) for user in members: - sync_remote_repositories.delay(user.pk) + n_task += 1 + sync_remote_repositories.delay( + user.pk, + # delay the task by 0, 5, 10, 15, ... seconds + countdown=n_task * 5, + ) @app.task(queue='web') From a759bd3e8c2be2762fc63d165f0240e16fc64513 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 21 Oct 2021 13:59:35 +0200 Subject: [PATCH 3/4] Add missing import --- readthedocs/oauth/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index 9673831ef9b..ba3fa9f6381 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -15,6 +15,7 @@ from readthedocs.oauth.services.base import SyncServiceError from readthedocs.oauth.utils import SERVICE_MAP from readthedocs.projects.models import Project +from readthedocs.sso.models import SSOIntegration from readthedocs.worker import app from .services import registry From a9a0379ecf14804cacbb230b0e3ed593915cad04 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 21 Oct 2021 14:07:18 +0200 Subject: [PATCH 4/4] Update code to support passing organization's slugs --- readthedocs/oauth/tasks.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index ba3fa9f6381..aaa5b10a378 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -14,6 +14,7 @@ ) from readthedocs.oauth.services.base import SyncServiceError from readthedocs.oauth.utils import SERVICE_MAP +from readthedocs.organizations.models import Organization from readthedocs.projects.models import Project from readthedocs.sso.models import SSOIntegration from readthedocs.worker import app @@ -52,23 +53,40 @@ def sync_remote_repositories(user_id): @app.task(queue='web') -def sync_remote_repositories_organizations(): +def sync_remote_repositories_organizations(organization_slugs=None): """ - Re-sync users member of organizations with SSO enabled. + Re-sync users member of organizations. It will trigger one `sync_remote_repositories` task per user. + + :param organization_slugs: list containg organization's slugs to sync. If + not passed, all organizations with ALLAUTH SSO enabled will be synced + + :type organization_slugs: list """ - query = ( - SSOIntegration.objects - .filter(provider=SSOIntegration.PROVIDER_ALLAUTH) - .values_list('organization', flat=True) - ) - log.info('Triggering scheduled SSO re-sync for all organizations. count=%s', query.count()) + if organization_slugs: + query = Organization.objects.filter(slug__in=organization_slugs) + log.info( + 'Triggering SSO re-sync for organizations. slugs=%s count=%s', + organization_slugs, + query.count(), + ) + else: + log.info( + 'Triggering SSO re-sync for all organizations. count=%s', + query.count(), + ) + query = ( + SSOIntegration.objects + .filter(provider=SSOIntegration.PROVIDER_ALLAUTH) + .values_list('organization', flat=True) + ) + n_task = -1 for organization in query: members = AdminPermission.members(organization) log.info( - 'Triggering scheduled SSO re-sync for organization. organization=%s users=%s', + 'Triggering SSO re-sync for organization. organization=%s users=%s', organization.slug, members.count(), )