From b97f82543918007d7878eb135586d7d36f1ff65e Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 8 Feb 2023 10:31:29 +0100 Subject: [PATCH 1/2] Celery: cleanup `pidbox` keys Simple task to remove `pidbox` keys older than 15 minutes. This is a workaround to avoid Redis OOM for now. We will need to find out a better solution here. There is an upstream issue opened that we should check in the near future and probably remove this workaround: https://github.com/celery/celery/issues/6089 --- readthedocs/core/tasks.py | 30 ++++++++++++++++++++++++++++++ readthedocs/settings/base.py | 7 ++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/readthedocs/core/tasks.py b/readthedocs/core/tasks.py index 7c9ca072699..b6179f74a6d 100644 --- a/readthedocs/core/tasks.py +++ b/readthedocs/core/tasks.py @@ -1,5 +1,7 @@ """Basic tasks.""" +import math + import structlog from django.conf import settings from django.core.mail import EmailMultiAlternatives @@ -56,3 +58,31 @@ def clear_persistent_messages(): expires__lt=timezone.now(), ) expired_messages.delete() + + +@app.task(queue="web") +def cleanup_pidbox_keys(): + """ + Remove "pidbox" objects from Redis. + + Celery creates some "pidbox" objects with TTL=-1, + producing a OOM on our Redis instance. + + This task is executed periodically to remove "pdibox" objects with an + idletime bigger than 5 minutes from Redis and free some RAM. + + https://github.com/celery/celery/issues/6089 + https://github.com/readthedocs/readthedocs-ops/issues/1260 + + """ + total_memory = 0 + keys = app.backend.client.keys("*reply.celery.pidbox*") + for key in keys: + idletime = app.backend.client.object("idletime", key) # seconds + memory = math.ceil(app.backend.client.memory_usage(key) / 1024 / 1024) # Mb + total_memory += memory + + if idletime > (60 * 15): # 15 minutes + app.backend.client.delete([key]) + + log.info("Redis pidbox objects.", memory=total_memory, keys=len(keys)) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 5e1a3df61d6..4fdbfa557d3 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -503,7 +503,12 @@ def TEMPLATES(self): 'task': 'readthedocs.domains.tasks.email_pending_custom_domains', 'schedule': crontab(minute=0, hour=3), 'options': {'queue': 'web'}, - } + }, + 'every-15m-delete-pidbox-objects': { + 'task': 'readthedocs.core.tasks.cleanup_pidbox_keys', + 'schedule': crontab(minute='*/15'), + 'options': {'queue': 'web'}, + }, } MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE] From 729f533925192650a0b6c52cfd84a5491a3a97ef Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 13 Feb 2023 11:10:34 +0100 Subject: [PATCH 2/2] Celery: use `redis` to get the client `app.backend.client` is not working anymore since we are not using a backend result anymore. --- readthedocs/core/tasks.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/readthedocs/core/tasks.py b/readthedocs/core/tasks.py index b6179f74a6d..9b013e21136 100644 --- a/readthedocs/core/tasks.py +++ b/readthedocs/core/tasks.py @@ -2,6 +2,7 @@ import math +import redis import structlog from django.conf import settings from django.core.mail import EmailMultiAlternatives @@ -75,14 +76,15 @@ def cleanup_pidbox_keys(): https://github.com/readthedocs/readthedocs-ops/issues/1260 """ + client = redis.from_url(settings.BROKER_URL) + keys = client.keys("*reply.celery.pidbox*") total_memory = 0 - keys = app.backend.client.keys("*reply.celery.pidbox*") for key in keys: - idletime = app.backend.client.object("idletime", key) # seconds - memory = math.ceil(app.backend.client.memory_usage(key) / 1024 / 1024) # Mb + idletime = client.object("idletime", key) # seconds + memory = math.ceil(client.memory_usage(key) / 1024 / 1024) # Mb total_memory += memory if idletime > (60 * 15): # 15 minutes - app.backend.client.delete([key]) + client.delete([key]) log.info("Redis pidbox objects.", memory=total_memory, keys=len(keys))