diff --git a/readthedocs/core/tasks.py b/readthedocs/core/tasks.py index 7c9ca072699..9b013e21136 100644 --- a/readthedocs/core/tasks.py +++ b/readthedocs/core/tasks.py @@ -1,5 +1,8 @@ """Basic tasks.""" +import math + +import redis import structlog from django.conf import settings from django.core.mail import EmailMultiAlternatives @@ -56,3 +59,32 @@ def clear_persistent_messages(): expires__lt=timezone.now(), ) expired_messages.delete() + + +@app.task(queue="web") +def cleanup_pidbox_keys(): + """ + Remove "pidbox" objects from Redis. + + Celery creates some "pidbox" objects with TTL=-1, + producing a OOM on our Redis instance. + + This task is executed periodically to remove "pdibox" objects with an + idletime bigger than 5 minutes from Redis and free some RAM. + + https://github.com/celery/celery/issues/6089 + https://github.com/readthedocs/readthedocs-ops/issues/1260 + + """ + client = redis.from_url(settings.BROKER_URL) + keys = client.keys("*reply.celery.pidbox*") + total_memory = 0 + for key in keys: + idletime = client.object("idletime", key) # seconds + memory = math.ceil(client.memory_usage(key) / 1024 / 1024) # Mb + total_memory += memory + + if idletime > (60 * 15): # 15 minutes + client.delete([key]) + + log.info("Redis pidbox objects.", memory=total_memory, keys=len(keys)) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 5e1a3df61d6..4fdbfa557d3 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -503,7 +503,12 @@ def TEMPLATES(self): 'task': 'readthedocs.domains.tasks.email_pending_custom_domains', 'schedule': crontab(minute=0, hour=3), 'options': {'queue': 'web'}, - } + }, + 'every-15m-delete-pidbox-objects': { + 'task': 'readthedocs.core.tasks.cleanup_pidbox_keys', + 'schedule': crontab(minute='*/15'), + 'options': {'queue': 'web'}, + }, } MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE]