Celery: cleanup pidbox keys

readthedocs/core/tasks.py

@@ -1,5 +1,7 @@
 """Basic tasks."""
 
+import math
+
 import structlog
 from django.conf import settings
 from django.core.mail import EmailMultiAlternatives
@@ -56,3 +58,31 @@ def clear_persistent_messages():
         expires__lt=timezone.now(),
     )
     expired_messages.delete()
+
+
+@app.task(queue="web")
+def cleanup_pidbox_keys():
+    """
+    Remove "pidbox" objects from Redis.
+
+    Celery creates some "pidbox" objects with TTL=-1,
+    producing a OOM on our Redis instance.
+
+    This task is executed periodically to remove "pdibox" objects with an
+    idletime bigger than 5 minutes from Redis and free some RAM.
+
+    https://github.com/celery/celery/issues/6089
+    https://github.com/readthedocs/readthedocs-ops/issues/1260
+
+    """
+    total_memory = 0
+    keys = app.backend.client.keys("*reply.celery.pidbox*")
+    for key in keys:
+        idletime = app.backend.client.object("idletime", key)  # seconds
+        memory = math.ceil(app.backend.client.memory_usage(key) / 1024 / 1024)  # Mb
+        total_memory += memory
+
+        if idletime > (60 * 15):  # 15 minutes
+            app.backend.client.delete([key])
+
+    log.info("Redis pidbox objects.", memory=total_memory, keys=len(keys))

readthedocs/settings/base.py

@@ -503,7 +503,12 @@ def TEMPLATES(self):
             'task': 'readthedocs.domains.tasks.email_pending_custom_domains',
             'schedule': crontab(minute=0, hour=3),
             'options': {'queue': 'web'},
-        }
+        },
+        'every-15m-delete-pidbox-objects': {
+            'task': 'readthedocs.core.tasks.cleanup_pidbox_keys',
+            'schedule': crontab(minute='*/15'),
+            'options': {'queue': 'web'},
+        },
     }
 
     MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE]