Celery: cleanup pidbox keys (#10002)

* Celery: cleanup `pidbox` keys

Simple task to remove `pidbox` keys older than 15 minutes.
This is a workaround to avoid Redis OOM for now.

We will need to find out a better solution here. There is an upstream issue
opened that we should check in the near future and probably remove this
workaround: celery/celery#6089

* Celery: use `redis` to get the client

`app.backend.client` is not working anymore since we are not using a backend
result anymore.

Author: humitos

readthedocs/core/tasks.py

@@ -1,5 +1,8 @@
 """Basic tasks."""
 
+import math
+
+import redis
 import structlog
 from django.conf import settings
 from django.core.mail import EmailMultiAlternatives
@@ -56,3 +59,32 @@ def clear_persistent_messages():
         expires__lt=timezone.now(),
     )
     expired_messages.delete()
+
+
+@app.task(queue="web")
+def cleanup_pidbox_keys():
+    """
+    Remove "pidbox" objects from Redis.
+
+    Celery creates some "pidbox" objects with TTL=-1,
+    producing a OOM on our Redis instance.
+
+    This task is executed periodically to remove "pdibox" objects with an
+    idletime bigger than 5 minutes from Redis and free some RAM.
+
+    https://github.com/celery/celery/issues/6089
+    https://github.com/readthedocs/readthedocs-ops/issues/1260
+
+    """
+    client = redis.from_url(settings.BROKER_URL)
+    keys = client.keys("*reply.celery.pidbox*")
+    total_memory = 0
+    for key in keys:
+        idletime = client.object("idletime", key)  # seconds
+        memory = math.ceil(client.memory_usage(key) / 1024 / 1024)  # Mb
+        total_memory += memory
+
+        if idletime > (60 * 15):  # 15 minutes
+            client.delete([key])
+
+    log.info("Redis pidbox objects.", memory=total_memory, keys=len(keys))

readthedocs/settings/base.py

@@ -505,7 +505,12 @@ def TEMPLATES(self):
             'task': 'readthedocs.domains.tasks.email_pending_custom_domains',
             'schedule': crontab(minute=0, hour=3),
             'options': {'queue': 'web'},
-        }
+        },
+        'every-15m-delete-pidbox-objects': {
+            'task': 'readthedocs.core.tasks.cleanup_pidbox_keys',
+            'schedule': crontab(minute='*/15'),
+            'options': {'queue': 'web'},
+        },
     }
 
     MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE]