Merge pull request #6879 from readthedocs/dynamic-build-queue

Allocate docker limits based on server size.

Author: ericholscher

readthedocs/doc_builder/constants.py

@@ -4,6 +4,7 @@
 
 import logging
 import re
+import subprocess
 
 from django.conf import settings
 
@@ -25,8 +26,44 @@
     )
     DOCKER_IMAGE_SETTINGS.update(old_config)
 
-DOCKER_LIMITS = {'memory': '200m', 'time': 600}
-DOCKER_LIMITS.update(settings.DOCKER_LIMITS)
+DOCKER_LIMITS = {
+    'memory': '200m',
+    'time': 600,
+}
+
+# Set docker limits dynamically based on system memory
+# This assumes 1-builder per server
+try:
+    total_memory = int(subprocess.check_output("free -m | awk '/^Mem:/{print $2}'", shell=True))
+except ValueError:
+    # On systems without a `free` command it will return a string to int and raise a ValueError
+    log.exception('Failed to get memory size. Using defaults docker limits')
+    total_memory = 0
+
+if total_memory > 14000:
+    DOCKER_LIMITS.update({
+        'memory': '13g',
+        'time': 2400,
+    })
+elif total_memory > 8000:
+    DOCKER_LIMITS.update({
+        'memory': '7g',
+        'time': 1800,
+    })
+elif total_memory > 4000:
+    DOCKER_LIMITS.update({
+        'memory': '3g',
+        'time': 900,
+    })
+elif total_memory > 2000:
+    DOCKER_LIMITS.update({
+        'memory': '1g',
+        'time': 600,
+    })
+
+if hasattr(settings, 'DOCKER_LIMITS'):
+    DOCKER_LIMITS.update(settings.DOCKER_LIMITS)
+
 
 DOCKER_TIMEOUT_EXIT_CODE = 42
 DOCKER_OOM_EXIT_CODE = 137