Make logs from logging.getLogger to look better

Logs coming from `logging.getLogger` (from other applications) have the same
structure that the ones logged by us via `structlog`.

Author: humitos

common

@@ -69,20 +69,21 @@ def __call__(self, logger, method_name, event_dict):
         event_dict.update(output)
         return event_dict
 
+shared_processors = [
+    structlog.processors.TimeStamper(fmt='iso'),
+    structlog.stdlib.add_logger_name,
+    structlog.stdlib.add_log_level,
+    structlog.stdlib.PositionalArgumentsFormatter(),
+    structlog.processors.StackInfoRenderer(),
+    structlog.processors.UnicodeDecoder(),
+]
 
 structlog.configure(
-    processors=[
+    processors=list([
         structlog.stdlib.filter_by_level,
-        structlog.processors.TimeStamper(fmt='iso'),
-        structlog.stdlib.add_logger_name,
-        structlog.stdlib.add_log_level,
-        structlog.stdlib.PositionalArgumentsFormatter(),
-        structlog.processors.StackInfoRenderer(),
-        # Remove `format_exc_info` from your processor chain if you want pretty exceptions.
-        # structlog.processors.format_exc_info,
-        structlog.processors.UnicodeDecoder(),
+        *shared_processors,
         structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
-    ],
+    ]),
     context_class=structlog.threadlocal.wrap_dict(dict),
     logger_factory=structlog.stdlib.LoggerFactory(),
     wrapper_class=structlog.stdlib.BoundLogger,

readthedocs/core/logs.py

@@ -8,6 +8,7 @@
 
 from celery.schedules import crontab
 
+from readthedocs.core.logs import shared_processors
 from readthedocs.core.settings import Settings
 from readthedocs.projects.constants import CELERY_LOW, CELERY_MEDIUM, CELERY_HIGH
 
@@ -803,11 +804,23 @@ def DOCKER_LIMITS(self):
             # structlog
             "plain_console": {
                 "()": structlog.stdlib.ProcessorFormatter,
-                "processor": structlog.dev.ConsoleRenderer(),
+                "processors": [
+                    structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+                    structlog.dev.ConsoleRenderer(),
+                ],
+                # Allows to add extra data to log entries generated via ``logging`` module
+                # See https://www.structlog.org/en/stable/standard-library.html#rendering-using-structlog-based-formatters-within-logging
+                "foreign_pre_chain": shared_processors,
             },
             "key_value": {
                 "()": structlog.stdlib.ProcessorFormatter,
-                "processor": structlog.processors.KeyValueRenderer(key_order=['timestamp', 'level', 'event', 'logger']),
+                "processors": [
+                    structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+                    structlog.processors.KeyValueRenderer(key_order=['timestamp', 'level', 'event', 'logger']),
+                ],
+                # Allows to add extra data to log entries generated via ``logging`` module
+                # See https://www.structlog.org/en/stable/standard-library.html#rendering-using-structlog-based-formatters-within-logging
+                "foreign_pre_chain": shared_processors,
             },
         },
         'handlers': {

readthedocs/settings/base.py

@@ -78,6 +78,12 @@ def RTD_EMBED_API_EXTERNAL_DOMAINS(self):
     def LOGGING(self):
         logging = super().LOGGING
         logging['loggers'].update({
+            # Disable Django access requests logging (e.g. GET /path/to/url)
+            # https://github.com/django/django/blob/ca9872905559026af82000e46cde6f7dedc897b6/django/core/servers/basehttp.py#L24
+            'django.server': {
+                'handlers': ['null'],
+                'propagate': False,
+            },
             # Disable S3 logging
             'boto3': {
                 'handlers': ['null'],