diff --git a/_delphi_utils_python/README.md b/_delphi_utils_python/README.md index a7f64f921..0ac6350bf 100644 --- a/_delphi_utils_python/README.md +++ b/_delphi_utils_python/README.md @@ -17,5 +17,32 @@ Submodules: - `validator`: Data sanity checks and anomaly detection. -Source code can be found here: -[https://github.com/cmu-delphi/covidcast-indicators/](https://github.com/cmu-delphi/covidcast-indicators/) \ No newline at end of file +Source code can be found here: +[https://github.com/cmu-delphi/covidcast-indicators/](https://github.com/cmu-delphi/covidcast-indicators/) + +## Logger Usage + +Single-thread usage. + +```py +from delphi_utils.logger import get_structured_logger + +logger = get_structured_logger('my_logger') +logger.info('Hello, world!') +``` + +Multi-thread usage. + +```py +from delphi_utils.logger import get_structured_logger, pool_and_threadedlogger + +def f(x, threaded_logger): + threaded_logger.info(f'x={x}') + return x*x + +logger = get_structured_logger('my_logger') +logger.info('Hello, world!') +with pool_and_threadedlogger(logger, n_cpu) as (pool, threaded_logger): + for i in range(10): + pool.apply_async(f, args=(i, threaded_logger)) +``` diff --git a/_delphi_utils_python/delphi_utils/logger.py b/_delphi_utils_python/delphi_utils/logger.py index d70ae4c8e..c0e4502a8 100644 --- a/_delphi_utils_python/delphi_utils/logger.py +++ b/_delphi_utils_python/delphi_utils/logger.py @@ -1,5 +1,7 @@ """Structured logger utility for creating JSON logs. +See the delphi_utils README.md for usage examples. + The Delphi group uses two ~identical versions of this file. Try to keep them in sync with edits, for sanity. https://github.com/cmu-delphi/covidcast-indicators/blob/main/_delphi_utils_python/delphi_utils/logger.py @@ -133,19 +135,17 @@ class LoggerThread(): """ A construct to use a logger from multiprocessing workers/jobs. - the bare structlog loggers are thread-safe but not multiprocessing-safe. - a `LoggerThread` will spawn a thread that listens to a mp.Queue - and logs messages from it with the provided logger, - so other processes can send logging messages to it - via the logger-like `SubLogger` interface. - the SubLogger even logs the pid of the caller. + The bare structlog loggers are thread-safe but not multiprocessing-safe. A + `LoggerThread` will spawn a thread that listens to a mp.Queue and logs + messages from it with the provided logger, so other processes can send + logging messages to it via the logger-like `SubLogger` interface. The + SubLogger even logs the pid of the caller. - this is good to use with a set of jobs that are part of a mp.Pool, - but isnt recommended for general use - because of overhead from threading and multiprocessing, - and because it might introduce lag to log messages. + This is good to use with a set of jobs that are part of a mp.Pool, but isnt + recommended for general use because of overhead from threading and + multiprocessing, and because it might introduce lag to log messages. - somewhat inspired by: + Somewhat inspired by: docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes """ @@ -236,13 +236,11 @@ def pool_and_threadedlogger(logger, *poolargs): """ Provide (to a context) a multiprocessing Pool and a proxy to the supplied logger. - Emulates the multiprocessing.Pool() context manager, - but also provides (via a LoggerThread) a SubLogger proxy to logger - that can be safely used by pool workers. - The SubLogger proxy interface supports these methods: debug, info, warning, error, - and critical. - Also "cleans up" the pool by waiting for workers to complete - as it exits the context. + Emulates the multiprocessing.Pool() context manager, but also provides (via + a LoggerThread) a SubLogger proxy to logger that can be safely used by pool + workers. The SubLogger proxy interface supports these methods: debug, info, + warning, error, and critical. Also "cleans up" the pool by waiting for + workers to complete as it exits the context. """ with multiprocessing.Manager() as manager: logger_thread = LoggerThread(logger, manager.Queue())