How to use Python logging QueueHandler with dictConfig

Decoupling

import queue
import logging
from logging.handlers import QueueHandler, QueueListener
log_queue = queue.Queue(-1)
queue_handler = QueueHandler(log_queue)

logger = logging.getLogger()
logger.addHandler(queue_handler)

console_handler = logging.StreamHandler()
formatter = logging.Formatter('%(threadName)s: %(message)s')
console_handler.setFormatter(formatter)

file_handler = logging.FileHandler("queue_example.log")
file_handler.setFormatter(formatter)

listener = QueueListener(log_queue, console_handler, file_handler)
listener.start()

logger.warning('Look out!')

listener.stop()

Configuration

version: 1
formatters:
simple:
format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
handlers:
console:
class: logging.StreamHandler
level: DEBUG
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
filename: 'my_script2.log'
formatter: simple
loggers:
examples.my_script1:
level: DEBUG
handlers: [console]
propagate: false
examples.my_script2:
level: WARNING
handlers: [console, file]
propagate: false
root:
level: INFO
handlers: [console]
# examples/my_script1.pyimport logging

logger = logging.getLogger(__name__)


def do_stuff1():
logger.debug('This is a debug message')
logger.info('This is an info message')
logger.warning('This is a warning message')
logger.error('This is an error message')
logger.critical('This is a critical message')

try:
x = 1 / 0
except:
logger.exception('This is an exception')
# examples/myscript2.pyimport logging

logger = logging.getLogger(__name__)


def do_stuff2():
logger.debug('This is a debug message')
logger.info('This is an info message')
logger.warning('This is a warning message')
logger.error('This is an error message')
logger.critical('This is a critical message')

try:
x = 1 / 0
except:
logger.exception('This is an exception')
import logging
import logging.config
import yaml
from examples.my_script1 import do_stuff1
from examples.my_script2 import do_stuff2

LOGGING_CONFIG = """
version: 1
formatters:
simple:
format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
handlers:
console:
class: logging.StreamHandler
level: DEBUG
formatter: simple
stream: ext://sys.stdout
loggers:
examples.my_script1:
level: DEBUG
handlers: [console]
propagate: false
examples.my_script2:
level: WARNING
handlers: [console]
propagate: false
root:
level: INFO
handlers: [console]
"""

logging_config = yaml.load(LOGGING_CONFIG)

logging.config.dictConfig(logging_config)

other_logger = logging.getLogger("foo")

do_stuff1()
do_stuff2()

other_logger.debug("A different debug message")
other_logger.info("A different info message")
other_logger.error("A different error message")
/home/rblackbourn/log-client/examples/config_example1.py
2019-01-02 09:58:21,908 - examples.my_script1 - DEBUG - This is a debug message
2019-01-02 09:58:21,908 - examples.my_script1 - INFO - This is an info message
2019-01-02 09:58:21,908 - examples.my_script1 - WARNING - This is a warning message
2019-01-02 09:58:21,908 - examples.my_script1 - ERROR - This is an error message
2019-01-02 09:58:21,908 - examples.my_script1 - CRITICAL - This is a critical message
2019-01-02 09:58:21,908 - examples.my_script1 - ERROR - This is an exception
Traceback (most recent call last):
File "/home/rblackbourn/log-client/examples/my_script1.py", line 14, in do_stuff1
x = 1 / 0
ZeroDivisionError: division by zero
2019-01-02 09:58:21,909 - examples.my_script2 - WARNING - This is a warning message
2019-01-02 09:58:21,909 - examples.my_script2 - ERROR - This is an error message
2019-01-02 09:58:21,909 - examples.my_script2 - CRITICAL - This is a critical message
2019-01-02 09:58:21,909 - examples.my_script2 - ERROR - This is an exception
Traceback (most recent call last):
File "/home/rblackbourn/log-client/examples/my_script2.py", line 14, in do_stuff2
x = 1 / 0
ZeroDivisionError: division by zero
2019-01-02 09:58:21,909 - foo - INFO - A different info message
2019-01-02 09:58:21,909 - foo - ERROR - A different error message
Process finished with exit code 0
# Invalid!!!version: 1
formatters:
simple:
format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
handlers:
console:
class: logging.StreamHandler
level: DEBUG
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
filename: 'config_example2.log'
formatter: simple
queue:
class: logging.handlers.QueueHandler
handlers: [console, file]
loggers:
examples.my_script1:
level: DEBUG
handlers: [queue]
propagate: false
examples.my_script2:
level: WARNING
handlers: [queue]
propagate: false
root:
level: WARN
handlers: [console]
version: 1
formatters:
simple:
format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
handlers:
console:
class: logging.StreamHandler
level: DEBUG
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
filename: 'config_example2.log'
formatter: simple
queue_listener:
class: examples.QueueListenerHandler
handlers:
- cfg://handlers.console
- cfg://handlers.file
loggers:
examples.my_script1:
level: DEBUG
handlers:
- queue_listener
propagate: false
examples.my_script2:
level: WARNING
handlers:
- queue_listener
propagate: false
root:
level: WARN
handlers:
- console
# !!! INVALID !!!
# examples/QueueListenerHandler.py
from logging.handlers import QueueHandler, QueueListener
from queue import Queue
from atexit import register


class QueueListenerHandler(QueueHandler):

def __init__(self, handlers, respect_handler_level=False, auto_run=True, queue=Queue(-1)):
super().__init__(queue)
self._listener = QueueListener(
self.queue,
*handlers,
respect_handler_level=respect_handler_level)
if auto_run:
self.start()
register(self.stop)


def start(self):
self._listener.start()


def stop(self):
self._listener.stop()


def emit(self, record):
return super().emit(record)
# examples/QueueListenerHandler.pyfrom logging.config import ConvertingList, ConvertingDict, valid_ident
from logging.handlers import QueueHandler, QueueListener
from queue import Queue
from atexit import register


def _resolve_handlers(l):
if not isinstance(l, ConvertingList):
return l

# Indexing the list performs the evaluation.
return [l[i] for i in range(len(l))]


class QueueListenerHandler(QueueHandler):

def __init__(self, handlers, respect_handler_level=False, auto_run=True, queue=Queue(-1)):
super().__init__(queue)
handlers = _resolve_handlers(handlers)
self._listener = QueueListener(
self.queue,
*handlers,
respect_handler_level=respect_handler_level)
if auto_run:
self.start()
register(self.stop)


def start(self):
self._listener.start()


def stop(self):
self._listener.stop()


def emit(self, record):
return super().emit(record)

Undocumented Configuration Possibilities

version: 1
objects:
queue:
class: queue.Queue
maxsize: 1000
formatters:
simple:
format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
handlers:
console:
class: logging.StreamHandler
level: DEBUG
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
filename: 'config_example2.log'
formatter: simple
queue_listener:
class: examples.QueueListenerHandler
handlers:
- cfg://handlers.console
- cfg://handlers.file
queue: cfg://objects.queue
loggers:
examples.my_script1:
level: DEBUG
handlers:
- queue_listener
propagate: false
examples.my_script2:
level: WARNING
handlers:
- queue_listener
propagate: false
root:
level: WARN
handlers:
- console
from logging.config import ConvertingList, ConvertingDict, valid_ident
from logging.handlers import QueueHandler, QueueListener
from queue import Queue
import atexit


def _resolve_handlers(l):
if not isinstance(l, ConvertingList):
return l

# Indexing the list performs the evaluation.
return [l[i] for i in range(len(l))]


def _resolve_queue(q):
if not isinstance(q, ConvertingDict):
return q
if '__resolved_value__' in q:
return q['__resolved_value__']

cname = q.pop('class')
klass = q.configurator.resolve(cname)
props = q.pop('.', None)
kwargs = {k: q[k] for k in q if valid_ident(k)}
result = klass(**kwargs)
if props:
for name, value in props.items():
setattr(result, name, value)

q['__resolved_value__'] = result
return result


class QueueListenerHandler(QueueHandler):

def __init__(self, handlers, respect_handler_level=False, auto_run=True, queue=Queue(-1)):
queue = _resolve_queue(queue)
super().__init__(queue)
handlers = _resolve_handlers(handlers)
self._listener = QueueListener(
self.queue,
*handlers,
respect_handler_level=respect_handler_level)
if auto_run:
self.start()
atexit.register(self.stop)


def start(self):
self._listener.start()


def stop(self):
self._listener.stop()


def emit(self, record):
return super().emit(record)

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store