Django Logging Setup: From Development to Production

Django's logging system is built on Python's powerful logging module, giving you fine-grained control over what gets logged and where it goes. This guide takes you from basic development logging to a production-ready centralized setup.

Understanding Django's Logging Architecture

Django logging has four key components:

• Loggers - Named channels for log messages
• Handlers - Destinations for log messages (files, console, remote)
• Filters - Rules for what messages get through
• Formatters - How messages are displayed

Basic Development Configuration

Start with a simple setup in your settings.py:

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },
    },
    'root': {
        'handlers': ['console'],
        'level': 'DEBUG',
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': False,
        },
    },
}

Structured Logging with JSON

For production, structured logs are essential for querying:

# Install python-json-logger
# pip install python-json-logger

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'json': {
            '()': 'pythonjsonlogger.jsonlogger.JsonFormatter',
            'format': '%(asctime)s %(levelname)s %(name)s %(message)s',
        },
        'verbose': {
            'format': '{levelname} {asctime} {module} {message}',
            'style': '{',
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'formatter': 'verbose',
        },
        'json_file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/django/app.json',
            'maxBytes': 10485760,  # 10MB
            'backupCount': 5,
            'formatter': 'json',
        },
    },
    'root': {
        'handlers': ['console', 'json_file'],
        'level': 'INFO',
    },
}

Adding Request Context

Include request information in your logs with middleware:

# middleware/logging.py
import logging
import uuid

logger = logging.getLogger(__name__)

class RequestLoggingMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        # Generate request ID
        request.id = str(uuid.uuid4())[:8]

        # Add context to all logs during this request
        with logging.LoggerAdapter(logger, {
            'request_id': request.id,
            'user_id': getattr(request.user, 'id', None),
            'path': request.path,
            'method': request.method,
        }):
            response = self.get_response(request)

        return response

Using Loggers in Your Code

import logging

# Get a logger for this module
logger = logging.getLogger(__name__)

class OrderService:
    def create_order(self, user, items):
        logger.info('Creating order', extra={
            'user_id': user.id,
            'item_count': len(items),
            'total': sum(item.price for item in items),
        })

        try:
            order = Order.objects.create(user=user)
            # ... create order items
            logger.info('Order created successfully', extra={
                'order_id': order.id,
            })
            return order
        except Exception as e:
            logger.exception('Failed to create order', extra={
                'user_id': user.id,
                'error': str(e),
            })
            raise

Handling Exceptions

Configure Django to log all uncaught exceptions:

LOGGING = {
    # ... other config ...
    'loggers': {
        'django.request': {
            'handlers': ['json_file', 'mail_admins'],
            'level': 'ERROR',
            'propagate': False,
        },
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
            'include_html': True,
        },
    },
}

Sending Logs to a Central Service

For production, send logs to a centralized platform:

Using Syslog

LOGGING = {
    'handlers': {
        'syslog': {
            'class': 'logging.handlers.SysLogHandler',
            'address': ('logs.example.com', 514),
            'facility': 'local0',
            'formatter': 'json',
        },
    },
}

Using HTTP (for services like 401 Clicks)

# Custom handler for HTTP log shipping
import requests
import json
from logging.handlers import QueueHandler, QueueListener
from queue import Queue

class HTTPHandler(logging.Handler):
    def __init__(self, url, api_key):
        super().__init__()
        self.url = url
        self.api_key = api_key
        self.session = requests.Session()

    def emit(self, record):
        try:
            payload = {
                'timestamp': record.created,
                'level': record.levelname,
                'logger': record.name,
                'message': record.getMessage(),
                **getattr(record, 'extra', {}),
            }
            self.session.post(
                self.url,
                json=payload,
                headers={'Authorization': f'Bearer {self.api_key}'},
                timeout=5,
            )
        except Exception:
            self.handleError(record)

Environment-Specific Configuration

# settings/base.py
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {...},
    'handlers': {
        'console': {...},
    },
    'root': {
        'handlers': ['console'],
        'level': 'INFO',
    },
}

# settings/development.py
LOGGING['root']['level'] = 'DEBUG'
LOGGING['loggers'] = {
    'django.db.backends': {
        'handlers': ['console'],
        'level': 'DEBUG',  # See SQL queries
    },
}

# settings/production.py
LOGGING['handlers']['json_file'] = {...}
LOGGING['handlers']['syslog'] = {...}
LOGGING['root']['handlers'] = ['console', 'json_file', 'syslog']

Performance Considerations

Async Logging

import logging
from logging.handlers import QueueHandler, QueueListener
from queue import Queue

# Create a queue for async logging
log_queue = Queue()

# Background listener
listener = QueueListener(
    log_queue,
    logging.handlers.RotatingFileHandler('/var/log/django/app.log'),
)
listener.start()

# Use QueueHandler in your config
LOGGING['handlers']['async'] = {
    'class': 'logging.handlers.QueueHandler',
    'queue': log_queue,
}

Sampling High-Volume Logs

import random

class SamplingFilter(logging.Filter):
    def __init__(self, rate=0.1):
        super().__init__()
        self.rate = rate

    def filter(self, record):
        return random.random() < self.rate

Celery Task Logging

from celery.utils.log import get_task_logger

logger = get_task_logger(__name__)

@app.task
def process_order(order_id):
    logger.info('Processing order', extra={'order_id': order_id})
    # ... task logic

Testing Your Logging

from django.test import TestCase
import logging

class LoggingTest(TestCase):
    def test_order_creation_logs(self):
        with self.assertLogs('myapp.services', level='INFO') as logs:
            OrderService().create_order(user, items)

        self.assertTrue(
            any('Order created' in log for log in logs.output)
        )

Conclusion

Proper Django logging setup pays dividends when debugging production issues. Start with structured JSON logging, add request context, and ship logs to a centralized service like 401 Clicks for easy searching and alerting.

Remember: the logs you don't have are the ones you'll need most when things go wrong.