log_filters.py 1.79 KB
import logging
import threading
import time
import traceback


WINDOW_SECONDS = 1
MAX_KEYS = 1024


class DuplicateErrorFilter(logging.Filter):
    def __init__(self, window_seconds=WINDOW_SECONDS, max_keys=MAX_KEYS):
        super().__init__()
        self.window_seconds = window_seconds
        self.max_keys = max_keys
        self._lock = threading.Lock()
        self._entries = {}

    def filter(self, record):
        if record.levelno < logging.ERROR:
            return True

        now = time.monotonic()
        key = self.build_key(record)

        with self._lock:
            self.prune(now)

            opened_until = self._entries.get(key, 0.)
            if opened_until > now:
                return False

            self._entries[key] = now + self.window_seconds

        return True

    def build_key(self, record):
        exc_type = ""
        frame = ("", 0, "")
        exc_info = record.exc_info

        if isinstance(exc_info, tuple) and len(exc_info) == 3 and exc_info[0] is not None:
            exc_type = exc_info[0].__name__
            extracted = traceback.extract_tb(exc_info[2])
            if extracted:
                last = extracted[-1]
                frame = (last.filename, last.lineno, last.name)

        return (record.name, exc_type, frame, record.msg)

    def prune(self, now):
        expired = [
            key
            for key, opened_until in self._entries.items()
            if opened_until <= now
        ]
        for key in expired:
            self._entries.pop(key, None)

        if len(self._entries) <= self.max_keys:
            return

        oldest = sorted(
            self._entries.items(),
            key=lambda item: item[1],
        )[:len(self._entries) - self.max_keys]
        for key, _ in oldest:
            self._entries.pop(key, None)