log_filters.py
1.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
import logging
import threading
import time
import traceback
WINDOW_SECONDS = 1
MAX_KEYS = 1024
class DuplicateErrorFilter(logging.Filter):
def __init__(self, window_seconds=WINDOW_SECONDS, max_keys=MAX_KEYS):
super().__init__()
self.window_seconds = window_seconds
self.max_keys = max_keys
self._lock = threading.Lock()
self._entries = {}
def filter(self, record):
if record.levelno < logging.ERROR:
return True
now = time.monotonic()
key = self.build_key(record)
with self._lock:
self.prune(now)
opened_until = self._entries.get(key, 0.)
if opened_until > now:
return False
self._entries[key] = now + self.window_seconds
return True
def build_key(self, record):
exc_type = ""
frame = ("", 0, "")
exc_info = record.exc_info
if isinstance(exc_info, tuple) and len(exc_info) == 3 and exc_info[0] is not None:
exc_type = exc_info[0].__name__
extracted = traceback.extract_tb(exc_info[2])
if extracted:
last = extracted[-1]
frame = (last.filename, last.lineno, last.name)
return (record.name, exc_type, frame, record.msg)
def prune(self, now):
expired = [
key
for key, opened_until in self._entries.items()
if opened_until <= now
]
for key in expired:
self._entries.pop(key, None)
if len(self._entries) <= self.max_keys:
return
oldest = sorted(
self._entries.items(),
key=lambda item: item[1],
)[:len(self._entries) - self.max_keys]
for key, _ in oldest:
self._entries.pop(key, None)