Gabriel Weisse / presupuestov1

Go to a project
Toggle navigation Toggle navigation pinning
pr0nstar
72380414 1 parent 783e6fac

XIII

Inline Side-by-side
Showing 27 changed files with 1024 additions and 105 deletions
...@@ -6,7 +6,7 @@ from starlette_caches.utils import cache as scache ...@@ -6,7 +6,7 @@ from starlette_caches.utils import cache as scache
6 from aiocache import Cache 6 from aiocache import Cache
7 7
8 8
9 -REDIS_CONN = open('./docs/postgres').read().strip() 9 +REDIS_CONN = open('./docs/redis').read().strip()
10 REDIS_CONN = REDIS_CONN.split(':') 10 REDIS_CONN = REDIS_CONN.split(':')
11 # cache = Cache( 11 # cache = Cache(
12 # Cache.REDIS, 12 # Cache.REDIS,
......
1 +import logging
2 +import threading
3 +import time
4 +import traceback
5 +
6 +
7 +WINDOW_SECONDS = 1
8 +MAX_KEYS = 1024
9 +
10 +
11 +class DuplicateErrorFilter(logging.Filter):
12 + def __init__(self, window_seconds=WINDOW_SECONDS, max_keys=MAX_KEYS):
13 + super().__init__()
14 + self.window_seconds = window_seconds
15 + self.max_keys = max_keys
16 + self._lock = threading.Lock()
17 + self._entries = {}
18 +
19 + def filter(self, record):
20 + if record.levelno < logging.ERROR:
21 + return True
22 +
23 + now = time.monotonic()
24 + key = self.build_key(record)
25 +
26 + with self._lock:
27 + self.prune(now)
28 +
29 + opened_until = self._entries.get(key, 0.)
30 + if opened_until > now:
31 + return False
32 +
33 + self._entries[key] = now + self.window_seconds
34 +
35 + return True
36 +
37 + def build_key(self, record):
38 + exc_type = ""
39 + frame = ("", 0, "")
40 + exc_info = record.exc_info
41 +
42 + if isinstance(exc_info, tuple) and len(exc_info) == 3 and exc_info[0] is not None:
43 + exc_type = exc_info[0].__name__
44 + extracted = traceback.extract_tb(exc_info[2])
45 + if extracted:
46 + last = extracted[-1]
47 + frame = (last.filename, last.lineno, last.name)
48 +
49 + return (record.name, exc_type, frame, record.msg)
50 +
51 + def prune(self, now):
52 + expired = [
53 + key
54 + for key, opened_until in self._entries.items()
55 + if opened_until <= now
56 + ]
57 + for key in expired:
58 + self._entries.pop(key, None)
59 +
60 + if len(self._entries) <= self.max_keys:
61 + return
62 +
63 + oldest = sorted(
64 + self._entries.items(),
65 + key=lambda item: item[1],
66 + )[:len(self._entries) - self.max_keys]
67 + for key, _ in oldest:
68 + self._entries.pop(key, None)
...@@ -15,7 +15,7 @@ from starlette.middleware import Middleware ...@@ -15,7 +15,7 @@ from starlette.middleware import Middleware
15 from starlette_caches.middleware import CacheMiddleware 15 from starlette_caches.middleware import CacheMiddleware
16 from starlette_caches.rules import Rule 16 from starlette_caches.rules import Rule
17 17
18 -from presupuestov1 import cache 18 +from presupuestov1 import cache, metrics
19 from presupuestov1.routers import ( 19 from presupuestov1.routers import (
20 project, 20 project,
21 entity, 21 entity,
...@@ -23,17 +23,10 @@ from presupuestov1.routers import ( ...@@ -23,17 +23,10 @@ from presupuestov1.routers import (
23 classifier_income, 23 classifier_income,
24 daily, 24 daily,
25 search, 25 search,
26 + private_metrics,
26 ) 27 )
27 28
28 29
29 -logging.basicConfig(
30 - level=logging.INFO,
31 - format='%(asctime)s %(message)s',
32 - filename='log.log',
33 -)
34 -logging.getLogger('api_proy2').setLevel(logging.DEBUG)
35 -
36 -
37 app = FastAPI( 30 app = FastAPI(
38 title='MEFP presupuestosv1 API', 31 title='MEFP presupuestosv1 API',
39 middleware=[ 32 middleware=[
...@@ -47,13 +40,16 @@ app = FastAPI( ...@@ -47,13 +40,16 @@ app = FastAPI(
47 CacheMiddleware, 40 CacheMiddleware,
48 cache=cache.cache, 41 cache=cache.cache,
49 rules=[ 42 rules=[
43 + Rule(match=re.compile(r"^/api/_metrics/.*"), ttl=0),
50 Rule(match=re.compile(r"^/api/search/.*"), ttl=0), 44 Rule(match=re.compile(r"^/api/search/.*"), ttl=0),
45 + Rule(match=re.compile(r"^/api/programa_proyecto/.*"), ttl=0),
51 Rule(match=re.compile(r'^/api/.+'), ttl=86400, status=200), 46 Rule(match=re.compile(r'^/api/.+'), ttl=86400, status=200),
52 ], 47 ],
53 ), 48 ),
54 ], 49 ],
55 root_path='/api' 50 root_path='/api'
56 ) 51 )
52 +
57 app.include_router(entity.router_entidad) 53 app.include_router(entity.router_entidad)
58 app.include_router(entity.router_ubigeo) 54 app.include_router(entity.router_ubigeo)
59 app.include_router(entity.router_da) 55 app.include_router(entity.router_da)
...@@ -65,6 +61,9 @@ app.include_router(classifier_income.router_organismo) ...@@ -65,6 +61,9 @@ app.include_router(classifier_income.router_organismo)
65 app.include_router(project.router) 61 app.include_router(project.router)
66 app.include_router(daily.router) 62 app.include_router(daily.router)
67 app.include_router(search.router) 63 app.include_router(search.router)
64 +app.include_router(private_metrics.router)
65 +
66 +# metrics.install(app)
68 67
69 68
70 # @app.on_event('startup') 69 # @app.on_event('startup')
......
1 +import logging
2 +import time
3 +
4 +from datetime import datetime, timedelta, timezone
5 +
6 +import redis.asyncio as redis
7 +
8 +from starlette.routing import Match
9 +
10 +
11 +PREFIX = "metrics:v1"
12 +TTL_SECONDS = 7 * 24 * 60 * 60
13 +STREAM_MAXLEN = 10000
14 +HTTP_BUCKETS_MS = (100, 300, 600, 1000, 2000, 3000)
15 +PRIVATE_API_PREFIXES = ("/_metrics", "/api/_metrics")
16 +
17 +redis_client = None
18 +logger = logging.getLogger("api_proy2")
19 +
20 +
21 +###############################################################################
22 +# middleware
23 +###############################################################################
24 +
25 +class MetricsMiddleware:
26 + def __init__(self, app, routes):
27 + self.app = app
28 + self.routes = routes
29 +
30 + async def __call__(self, scope, receive, send):
31 + if scope["type"] != "http":
32 + await self.app(scope, receive, send)
33 + return
34 +
35 + path = scope.get("path", "")
36 + if path.startswith(PRIVATE_API_PREFIXES):
37 + await self.app(scope, receive, send)
38 + return
39 +
40 + started_at = time.perf_counter()
41 + status_code = 500
42 + response_headers = []
43 +
44 + async def send_wrapper(message):
45 + nonlocal status_code, response_headers
46 +
47 + if message["type"] == "http.response.start":
48 + status_code = message["status"]
49 + response_headers = message.get("headers", [])
50 +
51 + await send(message)
52 +
53 + try:
54 + await self.app(scope, receive, send_wrapper)
55 + finally:
56 + try:
57 + await record_http(
58 + method=scope.get("method", "GET"),
59 + route=find_route_template(self.routes, scope),
60 + path=scope.get("path", ""),
61 + client_ip=get_client_ip(scope.get("headers", [])),
62 + status_code=status_code,
63 + duration_ms=(time.perf_counter() - started_at) * 1000,
64 + cache_status=get_header_value(response_headers, b"x-cache"),
65 + )
66 + except Exception:
67 + logger.exception("metrics write failed")
68 +
69 +
70 +def install(app):
71 + app.add_middleware(MetricsMiddleware, routes=app.router.routes)
72 +
73 +
74 +###############################################################################
75 +# redis lifecycle
76 +###############################################################################
77 +
78 +async def get_redis():
79 + global redis_client
80 +
81 + if redis_client is not None:
82 + return redis_client
83 +
84 + with open("./docs/redis") as f:
85 + redis_url = f.read().strip()
86 +
87 + if "://" not in redis_url:
88 + redis_url = f"redis://{redis_url}"
89 +
90 + redis_client = redis.from_url(redis_url, decode_responses=True)
91 + return redis_client
92 +
93 +
94 +async def close_redis():
95 + global redis_client
96 +
97 + if redis_client is None:
98 + return
99 +
100 + await redis_client.aclose()
101 + redis_client = None
102 +
103 +
104 +###############################################################################
105 +# write api
106 +###############################################################################
107 +
108 +async def record_http(
109 + method,
110 + route,
111 + path,
112 + client_ip,
113 + status_code,
114 + duration_ms,
115 + cache_status=None
116 +):
117 + redis_conn = await get_redis()
118 + minute = get_current_minute()
119 + http_minute_key = get_http_minute_key(minute)
120 + ip_minute_key = get_ip_minute_key(minute)
121 + route_label = join_label(method, route, int(status_code))
122 + route_key = join_label(method, route)
123 + pipe = redis_conn.pipeline(transaction=False)
124 +
125 + pipe.hincrby(http_minute_key, f"count|{route_label}", 1)
126 + pipe.hincrbyfloat(http_minute_key, f"sum_ms|{route_label}", float(duration_ms))
127 + pipe.hincrby(http_minute_key, f"status|{int(status_code)}", 1)
128 +
129 + if cache_status in {"hit", "miss"}:
130 + pipe.hincrby(http_minute_key, f"cache_{cache_status}|{route_key}", 1)
131 +
132 + for bucket_ms in HTTP_BUCKETS_MS:
133 + if duration_ms <= bucket_ms:
134 + pipe.hincrby(http_minute_key, f"le_{bucket_ms}|{route_label}", 1)
135 +
136 + pipe.hincrby(ip_minute_key, normalize_label_part(client_ip), 1)
137 +
138 + pipe.expire(http_minute_key, TTL_SECONDS)
139 + pipe.expire(ip_minute_key, TTL_SECONDS)
140 +
141 + # pipe.xadd(
142 + # get_requests_stream_key(),
143 + # {
144 + # "ts": get_now_utc().isoformat(),
145 + # "method": str(method),
146 + # "route": str(route),
147 + # "path": str(path),
148 + # "client_ip": str(client_ip or ""),
149 + # "status_code": str(int(status_code)),
150 + # "duration_ms": str(round(float(duration_ms), 3)),
151 + # "cache_status": str(cache_status or ""),
152 + # },
153 + # maxlen=STREAM_MAXLEN,
154 + # approximate=True,
155 + # )
156 + await pipe.execute()
157 +
158 +
159 +###############################################################################
160 +# read api
161 +###############################################################################
162 +
163 +async def read_summary(minutes=60, include_ts=False):
164 + if include_ts:
165 + return await read_summary_series(minutes)
166 +
167 + return await read_summary_window(minutes)
168 +
169 +
170 +async def read_routes(minutes=60, include_ts=False):
171 + if include_ts:
172 + return await read_route_series(minutes)
173 +
174 + return await read_route_window(minutes)
175 +
176 +
177 +async def read_ips(minutes=60, include_ts=False):
178 + if include_ts:
179 + return await read_ip_series(minutes)
180 +
181 + return await read_ip_window(minutes)
182 +
183 +
184 +# async def read_requests(limit=100):
185 +# redis_conn = await get_redis()
186 +# rows = await redis_conn.xrevrange(get_requests_stream_key(), count=limit)
187 +# return [{"id": row_id, "fields": payload} for row_id, payload in rows]
188 +
189 +
190 +async def read_http(minutes):
191 + return merge_minute_maps(await load_http_minutes(minutes))
192 +
193 +
194 +###############################################################################
195 +# summary readers
196 +###############################################################################
197 +
198 +async def read_summary_window(minutes):
199 + http_minutes = await load_http_minutes(minutes)
200 + http_fields = merge_minute_maps(http_minutes)
201 + total_count = get_total_count(http_fields)
202 + total_sum_ms = get_total_sum_ms(http_fields)
203 +
204 + return {
205 + "window_minutes": minutes,
206 + "generated_at": get_now_utc().isoformat(),
207 + "http": {
208 + "count": total_count,
209 + "sum_ms": total_sum_ms,
210 + "avg_ms": round(total_sum_ms / total_count, 3) if total_count else 0.0,
211 + "status": get_status_totals(http_fields),
212 + "cache": get_cache_totals(http_fields),
213 + "buckets_ms": get_bucket_totals(http_fields),
214 + },
215 + }
216 +
217 +
218 +async def read_summary_series(minutes):
219 + http_minutes = await load_http_minutes(minutes)
220 + http_fields = merge_minute_maps(http_minutes)
221 + total_count = get_total_count(http_fields)
222 + total_sum_ms = get_total_sum_ms(http_fields)
223 + series = []
224 +
225 + for minute, minute_fields in http_minutes:
226 + minute_count = get_total_count(minute_fields)
227 + minute_sum_ms = get_total_sum_ms(minute_fields)
228 + series.append({
229 + "minute": minute,
230 + "http": {
231 + "count": minute_count,
232 + "sum_ms": minute_sum_ms,
233 + "avg_ms": round(minute_sum_ms / minute_count, 3) if minute_count else 0.0,
234 + "status": get_status_totals(minute_fields),
235 + "cache": get_cache_totals(minute_fields),
236 + "buckets_ms": get_bucket_totals(minute_fields),
237 + },
238 + })
239 +
240 + return {
241 + "window_minutes": minutes,
242 + "generated_at": get_now_utc().isoformat(),
243 + "http": {
244 + "count": total_count,
245 + "sum_ms": total_sum_ms,
246 + "avg_ms": round(total_sum_ms / total_count, 3) if total_count else 0.0,
247 + "status": get_status_totals(http_fields),
248 + "cache": get_cache_totals(http_fields),
249 + "buckets_ms": get_bucket_totals(http_fields),
250 + },
251 + "series": series,
252 + }
253 +
254 +
255 +###############################################################################
256 +# route readers
257 +###############################################################################
258 +
259 +async def read_route_window(minutes):
260 + return build_route_rows(await read_http(minutes))
261 +
262 +
263 +async def read_route_series(minutes):
264 + rows = []
265 +
266 + for minute, minute_fields in await load_http_minutes(minutes):
267 + rows.extend(build_route_rows(minute_fields, minute=minute))
268 +
269 + rows.sort(key=lambda row: (
270 + row["minute"], -row["count"], -row["sum_ms"], row["route"]
271 + ))
272 + return rows
273 +
274 +
275 +###############################################################################
276 +# ip readers
277 +###############################################################################
278 +
279 +async def read_ip_window(minutes):
280 + ip_fields = merge_minute_maps(await load_ip_minutes(minutes))
281 + rows = []
282 +
283 + for ip, count in ip_fields.items():
284 + rows.append({
285 + "ip": ip,
286 + "count": int(count),
287 + })
288 +
289 + rows.sort(key=lambda row: (-row["count"], row["ip"]))
290 + return rows
291 +
292 +
293 +async def read_ip_series(minutes):
294 + rows = []
295 +
296 + for minute, minute_fields in await load_ip_minutes(minutes):
297 + for ip, count in minute_fields.items():
298 + rows.append({
299 + "minute": minute,
300 + "ip": ip,
301 + "count": int(count),
302 + })
303 +
304 + rows.sort(key=lambda row: (row["minute"], -row["count"], row["ip"]))
305 + return rows
306 +
307 +
308 +###############################################################################
309 +# minute loaders
310 +###############################################################################
311 +
312 +async def load_http_minutes(minutes):
313 + minute_keys = get_minute_buckets(minutes)
314 + if not minute_keys:
315 + return []
316 +
317 + redis_conn = await get_redis()
318 + pipe = redis_conn.pipeline(transaction=False)
319 +
320 + for minute in minute_keys:
321 + pipe.hgetall(get_http_minute_key(minute))
322 +
323 + rows = await pipe.execute()
324 + return build_minute_maps(minute_keys, rows)
325 +
326 +
327 +async def load_ip_minutes(minutes):
328 + minute_keys = get_minute_buckets(minutes)
329 + if not minute_keys:
330 + return []
331 +
332 + redis_conn = await get_redis()
333 + pipe = redis_conn.pipeline(transaction=False)
334 +
335 + for minute in minute_keys:
336 + pipe.hgetall(get_ip_minute_key(minute))
337 +
338 + rows = await pipe.execute()
339 + return build_minute_maps(minute_keys, rows)
340 +
341 +
342 +###############################################################################
343 +# route matching
344 +###############################################################################
345 +
346 +def find_route_template(routes, scope):
347 + route_template = match_route_template(routes, scope)
348 + if route_template is not None:
349 + return route_template
350 +
351 + root_path = scope.get("root_path", "")
352 + path = scope.get("path", "")
353 +
354 + if root_path and path.startswith(root_path):
355 + stripped_scope = dict(scope)
356 + stripped_scope["path"] = path[len(root_path):] or "/"
357 + route_template = match_route_template(routes, stripped_scope)
358 + if route_template is not None:
359 + return route_template
360 +
361 + return path
362 +
363 +
364 +def match_route_template(routes, scope):
365 + for route in routes:
366 + match, _ = route.matches(scope)
367 + if match == Match.FULL:
368 + return getattr(
369 + route, "path_format", getattr(route, "path", scope.get("path", ""))
370 + )
371 +
372 + return None
373 +
374 +
375 +###############################################################################
376 +# request parsing
377 +###############################################################################
378 +
379 +def get_header_value(headers, key):
380 + for header_key, header_value in headers:
381 + if header_key.lower() == key:
382 + return header_value.decode()
383 +
384 + return None
385 +
386 +
387 +def get_client_ip(headers):
388 + forwarded_for = get_header_value(headers, b"x-forwarded-for")
389 + if forwarded_for:
390 + return forwarded_for.split(",", 1)[0].strip()
391 +
392 + real_ip = get_header_value(headers, b"x-real-ip")
393 + if real_ip:
394 + return real_ip.strip()
395 +
396 + return None
397 +
398 +
399 +###############################################################################
400 +# row builders
401 +###############################################################################
402 +
403 +def build_route_rows(fields, minute=None):
404 + routes = {}
405 +
406 + for field, value in fields.items():
407 + parts = field.split("|")
408 + if len(parts) < 4:
409 + continue
410 +
411 + metric_name = parts[0]
412 + method = parts[1]
413 + route = parts[2]
414 + status_code = parts[3]
415 +
416 + if metric_name not in {"count", "sum_ms"} and not metric_name.startswith("le_"):
417 + continue
418 +
419 + route_key = (method, route, status_code)
420 + if route_key not in routes:
421 + routes[route_key] = {
422 + "method": method,
423 + "route": route,
424 + "status_code": int(status_code),
425 + "count": 0,
426 + "sum_ms": 0.0,
427 + "avg_ms": 0.0,
428 + "buckets_ms": {},
429 + }
430 +
431 + if minute is not None:
432 + routes[route_key]["minute"] = minute
433 +
434 + if metric_name == "count":
435 + routes[route_key]["count"] += int(value)
436 + elif metric_name == "sum_ms":
437 + routes[route_key]["sum_ms"] += float(value)
438 + else:
439 + bucket_ms = metric_name[3:]
440 + routes[route_key]["buckets_ms"][bucket_ms] = (
441 + routes[route_key]["buckets_ms"].get(bucket_ms, 0) + int(value)
442 + )
443 +
444 + rows = []
445 + for row in routes.values():
446 + if row["count"]:
447 + row["avg_ms"] = round(row["sum_ms"] / row["count"], 3)
448 + row["sum_ms"] = round(row["sum_ms"], 3)
449 + rows.append(row)
450 +
451 + rows.sort(key=lambda row: (-row["count"], -row["sum_ms"], row["route"]))
452 + return rows
453 +
454 +
455 +def build_minute_maps(minute_keys, rows):
456 + minute_maps = []
457 +
458 + for minute, row in zip(minute_keys, rows):
459 + minute_fields = {}
460 + for field, raw_value in row.items():
461 + minute_fields[field] = float(raw_value)
462 + minute_maps.append((minute, minute_fields))
463 +
464 + return minute_maps
465 +
466 +
467 +###############################################################################
468 +# aggregation helpers
469 +###############################################################################
470 +
471 +def merge_minute_maps(minute_maps):
472 + merged = {}
473 +
474 + for _, minute_fields in minute_maps:
475 + for field, value in minute_fields.items():
476 + merged[field] = merged.get(field, 0.0) + float(value)
477 +
478 + return merged
479 +
480 +
481 +def get_total_count(fields):
482 + count = 0
483 +
484 + for field, value in fields.items():
485 + if field.startswith("count|"):
486 + count += int(value)
487 +
488 + return count
489 +
490 +
491 +def get_total_sum_ms(fields):
492 + total = 0.0
493 +
494 + for field, value in fields.items():
495 + if field.startswith("sum_ms|"):
496 + total += float(value)
497 +
498 + return round(total, 3)
499 +
500 +
501 +def get_status_totals(fields):
502 + totals = {}
503 +
504 + for field, value in fields.items():
505 + if field.startswith("status|"):
506 + status_code = field.split("|", 1)[1]
507 + totals[status_code] = totals.get(status_code, 0) + int(value)
508 +
509 + return totals
510 +
511 +
512 +def get_cache_totals(fields):
513 + totals = {"hit": 0, "miss": 0}
514 +
515 + for field, value in fields.items():
516 + if field.startswith("cache_hit|"):
517 + totals["hit"] += int(value)
518 + elif field.startswith("cache_miss|"):
519 + totals["miss"] += int(value)
520 +
521 + return totals
522 +
523 +
524 +def get_bucket_totals(fields):
525 + totals = {}
526 +
527 + for field, value in fields.items():
528 + if field.startswith("le_"):
529 + bucket_ms = field.split("|", 1)[0][3:]
530 + totals[bucket_ms] = totals.get(bucket_ms, 0) + int(value)
531 +
532 + return totals
533 +
534 +
535 +###############################################################################
536 +# time and key helpers
537 +###############################################################################
538 +
539 +def get_now_utc():
540 + return datetime.now(timezone.utc)
541 +
542 +
543 +def get_current_minute(now=None):
544 + now = now or get_now_utc()
545 + return now.strftime("%Y%m%d%H%M")
546 +
547 +
548 +def get_minute_buckets(minutes, now=None):
549 + if minutes <= 0:
550 + return []
551 +
552 + now = now or get_now_utc()
553 + base = now.replace(second=0, microsecond=0)
554 + return [
555 + (base - timedelta(minutes=offset)).strftime("%Y%m%d%H%M")
556 + for offset in range(minutes)
557 + ]
558 +
559 +
560 +def get_http_minute_key(minute):
561 + return f"{PREFIX}:http:{minute}"
562 +
563 +
564 +def get_ip_minute_key(minute):
565 + return f"{PREFIX}:ip:{minute}"
566 +
567 +
568 +def get_requests_stream_key():
569 + return f"{PREFIX}:requests"
570 +
571 +
572 +###############################################################################
573 +# label helpers
574 +###############################################################################
575 +
576 +def join_label(*parts):
577 + return "|".join(normalize_label_part(part) for part in parts)
578 +
579 +
580 +def normalize_label_part(value):
581 + if value is None:
582 + return "__none__"
583 +
584 + text = str(value).strip()
585 + if not text:
586 + return "__empty__"
587 +
588 + return text.replace("|", "_")
...@@ -17,7 +17,6 @@ SessionLocal = sessionmaker( ...@@ -17,7 +17,6 @@ SessionLocal = sessionmaker(
17 autocommit=False, 17 autocommit=False,
18 autoflush=False, 18 autoflush=False,
19 bind=engine, 19 bind=engine,
20 - # pool_size=1,
21 ) 20 )
22 21
23 Base = declarative_base() 22 Base = declarative_base()
......
...@@ -4,6 +4,7 @@ from . import ( ...@@ -4,6 +4,7 @@ from . import (
4 finfun, 4 finfun,
5 objeto, 5 objeto,
6 treemap, 6 treemap,
7 + project,
7 entity, 8 entity,
8 rubro, 9 rubro,
9 organismo, 10 organismo,
......
...@@ -57,6 +57,7 @@ class ActecoEntidad(Base): ...@@ -57,6 +57,7 @@ class ActecoEntidad(Base):
57 Index("idx_acteco_entidad_gestion", "gestion"), 57 Index("idx_acteco_entidad_gestion", "gestion"),
58 Index("idx_acteco_entidad_acteco", "acteco"), 58 Index("idx_acteco_entidad_acteco", "acteco"),
59 Index("idx_acteco_entidad_acteco_entidad", "acteco", "entidad"), 59 Index("idx_acteco_entidad_acteco_entidad", "acteco", "entidad"),
60 + Index("idx_acteco_entidad_acteco_gestion", "acteco", "gestion"),
60 {"schema": "web"}, 61 {"schema": "web"},
61 ) 62 )
62 63
...@@ -81,6 +82,7 @@ class ActecoUbigeo(Base): ...@@ -81,6 +82,7 @@ class ActecoUbigeo(Base):
81 Index("idx_acteco_ubigeo_gestion", "gestion"), 82 Index("idx_acteco_ubigeo_gestion", "gestion"),
82 Index("idx_acteco_ubigeo_acteco", "acteco"), 83 Index("idx_acteco_ubigeo_acteco", "acteco"),
83 Index("idx_acteco_ubigeo_acteco_entidad", "acteco", "ubigeo"), 84 Index("idx_acteco_ubigeo_acteco_entidad", "acteco", "ubigeo"),
85 + Index("idx_acteco_ubigeo_acteco_gestion", "acteco", "gestion"),
84 {"schema": "web"}, 86 {"schema": "web"},
85 ) 87 )
86 88
......
...@@ -19,6 +19,7 @@ class EntidadDistribuciones(Base): ...@@ -19,6 +19,7 @@ class EntidadDistribuciones(Base):
19 Index("idx_entidad_distribuciones_codigo", "codigo"), 19 Index("idx_entidad_distribuciones_codigo", "codigo"),
20 Index("idx_entidad_distribuciones_dimension", "dimension"), 20 Index("idx_entidad_distribuciones_dimension", "dimension"),
21 Index("idx_entidad_distribuciones_gestion", "gestion"), 21 Index("idx_entidad_distribuciones_gestion", "gestion"),
22 + Index("idx_entidad_distribuciones_compos1", "tipo_codigo", "codigo", "dimension", "gestion"),
22 {"schema": "web"}, 23 {"schema": "web"},
23 ) 24 )
24 25
...@@ -44,11 +45,12 @@ class EntidadResumenes(Base): ...@@ -44,11 +45,12 @@ class EntidadResumenes(Base):
44 Index("idx_entidad_resumenes_tipo_codigo", "tipo_codigo"), 45 Index("idx_entidad_resumenes_tipo_codigo", "tipo_codigo"),
45 Index("idx_entidad_resumenes_codigo", "codigo"), 46 Index("idx_entidad_resumenes_codigo", "codigo"),
46 Index("idx_entidad_resumenes_gestion", "gestion"), 47 Index("idx_entidad_resumenes_gestion", "gestion"),
48 + Index("idx_entidad_resumenes_compos1", "tipo_codigo", "codigo"),
47 {"schema": "web"}, 49 {"schema": "web"},
48 ) 50 )
49 51
50 id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) 52 id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
51 - 53 +
52 tipo: Mapped[str] = mapped_column(String(8)) 54 tipo: Mapped[str] = mapped_column(String(8))
53 tipo_codigo: Mapped[str] = mapped_column(String(16)) 55 tipo_codigo: Mapped[str] = mapped_column(String(16))
54 codigo: Mapped[str] = mapped_column(String(8)) 56 codigo: Mapped[str] = mapped_column(String(8))
......
...@@ -71,6 +71,7 @@ class FinfunEntidad(Base): ...@@ -71,6 +71,7 @@ class FinfunEntidad(Base):
71 Index("idx_finfun_entidad_gestion", "gestion"), 71 Index("idx_finfun_entidad_gestion", "gestion"),
72 Index("idx_finfun_entidad_finfun", "finfun"), 72 Index("idx_finfun_entidad_finfun", "finfun"),
73 Index("idx_finfun_entidad_finfun_entidad", "finfun", "entidad"), 73 Index("idx_finfun_entidad_finfun_entidad", "finfun", "entidad"),
74 + Index("idx_finfun_entidad_finfun_gestion", "finfun", "gestion"),
74 {"schema": "web"}, 75 {"schema": "web"},
75 ) 76 )
76 77
...@@ -95,6 +96,7 @@ class FinfunUbigeo(Base): ...@@ -95,6 +96,7 @@ class FinfunUbigeo(Base):
95 Index("idx_finfun_ubigeo_gestion", "gestion"), 96 Index("idx_finfun_ubigeo_gestion", "gestion"),
96 Index("idx_finfun_ubigeo_finfun", "finfun"), 97 Index("idx_finfun_ubigeo_finfun", "finfun"),
97 Index("idx_finfun_ubigeo_finfun_entidad", "finfun", "ubigeo"), 98 Index("idx_finfun_ubigeo_finfun_entidad", "finfun", "ubigeo"),
99 + Index("idx_finfun_ubigeo_finfun_gestion", "finfun", "gestion"),
98 {"schema": "web"}, 100 {"schema": "web"},
99 ) 101 )
100 102
......
...@@ -72,6 +72,7 @@ class ObjetoEntidad(Base): ...@@ -72,6 +72,7 @@ class ObjetoEntidad(Base):
72 Index("idx_objeto_entidad_gestion", "gestion"), 72 Index("idx_objeto_entidad_gestion", "gestion"),
73 Index("idx_objeto_entidad_objeto", "objeto"), 73 Index("idx_objeto_entidad_objeto", "objeto"),
74 Index("idx_objeto_entidad_objeto_entidad", "objeto", "entidad"), 74 Index("idx_objeto_entidad_objeto_entidad", "objeto", "entidad"),
75 + Index("idx_objeto_entidad_objeto_gestion", "objeto", "gestion"),
75 {"schema": "web"}, 76 {"schema": "web"},
76 ) 77 )
77 78
...@@ -96,6 +97,7 @@ class ObjetoUbigeo(Base): ...@@ -96,6 +97,7 @@ class ObjetoUbigeo(Base):
96 Index("idx_objeto_ubigeo_gestion", "gestion"), 97 Index("idx_objeto_ubigeo_gestion", "gestion"),
97 Index("idx_objeto_ubigeo_objeto", "objeto"), 98 Index("idx_objeto_ubigeo_objeto", "objeto"),
98 Index("idx_objeto_ubigeo_objeto_entidad", "objeto", "ubigeo"), 99 Index("idx_objeto_ubigeo_objeto_entidad", "objeto", "ubigeo"),
100 + Index("idx_objeto_ubigeo_objeto_gestion", "objeto", "gestion"),
99 {"schema": "web"}, 101 {"schema": "web"},
100 ) 102 )
101 103
......
...@@ -57,6 +57,7 @@ class OrganismoEntidad(Base): ...@@ -57,6 +57,7 @@ class OrganismoEntidad(Base):
57 Index("idx_organismo_entidad_gestion", "gestion"), 57 Index("idx_organismo_entidad_gestion", "gestion"),
58 Index("idx_organismo_entidad_objeto", "organismo"), 58 Index("idx_organismo_entidad_objeto", "organismo"),
59 Index("idx_organismo_entidad_rubro_entidad", "organismo", "entidad"), 59 Index("idx_organismo_entidad_rubro_entidad", "organismo", "entidad"),
60 + Index("idx_organismo_entidad_rubro_gestion", "organismo", "gestion"),
60 {"schema": "web"}, 61 {"schema": "web"},
61 ) 62 )
62 63
......
1 +from sqlalchemy.orm import Mapped, mapped_column
2 +from presupuestov1.model import Base
3 +
4 +from sqlalchemy import (
5 + Integer,
6 + String,
7 + Text,
8 + Numeric,
9 + Index,
10 + MetaData,
11 +)
12 +
13 +class ProyectoResumenes(Base):
14 + __tablename__ = "proyecto_resumenes"
15 + __table_args__ = (
16 + Index("idx_proyecto_resumenes_gestion", "gestion"),
17 + Index("idx_proyecto_resumenes_nivel", "nivel"),
18 + Index("idx_proyecto_resumenes_codigo", "codigo"),
19 + Index("idx_proyecto_resumenes_codigo_padre", "codigo_padre"),
20 + {"schema": "web"},
21 + )
22 +
23 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
24 +
25 + gestion: Mapped[int] = mapped_column(Integer)
26 + nivel: Mapped[str] = mapped_column(Text)
27 + codigo: Mapped[str] = mapped_column(String(32))
28 + codigo_padre: Mapped[str | None] = mapped_column(String(32))
29 +
30 + devengado: Mapped[float] = mapped_column(Numeric(30, 15))
31 +
32 + n_entidad: Mapped[int] = mapped_column(Integer)
33 + n_objeto: Mapped[int] = mapped_column(Integer)
34 + n_finfun: Mapped[int] = mapped_column(Integer)
35 + n_acteco: Mapped[int] = mapped_column(Integer)
36 +
37 + desc: Mapped[str] = mapped_column(Text)
38 + desc_padre: Mapped[str | None] = mapped_column(Text)
39 +
40 + entidad: Mapped[str] = mapped_column(Text)
41 + objeto: Mapped[str] = mapped_column(Text)
42 + finfun: Mapped[str] = mapped_column(Text)
43 + acteco: Mapped[str] = mapped_column(Text)
...@@ -57,6 +57,7 @@ class RubroEntidad(Base): ...@@ -57,6 +57,7 @@ class RubroEntidad(Base):
57 Index("idx_rubro_entidad_gestion", "gestion"), 57 Index("idx_rubro_entidad_gestion", "gestion"),
58 Index("idx_rubro_entidad_objeto", "rubro"), 58 Index("idx_rubro_entidad_objeto", "rubro"),
59 Index("idx_rubro_entidad_rubro_entidad", "rubro", "entidad"), 59 Index("idx_rubro_entidad_rubro_entidad", "rubro", "entidad"),
60 + Index("idx_rubro_entidad_rubro_gestion", "rubro", "gestion"),
60 {"schema": "web"}, 61 {"schema": "web"},
61 ) 62 )
62 63
......
...@@ -15,6 +15,8 @@ from sqlalchemy import and_, or_, select ...@@ -15,6 +15,8 @@ from sqlalchemy import and_, or_, select
15 class TreemapObjeto(Base): 15 class TreemapObjeto(Base):
16 __tablename__ = "treemap_objeto" 16 __tablename__ = "treemap_objeto"
17 __table_args__ = ( 17 __table_args__ = (
18 + Index("idx_treemap_objeto_objeto", "objeto"),
19 + Index("idx_treemap_objeto_objeto_gestion", "objeto", "gestion"),
18 Index("idx_treemap_objeto_gestion_entidad", "gestion", "entidad"), 20 Index("idx_treemap_objeto_gestion_entidad", "gestion", "entidad"),
19 Index("idx_treemap_objeto_nivel", "nivel"), 21 Index("idx_treemap_objeto_nivel", "nivel"),
20 {"schema": "web"}, 22 {"schema": "web"},
...@@ -39,6 +41,8 @@ class TreemapObjeto(Base): ...@@ -39,6 +41,8 @@ class TreemapObjeto(Base):
39 class TreemapFinFun(Base): 41 class TreemapFinFun(Base):
40 __tablename__ = "treemap_finfun" 42 __tablename__ = "treemap_finfun"
41 __table_args__ = ( 43 __table_args__ = (
44 + Index("idx_treemap_finfun_finfun", "finfun"),
45 + Index("idx_treemap_finfun_finfun_gestion", "finfun", "gestion"),
42 Index("idx_treemap_finfun_gestion_entidad", "gestion", "entidad"), 46 Index("idx_treemap_finfun_gestion_entidad", "gestion", "entidad"),
43 Index("idx_treemap_finfun_nivel", "nivel"), 47 Index("idx_treemap_finfun_nivel", "nivel"),
44 {"schema": "web"}, 48 {"schema": "web"},
...@@ -63,6 +67,8 @@ class TreemapFinFun(Base): ...@@ -63,6 +67,8 @@ class TreemapFinFun(Base):
63 class TreemapActeco(Base): 67 class TreemapActeco(Base):
64 __tablename__ = "treemap_acteco" 68 __tablename__ = "treemap_acteco"
65 __table_args__ = ( 69 __table_args__ = (
70 + Index("idx_treemap_acteco_acteco", "acteco"),
71 + Index("idx_treemap_acteco_acteco_gestion", "acteco", "gestion"),
66 Index("idx_treemap_acteco_gestion_entidad", "gestion", "entidad"), 72 Index("idx_treemap_acteco_gestion_entidad", "gestion", "entidad"),
67 Index("idx_treemap_acteco_nivel", "nivel"), 73 Index("idx_treemap_acteco_nivel", "nivel"),
68 {"schema": "web"}, 74 {"schema": "web"},
...@@ -87,6 +93,8 @@ class TreemapActeco(Base): ...@@ -87,6 +93,8 @@ class TreemapActeco(Base):
87 class TreemapRubro(Base): 93 class TreemapRubro(Base):
88 __tablename__ = "treemap_rubro" 94 __tablename__ = "treemap_rubro"
89 __table_args__ = ( 95 __table_args__ = (
96 + Index("idx_treemap_rubro_rubro", "rubro"),
97 + Index("idx_treemap_rubro_rubro_gestion", "rubro", "gestion"),
90 Index("idx_treemap_rubro_gestion_entidad", "gestion", "entidad"), 98 Index("idx_treemap_rubro_gestion_entidad", "gestion", "entidad"),
91 Index("idx_treemap_rubro_nivel", "nivel"), 99 Index("idx_treemap_rubro_nivel", "nivel"),
92 {"schema": "web"}, 100 {"schema": "web"},
......
...@@ -194,7 +194,7 @@ def fetch_acteco_ubigeo( ...@@ -194,7 +194,7 @@ def fetch_acteco_ubigeo(
194 query: schema.QueryParams = Depends(), 194 query: schema.QueryParams = Depends(),
195 ): 195 ):
196 gestion = query.gestion or 0 196 gestion = query.gestion or 0
197 - ClasObjetos = models.objeto.ActecoUbigeo 197 + ClasObjetos = models.acteco.ActecoUbigeo
198 198
199 return db.query(ClasObjetos).filter( 199 return db.query(ClasObjetos).filter(
200 ClasObjetos.acteco == acteco_id, 200 ClasObjetos.acteco == acteco_id,
......
...@@ -73,50 +73,74 @@ def fetch_entidades( ...@@ -73,50 +73,74 @@ def fetch_entidades(
73 @router.get('/timeline', response_model=List[schemas.daily.Timeline]) 73 @router.get('/timeline', response_model=List[schemas.daily.Timeline])
74 @not_found 74 @not_found
75 def fetch_timeline( 75 def fetch_timeline(
76 - db: Session = Depends(model.get_db), 76 + db: Session = Depends(model.get_db),
77 - query: schema.DailyParams = Depends(), 77 + query: schema.DailyParams = Depends(),
78 ): 78 ):
79 - Daily = models.daily.Timeline 79 + Daily = models.daily.Timeline
80 - page = query.page or 1 80 + page = query.page or 1
81 - page_size = 10 81 + page_size = 10
82 - 82 +
83 - if query.entidad is not None: 83 + if query.entidad is not None:
84 - return db.query(Daily).filter( 84 + stmt = (
85 - Daily.entidad == query.entidad 85 + select(Daily)
86 - ).order_by( 86 + .where(Daily.entidad == query.entidad)
87 - Daily.devengado.desc() 87 + .order_by(Daily.devengado.desc(), Daily.id)
88 - ).offset( 88 + .offset((page - 1) * page_size)
89 - (page - 1) * page_size 89 + .limit(page_size)
90 - ).limit(page_size).all() 90 + )
91 - 91 + return db.execute(stmt).scalars().all()
92 - # top 10 entidades 92 +
93 - top_entities = db.query( 93 + total_devengado = func.sum(Daily.devengado).label("total_devengado")
94 - Daily.entidad 94 +
95 - ).group_by( 95 + top_entities = (
96 - Daily.entidad 96 + select(
97 - ).order_by( 97 + Daily.entidad.label("entidad"),
98 - func.sum(Daily.devengado).desc() 98 + total_devengado,
99 - ).limit(10).subquery() 99 + )
100 - 100 + .group_by(Daily.entidad)
101 - # ordena las filas por devengado desc por cada una de las top 10 entidades 101 + .order_by(total_devengado.desc(), Daily.entidad)
102 - ranked = db.query( 102 + .limit(10)
103 - Daily, 103 + .subquery()
104 - func.row_number().over( 104 + )
105 - partition_by=Daily.entidad, 105 +
106 - order_by=Daily.devengado.desc() 106 + ranked = (
107 - ).label('row_num') 107 + select(
108 - ).filter( 108 + Daily.entidad.label("entidad"),
109 - Daily.entidad.in_(top_entities) 109 + Daily.devengado.label("devengado"),
110 - ).subquery() 110 + Daily.entidad_desc_area.label("entidad_desc_area"),
111 - 111 + Daily.entidad_desc_entidad.label("entidad_desc_entidad"),
112 - # top 10 filas por cada una de las top 10 entidades 112 + Daily.desc_programa.label("desc_programa"),
113 - RankedTimeline = aliased(Daily, ranked) 113 + Daily.desc_actividad_proyecto.label("desc_actividad_proyecto"),
114 - return db.query(RankedTimeline).filter( 114 + Daily.id.label("id"),
115 - ranked.c.row_num <= page_size 115 + top_entities.c.total_devengado,
116 - ).order_by( 116 + func.row_number().over(
117 - RankedTimeline.entidad, 117 + partition_by=Daily.entidad,
118 - RankedTimeline.devengado.desc() 118 + order_by=(Daily.devengado.desc(), Daily.id),
119 - ).all() 119 + ).label("row_num"),
120 + )
121 + .join(top_entities, top_entities.c.entidad == Daily.entidad)
122 + .subquery()
123 + )
124 +
125 + stmt = (
126 + select(
127 + ranked.c.entidad,
128 + ranked.c.devengado,
129 + ranked.c.entidad_desc_area,
130 + ranked.c.entidad_desc_entidad,
131 + ranked.c.desc_programa,
132 + ranked.c.desc_actividad_proyecto,
133 + )
134 + .where(ranked.c.row_num <= page_size)
135 + .order_by(
136 + ranked.c.total_devengado.desc(),
137 + ranked.c.entidad,
138 + ranked.c.devengado.desc(),
139 + ranked.c.id,
140 + )
141 + )
142 +
143 + return db.execute(stmt).mappings().all()
120 144
121 145
122 @router.get('/timeseries', response_model=List[schemas.daily.TimeSeries]) 146 @router.get('/timeseries', response_model=List[schemas.daily.TimeSeries])
......
...@@ -46,7 +46,6 @@ def fetch_ubigeo_classifier( ...@@ -46,7 +46,6 @@ def fetch_ubigeo_classifier(
46 return db.query(models.classifier.ClasGeografico).all() 46 return db.query(models.classifier.ClasGeografico).all()
47 47
48 48
49 -
50 # base 49 # base
51 50
52 @router_entidad.get('/{entity_id:str}', response_model=schemas.entity.ClasInstitucionalIngresosGastos) 51 @router_entidad.get('/{entity_id:str}', response_model=schemas.entity.ClasInstitucionalIngresosGastos)
......
1 +from fastapi import APIRouter, Query
2 +
3 +from presupuestov1 import metrics
4 +
5 +
6 +WINDOW_MINUTES_MAX = 7 * 24 * 60
7 +
8 +router = APIRouter(
9 + prefix='/_metrics',
10 + include_in_schema=False,
11 +)
12 +
13 +
14 +###############################################################################
15 +# GET
16 +###############################################################################
17 +
18 +@router.get('/summary')
19 +async def fetch_metrics_summary(
20 + minutes: int = Query(default=60, ge=1, le=WINDOW_MINUTES_MAX),
21 + include_ts: bool = Query(default=False),
22 +):
23 + return await metrics.read_summary(minutes, include_ts)
24 +
25 +
26 +@router.get('/routes')
27 +async def fetch_metrics_routes(
28 + minutes: int = Query(default=60, ge=1, le=WINDOW_MINUTES_MAX),
29 + include_ts: bool = Query(default=False),
30 +):
31 + return await metrics.read_routes(minutes, include_ts)
32 +
33 +
34 +@router.get('/ips')
35 +async def fetch_metrics_ips(
36 + minutes: int = Query(default=60, ge=1, le=WINDOW_MINUTES_MAX),
37 + include_ts: bool = Query(default=False),
38 +):
39 + return await metrics.read_ips(minutes, include_ts)
1 -import time 1 +import pydantic_core
2 2
3 from typing import List, Any, Optional, Union 3 from typing import List, Any, Optional, Union
4 4
5 from fastapi import APIRouter, Depends, Request, HTTPException, Body 5 from fastapi import APIRouter, Depends, Request, HTTPException, Body
6 from sqlalchemy.orm import Session 6 from sqlalchemy.orm import Session
7 +from sqlalchemy import and_, or_, select, func, desc
7 8
8 from presupuestov1 import model, schema, constant, schemas, models, search 9 from presupuestov1 import model, schema, constant, schemas, models, search
9 from presupuestov1.routers._decorators import not_found 10 from presupuestov1.routers._decorators import not_found
...@@ -19,23 +20,55 @@ router = APIRouter( ...@@ -19,23 +20,55 @@ router = APIRouter(
19 # GET 20 # GET
20 ############################################################################### 21 ###############################################################################
21 22
22 -@router.get('/{project_id}') 23 +@router.get('/{project_id}', response_model=List[schemas.project.ProyectoResumen])
23 @not_found 24 @not_found
24 def fetch_programa_proyecto( 25 def fetch_programa_proyecto(
25 - project_id: int, 26 + project_id: str,
26 db: Session = Depends(model.get_db), 27 db: Session = Depends(model.get_db),
27 ): 28 ):
28 - return [] 29 + ProyectoResumenes = models.project.ProyectoResumenes
30 + project_items = db.query(ProyectoResumenes).filter(
31 + ProyectoResumenes.codigo == project_id
32 + ).all()
33 +
34 + # TODO if devengado > mucho_dinero: cache
35 +
36 + for _ in ['entidad', 'objeto', 'finfun', 'acteco']:
37 + for __ in project_items:
38 + obj_ = getattr(__, _)
39 + setattr(__, _, pydantic_core.from_json(obj_))
40 +
41 + return project_items
29 42
30 43
31 @router.get('/{project_id}/proyectos') 44 @router.get('/{project_id}/proyectos')
32 @not_found 45 @not_found
33 def fetch_programa_proyecto_proyectos( 46 def fetch_programa_proyecto_proyectos(
34 - project_id: int, 47 + project_id: str,
35 db: Session = Depends(model.get_db), 48 db: Session = Depends(model.get_db),
36 - query: schema.QueryParams = Depends(), 49 + query: schema.SearchQueryParams = Depends(),
37 ): 50 ):
38 - return [] 51 + page = query.page or 1
52 + page_size = 20
53 +
54 + ProyectoResumenes = models.project.ProyectoResumenes
55 + return db.execute(
56 + select(
57 + ProyectoResumenes.nivel,
58 + ProyectoResumenes.codigo,
59 + ProyectoResumenes.desc,
60 + func.sum(ProyectoResumenes.devengado).label('devengado'),
61 + ).where(
62 + ProyectoResumenes.codigo_padre == project_id,
63 + ProyectoResumenes.nivel != 'programa',
64 + ).group_by(
65 + ProyectoResumenes.nivel,
66 + ProyectoResumenes.codigo,
67 + ProyectoResumenes.desc,
68 + ).order_by(desc('devengado')).offset(
69 + (page - 1) * page_size
70 + ).limit(page_size)
71 + ).mappings().all()
39 72
40 73
41 @router.get('/{project_id}/objetos') 74 @router.get('/{project_id}/objetos')
......
1 from typing import List, Any, Optional, Union 1 from typing import List, Any, Optional, Union
2 2
3 -from fastapi import APIRouter, Depends, Request, HTTPException, Body 3 +from fastapi import APIRouter, Depends, HTTPException
4 from sqlalchemy.orm import Session 4 from sqlalchemy.orm import Session
5 5
6 from presupuestov1 import model, schema, constant, schemas, models, search 6 from presupuestov1 import model, schema, constant, schemas, models, search
...@@ -28,6 +28,13 @@ def fetch_search( ...@@ -28,6 +28,13 @@ def fetch_search(
28 'hits': [], 28 'hits': [],
29 } 29 }
30 30
31 - return search.do_search( 31 + try:
32 - query.q, query.page, query.is_class, query.class_, query.order 32 + return search.do_search(
33 - ) 33 + query.q, query.page, query.is_class, query.class_, query.order
34 + )
35 + except search.SearchUnavailable as exc:
36 + raise HTTPException(
37 + status_code=503,
38 + detail='Search temporarily unavailable',
39 + headers={'Retry-After': str(exc.retry_after)},
40 + ) from exc
......
...@@ -19,14 +19,14 @@ class ActecoEstado(OrmBaseModel): ...@@ -19,14 +19,14 @@ class ActecoEstado(OrmBaseModel):
19 top1_entidad_desc: str | None 19 top1_entidad_desc: str | None
20 top1_monto: float | None 20 top1_monto: float | None
21 top1_pct: float | None 21 top1_pct: float | None
22 - top2_entidad: int | None | float 22 + top2_entidad: int | None | float | str
23 top2_entidad_desc: str | None 23 top2_entidad_desc: str | None
24 - top2_monto: float | None 24 + top2_monto: float | None | str
25 - top2_pct: float | None 25 + top2_pct: float | None | str
26 - top3_entidad: int | None | float 26 + top3_entidad: int | None | float | str
27 top3_entidad_desc: str | None 27 top3_entidad_desc: str | None
28 - top3_monto: float | None 28 + top3_monto: float | None | str
29 - top3_pct: float | None 29 + top3_pct: float | None | str
30 30
31 31
32 class ActecoEntidad(OrmBaseModel): 32 class ActecoEntidad(OrmBaseModel):
......
...@@ -26,14 +26,14 @@ class FinfunEstado(OrmBaseModel): ...@@ -26,14 +26,14 @@ class FinfunEstado(OrmBaseModel):
26 top1_entidad_desc: str | None 26 top1_entidad_desc: str | None
27 top1_monto: float | None 27 top1_monto: float | None
28 top1_pct: float | None 28 top1_pct: float | None
29 - top2_entidad: int | None | float 29 + top2_entidad: int | None | float | str
30 top2_entidad_desc: str | None 30 top2_entidad_desc: str | None
31 - top2_monto: float | None 31 + top2_monto: float | None | str
32 - top2_pct: float | None 32 + top2_pct: float | None | str
33 - top3_entidad: int | None | float 33 + top3_entidad: int | None | float | str
34 top3_entidad_desc: str | None 34 top3_entidad_desc: str | None
35 - top3_monto: float | None 35 + top3_monto: float | None | str
36 - top3_pct: float | None 36 + top3_pct: float | None | str
37 37
38 38
39 class FinfunEntidad(OrmBaseModel): 39 class FinfunEntidad(OrmBaseModel):
......
...@@ -26,14 +26,14 @@ class ObjetoEstado(OrmBaseModel): ...@@ -26,14 +26,14 @@ class ObjetoEstado(OrmBaseModel):
26 top1_entidad_desc: str | None 26 top1_entidad_desc: str | None
27 top1_monto: float | None 27 top1_monto: float | None
28 top1_pct: float | None 28 top1_pct: float | None
29 - top2_entidad: int | None 29 + top2_entidad: int | None | str
30 top2_entidad_desc: str | None 30 top2_entidad_desc: str | None
31 - top2_monto: float | None 31 + top2_monto: float | None | str
32 - top2_pct: float | None 32 + top2_pct: float | None | str
33 - top3_entidad: int | None 33 + top3_entidad: int | None | str
34 top3_entidad_desc: str | None 34 top3_entidad_desc: str | None
35 - top3_monto: float | None 35 + top3_monto: float | None | str
36 - top3_pct: float | None 36 + top3_pct: float | None | str
37 37
38 38
39 class ObjetoEntidad(OrmBaseModel): 39 class ObjetoEntidad(OrmBaseModel):
......
...@@ -20,14 +20,14 @@ class OrganismoEstado(OrmBaseModel): ...@@ -20,14 +20,14 @@ class OrganismoEstado(OrmBaseModel):
20 top1_entidad_desc: str | None 20 top1_entidad_desc: str | None
21 top1_monto: float | None 21 top1_monto: float | None
22 top1_pct: float | None 22 top1_pct: float | None
23 - top2_entidad: int | None | float 23 + top2_entidad: int | None | float | str
24 top2_entidad_desc: str | None 24 top2_entidad_desc: str | None
25 - top2_monto: float | None 25 + top2_monto: float | None | str
26 - top2_pct: float | None 26 + top2_pct: float | None | str
27 - top3_entidad: int | None | float 27 + top3_entidad: int | None | float | str
28 top3_entidad_desc: str | None 28 top3_entidad_desc: str | None
29 - top3_monto: float | None 29 + top3_monto: float | None | str
30 - top3_pct: float | None 30 + top3_pct: float | None | str
31 31
32 32
33 class OrganismoEntidad(OrmBaseModel): 33 class OrganismoEntidad(OrmBaseModel):
......
1 -from typing import List, Any, Dict 1 +from typing import List
2 2
3 -from presupuestov1.schema import BaseModel 3 +from presupuestov1.schema import BaseModel, OrmBaseModel
4 +
5 +
6 +class ProyectoResumenObj(BaseModel):
7 + codigo: int | str
8 + desc: str | None
9 + desc_padre: str | None
10 + devengado: float
11 +
12 +class ProyectoResumen(OrmBaseModel):
13 + gestion: int
14 + codigo: str
15 + codigo_padre: str | None
16 + devengado: float
17 + n_entidad: int
18 + n_objeto: int
19 + n_finfun: int
20 + n_acteco: int
21 + desc: str
22 + desc_padre: str | None
23 + entidad: str | List[ProyectoResumenObj]
24 + objeto: str | List[ProyectoResumenObj]
25 + finfun: str | List[ProyectoResumenObj]
26 + acteco: str | List[ProyectoResumenObj]
27 +
28 +
29 +class ProyectoProyectos(OrmBaseModel):
30 + nivel: str
31 + codigo: str
32 + desc: str
4 33
5 34
6 class ProjectSearch(BaseModel): 35 class ProjectSearch(BaseModel):
......
...@@ -20,14 +20,14 @@ class RubroEstado(OrmBaseModel): ...@@ -20,14 +20,14 @@ class RubroEstado(OrmBaseModel):
20 top1_entidad_desc: str | None 20 top1_entidad_desc: str | None
21 top1_monto: float | None 21 top1_monto: float | None
22 top1_pct: float | None 22 top1_pct: float | None
23 - top2_entidad: int | None | float 23 + top2_entidad: int | None | float | str
24 top2_entidad_desc: str | None 24 top2_entidad_desc: str | None
25 - top2_monto: float | None 25 + top2_monto: float | None | str
26 - top2_pct: float | None 26 + top2_pct: float | None | str
27 - top3_entidad: int | None | float 27 + top3_entidad: int | None | float | str
28 top3_entidad_desc: str | None 28 top3_entidad_desc: str | None
29 - top3_monto: float | None 29 + top3_monto: float | None | str
30 - top3_pct: float | None 30 + top3_pct: float | None | str
31 31
32 32
33 class RubroEntidad(OrmBaseModel): 33 class RubroEntidad(OrmBaseModel):
......
1 +import math
2 +import threading
3 +import time
4 +
1 import typesense 5 import typesense
6 +from typesense import exceptions as typesense_exceptions
2 7
3 TYPESENSE_API = open('./docs/typesense').read().strip() 8 TYPESENSE_API = open('./docs/typesense').read().strip()
4 client = typesense.Client({ 9 client = typesense.Client({
...@@ -8,7 +13,7 @@ client = typesense.Client({ ...@@ -8,7 +13,7 @@ client = typesense.Client({
8 'port': '8108', 13 'port': '8108',
9 'protocol': 'http' 14 'protocol': 'http'
10 }], 15 }],
11 - 'connection_timeout_seconds': 2 16 + 'connection_timeout_seconds': 0.5
12 }) 17 })
13 18
14 COLLECTION = 'presup_actpro_fts' 19 COLLECTION = 'presup_actpro_fts'
...@@ -34,9 +39,70 @@ search_parameters = { ...@@ -34,9 +39,70 @@ search_parameters = {
34 39
35 40
36 AVAILABLE_CLASSES = [ 41 AVAILABLE_CLASSES = [
37 - 'programa_proyecto', 'entidad', 'objeto', 'finfun', 'acteco' 42 + 'programa_proyecto',
43 + 'entidad',
44 + 'objeto',
45 + 'finfun',
46 + 'acteco',
47 + 'ubigeo',
48 + 'rubro',
49 + 'organismo',
38 ] 50 ]
51 +
52 +BACKEND_ERRORS = (
53 + typesense_exceptions.Timeout,
54 + typesense_exceptions.ServiceUnavailable,
55 + typesense_exceptions.ServerError,
56 + typesense_exceptions.HTTPStatus0Error,
57 + OSError,
58 +)
59 +FAILURE_THRESHOLD = 2
60 +OPEN_INTERVAL_SECONDS = 15.0
61 +
62 +
63 +class SearchUnavailable(Exception):
64 + def __init__(self, retry_after: int):
65 + self.retry_after = retry_after
66 + super().__init__('search backend unavailable')
67 +
68 +
69 +class SearchGuard:
70 + def __init__(self):
71 + self._lock = threading.Lock()
72 + self._failures = 0
73 + self._opened_until = 0.0
74 +
75 + def enter(self):
76 + now = time.monotonic()
77 +
78 + with self._lock:
79 + if self._opened_until > now:
80 + raise SearchUnavailable(max(1, math.ceil(self._opened_until - now)))
81 +
82 + def record_success(self):
83 + with self._lock:
84 + self._failures = 0
85 + self._opened_until = 0.0
86 +
87 + def record_failure(self):
88 + now = time.monotonic()
89 +
90 + with self._lock:
91 + self._failures += 1
92 +
93 + if self._failures >= FAILURE_THRESHOLD:
94 + self._opened_until = now + OPEN_INTERVAL_SECONDS
95 + return int(OPEN_INTERVAL_SECONDS)
96 +
97 + return 1
98 +
99 +
100 +search_guard = SearchGuard()
101 +
102 +
39 def do_search(query, page, is_class, class_=None, order=None): 103 def do_search(query, page, is_class, class_=None, order=None):
104 + search_guard.enter()
105 +
40 search_parameters_ = search_parameters.copy() 106 search_parameters_ = search_parameters.copy()
41 search_parameters_['q'] = query 107 search_parameters_['q'] = query
42 108
...@@ -60,6 +126,12 @@ def do_search(query, page, is_class, class_=None, order=None): ...@@ -60,6 +126,12 @@ def do_search(query, page, is_class, class_=None, order=None):
60 if order == 'asc': 126 if order == 'asc':
61 search_parameters_['sort_by'] = 'devengado:asc' 127 search_parameters_['sort_by'] = 'devengado:asc'
62 128
63 - return client.collections[COLLECTION].documents.search( 129 + try:
64 - search_parameters_ 130 + response = client.collections[COLLECTION].documents.search(
65 - ) 131 + search_parameters_
132 + )
133 + except BACKEND_ERRORS:
134 + raise SearchUnavailable(search_guard.record_failure())
135 +
136 + search_guard.record_success()
137 + return response
......