Gabriel Weisse / presupuestov1

Go to a project
Toggle navigation Toggle navigation pinning
pr0nstar
72380414 1 parent 783e6fac

XIII

Expand all
Inline Side-by-side
Showing 27 changed files with 436 additions and 105 deletions
...@@ -6,7 +6,7 @@ from starlette_caches.utils import cache as scache ...@@ -6,7 +6,7 @@ from starlette_caches.utils import cache as scache
6 from aiocache import Cache 6 from aiocache import Cache
7 7
8 8
9 -REDIS_CONN = open('./docs/postgres').read().strip() 9 +REDIS_CONN = open('./docs/redis').read().strip()
10 REDIS_CONN = REDIS_CONN.split(':') 10 REDIS_CONN = REDIS_CONN.split(':')
11 # cache = Cache( 11 # cache = Cache(
12 # Cache.REDIS, 12 # Cache.REDIS,
......
1 +import logging
2 +import threading
3 +import time
4 +import traceback
5 +
6 +
7 +WINDOW_SECONDS = 1
8 +MAX_KEYS = 1024
9 +
10 +
11 +class DuplicateErrorFilter(logging.Filter):
12 + def __init__(self, window_seconds=WINDOW_SECONDS, max_keys=MAX_KEYS):
13 + super().__init__()
14 + self.window_seconds = window_seconds
15 + self.max_keys = max_keys
16 + self._lock = threading.Lock()
17 + self._entries = {}
18 +
19 + def filter(self, record):
20 + if record.levelno < logging.ERROR:
21 + return True
22 +
23 + now = time.monotonic()
24 + key = self.build_key(record)
25 +
26 + with self._lock:
27 + self.prune(now)
28 +
29 + opened_until = self._entries.get(key, 0.)
30 + if opened_until > now:
31 + return False
32 +
33 + self._entries[key] = now + self.window_seconds
34 +
35 + return True
36 +
37 + def build_key(self, record):
38 + exc_type = ""
39 + frame = ("", 0, "")
40 + exc_info = record.exc_info
41 +
42 + if isinstance(exc_info, tuple) and len(exc_info) == 3 and exc_info[0] is not None:
43 + exc_type = exc_info[0].__name__
44 + extracted = traceback.extract_tb(exc_info[2])
45 + if extracted:
46 + last = extracted[-1]
47 + frame = (last.filename, last.lineno, last.name)
48 +
49 + return (record.name, exc_type, frame, record.msg)
50 +
51 + def prune(self, now):
52 + expired = [
53 + key
54 + for key, opened_until in self._entries.items()
55 + if opened_until <= now
56 + ]
57 + for key in expired:
58 + self._entries.pop(key, None)
59 +
60 + if len(self._entries) <= self.max_keys:
61 + return
62 +
63 + oldest = sorted(
64 + self._entries.items(),
65 + key=lambda item: item[1],
66 + )[:len(self._entries) - self.max_keys]
67 + for key, _ in oldest:
68 + self._entries.pop(key, None)
...@@ -15,7 +15,7 @@ from starlette.middleware import Middleware ...@@ -15,7 +15,7 @@ from starlette.middleware import Middleware
15 from starlette_caches.middleware import CacheMiddleware 15 from starlette_caches.middleware import CacheMiddleware
16 from starlette_caches.rules import Rule 16 from starlette_caches.rules import Rule
17 17
18 -from presupuestov1 import cache 18 +from presupuestov1 import cache, metrics
19 from presupuestov1.routers import ( 19 from presupuestov1.routers import (
20 project, 20 project,
21 entity, 21 entity,
...@@ -23,17 +23,10 @@ from presupuestov1.routers import ( ...@@ -23,17 +23,10 @@ from presupuestov1.routers import (
23 classifier_income, 23 classifier_income,
24 daily, 24 daily,
25 search, 25 search,
26 + private_metrics,
26 ) 27 )
27 28
28 29
29 -logging.basicConfig(
30 - level=logging.INFO,
31 - format='%(asctime)s %(message)s',
32 - filename='log.log',
33 -)
34 -logging.getLogger('api_proy2').setLevel(logging.DEBUG)
35 -
36 -
37 app = FastAPI( 30 app = FastAPI(
38 title='MEFP presupuestosv1 API', 31 title='MEFP presupuestosv1 API',
39 middleware=[ 32 middleware=[
...@@ -47,13 +40,16 @@ app = FastAPI( ...@@ -47,13 +40,16 @@ app = FastAPI(
47 CacheMiddleware, 40 CacheMiddleware,
48 cache=cache.cache, 41 cache=cache.cache,
49 rules=[ 42 rules=[
43 + Rule(match=re.compile(r"^/api/_metrics/.*"), ttl=0),
50 Rule(match=re.compile(r"^/api/search/.*"), ttl=0), 44 Rule(match=re.compile(r"^/api/search/.*"), ttl=0),
45 + Rule(match=re.compile(r"^/api/programa_proyecto/.*"), ttl=0),
51 Rule(match=re.compile(r'^/api/.+'), ttl=86400, status=200), 46 Rule(match=re.compile(r'^/api/.+'), ttl=86400, status=200),
52 ], 47 ],
53 ), 48 ),
54 ], 49 ],
55 root_path='/api' 50 root_path='/api'
56 ) 51 )
52 +
57 app.include_router(entity.router_entidad) 53 app.include_router(entity.router_entidad)
58 app.include_router(entity.router_ubigeo) 54 app.include_router(entity.router_ubigeo)
59 app.include_router(entity.router_da) 55 app.include_router(entity.router_da)
...@@ -65,6 +61,9 @@ app.include_router(classifier_income.router_organismo) ...@@ -65,6 +61,9 @@ app.include_router(classifier_income.router_organismo)
65 app.include_router(project.router) 61 app.include_router(project.router)
66 app.include_router(daily.router) 62 app.include_router(daily.router)
67 app.include_router(search.router) 63 app.include_router(search.router)
64 +app.include_router(private_metrics.router)
65 +
66 +# metrics.install(app)
68 67
69 68
70 # @app.on_event('startup') 69 # @app.on_event('startup')
......
This diff is collapsed. Click to expand it.
...@@ -17,7 +17,6 @@ SessionLocal = sessionmaker( ...@@ -17,7 +17,6 @@ SessionLocal = sessionmaker(
17 autocommit=False, 17 autocommit=False,
18 autoflush=False, 18 autoflush=False,
19 bind=engine, 19 bind=engine,
20 - # pool_size=1,
21 ) 20 )
22 21
23 Base = declarative_base() 22 Base = declarative_base()
......
...@@ -4,6 +4,7 @@ from . import ( ...@@ -4,6 +4,7 @@ from . import (
4 finfun, 4 finfun,
5 objeto, 5 objeto,
6 treemap, 6 treemap,
7 + project,
7 entity, 8 entity,
8 rubro, 9 rubro,
9 organismo, 10 organismo,
......
...@@ -57,6 +57,7 @@ class ActecoEntidad(Base): ...@@ -57,6 +57,7 @@ class ActecoEntidad(Base):
57 Index("idx_acteco_entidad_gestion", "gestion"), 57 Index("idx_acteco_entidad_gestion", "gestion"),
58 Index("idx_acteco_entidad_acteco", "acteco"), 58 Index("idx_acteco_entidad_acteco", "acteco"),
59 Index("idx_acteco_entidad_acteco_entidad", "acteco", "entidad"), 59 Index("idx_acteco_entidad_acteco_entidad", "acteco", "entidad"),
60 + Index("idx_acteco_entidad_acteco_gestion", "acteco", "gestion"),
60 {"schema": "web"}, 61 {"schema": "web"},
61 ) 62 )
62 63
...@@ -81,6 +82,7 @@ class ActecoUbigeo(Base): ...@@ -81,6 +82,7 @@ class ActecoUbigeo(Base):
81 Index("idx_acteco_ubigeo_gestion", "gestion"), 82 Index("idx_acteco_ubigeo_gestion", "gestion"),
82 Index("idx_acteco_ubigeo_acteco", "acteco"), 83 Index("idx_acteco_ubigeo_acteco", "acteco"),
83 Index("idx_acteco_ubigeo_acteco_entidad", "acteco", "ubigeo"), 84 Index("idx_acteco_ubigeo_acteco_entidad", "acteco", "ubigeo"),
85 + Index("idx_acteco_ubigeo_acteco_gestion", "acteco", "gestion"),
84 {"schema": "web"}, 86 {"schema": "web"},
85 ) 87 )
86 88
......
...@@ -19,6 +19,7 @@ class EntidadDistribuciones(Base): ...@@ -19,6 +19,7 @@ class EntidadDistribuciones(Base):
19 Index("idx_entidad_distribuciones_codigo", "codigo"), 19 Index("idx_entidad_distribuciones_codigo", "codigo"),
20 Index("idx_entidad_distribuciones_dimension", "dimension"), 20 Index("idx_entidad_distribuciones_dimension", "dimension"),
21 Index("idx_entidad_distribuciones_gestion", "gestion"), 21 Index("idx_entidad_distribuciones_gestion", "gestion"),
22 + Index("idx_entidad_distribuciones_compos1", "tipo_codigo", "codigo", "dimension", "gestion"),
22 {"schema": "web"}, 23 {"schema": "web"},
23 ) 24 )
24 25
...@@ -44,11 +45,12 @@ class EntidadResumenes(Base): ...@@ -44,11 +45,12 @@ class EntidadResumenes(Base):
44 Index("idx_entidad_resumenes_tipo_codigo", "tipo_codigo"), 45 Index("idx_entidad_resumenes_tipo_codigo", "tipo_codigo"),
45 Index("idx_entidad_resumenes_codigo", "codigo"), 46 Index("idx_entidad_resumenes_codigo", "codigo"),
46 Index("idx_entidad_resumenes_gestion", "gestion"), 47 Index("idx_entidad_resumenes_gestion", "gestion"),
48 + Index("idx_entidad_resumenes_compos1", "tipo_codigo", "codigo"),
47 {"schema": "web"}, 49 {"schema": "web"},
48 ) 50 )
49 51
50 id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) 52 id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
51 - 53 +
52 tipo: Mapped[str] = mapped_column(String(8)) 54 tipo: Mapped[str] = mapped_column(String(8))
53 tipo_codigo: Mapped[str] = mapped_column(String(16)) 55 tipo_codigo: Mapped[str] = mapped_column(String(16))
54 codigo: Mapped[str] = mapped_column(String(8)) 56 codigo: Mapped[str] = mapped_column(String(8))
......
...@@ -71,6 +71,7 @@ class FinfunEntidad(Base): ...@@ -71,6 +71,7 @@ class FinfunEntidad(Base):
71 Index("idx_finfun_entidad_gestion", "gestion"), 71 Index("idx_finfun_entidad_gestion", "gestion"),
72 Index("idx_finfun_entidad_finfun", "finfun"), 72 Index("idx_finfun_entidad_finfun", "finfun"),
73 Index("idx_finfun_entidad_finfun_entidad", "finfun", "entidad"), 73 Index("idx_finfun_entidad_finfun_entidad", "finfun", "entidad"),
74 + Index("idx_finfun_entidad_finfun_gestion", "finfun", "gestion"),
74 {"schema": "web"}, 75 {"schema": "web"},
75 ) 76 )
76 77
...@@ -95,6 +96,7 @@ class FinfunUbigeo(Base): ...@@ -95,6 +96,7 @@ class FinfunUbigeo(Base):
95 Index("idx_finfun_ubigeo_gestion", "gestion"), 96 Index("idx_finfun_ubigeo_gestion", "gestion"),
96 Index("idx_finfun_ubigeo_finfun", "finfun"), 97 Index("idx_finfun_ubigeo_finfun", "finfun"),
97 Index("idx_finfun_ubigeo_finfun_entidad", "finfun", "ubigeo"), 98 Index("idx_finfun_ubigeo_finfun_entidad", "finfun", "ubigeo"),
99 + Index("idx_finfun_ubigeo_finfun_gestion", "finfun", "gestion"),
98 {"schema": "web"}, 100 {"schema": "web"},
99 ) 101 )
100 102
......
...@@ -72,6 +72,7 @@ class ObjetoEntidad(Base): ...@@ -72,6 +72,7 @@ class ObjetoEntidad(Base):
72 Index("idx_objeto_entidad_gestion", "gestion"), 72 Index("idx_objeto_entidad_gestion", "gestion"),
73 Index("idx_objeto_entidad_objeto", "objeto"), 73 Index("idx_objeto_entidad_objeto", "objeto"),
74 Index("idx_objeto_entidad_objeto_entidad", "objeto", "entidad"), 74 Index("idx_objeto_entidad_objeto_entidad", "objeto", "entidad"),
75 + Index("idx_objeto_entidad_objeto_gestion", "objeto", "gestion"),
75 {"schema": "web"}, 76 {"schema": "web"},
76 ) 77 )
77 78
...@@ -96,6 +97,7 @@ class ObjetoUbigeo(Base): ...@@ -96,6 +97,7 @@ class ObjetoUbigeo(Base):
96 Index("idx_objeto_ubigeo_gestion", "gestion"), 97 Index("idx_objeto_ubigeo_gestion", "gestion"),
97 Index("idx_objeto_ubigeo_objeto", "objeto"), 98 Index("idx_objeto_ubigeo_objeto", "objeto"),
98 Index("idx_objeto_ubigeo_objeto_entidad", "objeto", "ubigeo"), 99 Index("idx_objeto_ubigeo_objeto_entidad", "objeto", "ubigeo"),
100 + Index("idx_objeto_ubigeo_objeto_gestion", "objeto", "gestion"),
99 {"schema": "web"}, 101 {"schema": "web"},
100 ) 102 )
101 103
......
...@@ -57,6 +57,7 @@ class OrganismoEntidad(Base): ...@@ -57,6 +57,7 @@ class OrganismoEntidad(Base):
57 Index("idx_organismo_entidad_gestion", "gestion"), 57 Index("idx_organismo_entidad_gestion", "gestion"),
58 Index("idx_organismo_entidad_objeto", "organismo"), 58 Index("idx_organismo_entidad_objeto", "organismo"),
59 Index("idx_organismo_entidad_rubro_entidad", "organismo", "entidad"), 59 Index("idx_organismo_entidad_rubro_entidad", "organismo", "entidad"),
60 + Index("idx_organismo_entidad_rubro_gestion", "organismo", "gestion"),
60 {"schema": "web"}, 61 {"schema": "web"},
61 ) 62 )
62 63
......
1 +from sqlalchemy.orm import Mapped, mapped_column
2 +from presupuestov1.model import Base
3 +
4 +from sqlalchemy import (
5 + Integer,
6 + String,
7 + Text,
8 + Numeric,
9 + Index,
10 + MetaData,
11 +)
12 +
13 +class ProyectoResumenes(Base):
14 + __tablename__ = "proyecto_resumenes"
15 + __table_args__ = (
16 + Index("idx_proyecto_resumenes_gestion", "gestion"),
17 + Index("idx_proyecto_resumenes_nivel", "nivel"),
18 + Index("idx_proyecto_resumenes_codigo", "codigo"),
19 + Index("idx_proyecto_resumenes_codigo_padre", "codigo_padre"),
20 + {"schema": "web"},
21 + )
22 +
23 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
24 +
25 + gestion: Mapped[int] = mapped_column(Integer)
26 + nivel: Mapped[str] = mapped_column(Text)
27 + codigo: Mapped[str] = mapped_column(String(32))
28 + codigo_padre: Mapped[str | None] = mapped_column(String(32))
29 +
30 + devengado: Mapped[float] = mapped_column(Numeric(30, 15))
31 +
32 + n_entidad: Mapped[int] = mapped_column(Integer)
33 + n_objeto: Mapped[int] = mapped_column(Integer)
34 + n_finfun: Mapped[int] = mapped_column(Integer)
35 + n_acteco: Mapped[int] = mapped_column(Integer)
36 +
37 + desc: Mapped[str] = mapped_column(Text)
38 + desc_padre: Mapped[str | None] = mapped_column(Text)
39 +
40 + entidad: Mapped[str] = mapped_column(Text)
41 + objeto: Mapped[str] = mapped_column(Text)
42 + finfun: Mapped[str] = mapped_column(Text)
43 + acteco: Mapped[str] = mapped_column(Text)
...@@ -57,6 +57,7 @@ class RubroEntidad(Base): ...@@ -57,6 +57,7 @@ class RubroEntidad(Base):
57 Index("idx_rubro_entidad_gestion", "gestion"), 57 Index("idx_rubro_entidad_gestion", "gestion"),
58 Index("idx_rubro_entidad_objeto", "rubro"), 58 Index("idx_rubro_entidad_objeto", "rubro"),
59 Index("idx_rubro_entidad_rubro_entidad", "rubro", "entidad"), 59 Index("idx_rubro_entidad_rubro_entidad", "rubro", "entidad"),
60 + Index("idx_rubro_entidad_rubro_gestion", "rubro", "gestion"),
60 {"schema": "web"}, 61 {"schema": "web"},
61 ) 62 )
62 63
......
...@@ -15,6 +15,8 @@ from sqlalchemy import and_, or_, select ...@@ -15,6 +15,8 @@ from sqlalchemy import and_, or_, select
15 class TreemapObjeto(Base): 15 class TreemapObjeto(Base):
16 __tablename__ = "treemap_objeto" 16 __tablename__ = "treemap_objeto"
17 __table_args__ = ( 17 __table_args__ = (
18 + Index("idx_treemap_objeto_objeto", "objeto"),
19 + Index("idx_treemap_objeto_objeto_gestion", "objeto", "gestion"),
18 Index("idx_treemap_objeto_gestion_entidad", "gestion", "entidad"), 20 Index("idx_treemap_objeto_gestion_entidad", "gestion", "entidad"),
19 Index("idx_treemap_objeto_nivel", "nivel"), 21 Index("idx_treemap_objeto_nivel", "nivel"),
20 {"schema": "web"}, 22 {"schema": "web"},
...@@ -39,6 +41,8 @@ class TreemapObjeto(Base): ...@@ -39,6 +41,8 @@ class TreemapObjeto(Base):
39 class TreemapFinFun(Base): 41 class TreemapFinFun(Base):
40 __tablename__ = "treemap_finfun" 42 __tablename__ = "treemap_finfun"
41 __table_args__ = ( 43 __table_args__ = (
44 + Index("idx_treemap_finfun_finfun", "finfun"),
45 + Index("idx_treemap_finfun_finfun_gestion", "finfun", "gestion"),
42 Index("idx_treemap_finfun_gestion_entidad", "gestion", "entidad"), 46 Index("idx_treemap_finfun_gestion_entidad", "gestion", "entidad"),
43 Index("idx_treemap_finfun_nivel", "nivel"), 47 Index("idx_treemap_finfun_nivel", "nivel"),
44 {"schema": "web"}, 48 {"schema": "web"},
...@@ -63,6 +67,8 @@ class TreemapFinFun(Base): ...@@ -63,6 +67,8 @@ class TreemapFinFun(Base):
63 class TreemapActeco(Base): 67 class TreemapActeco(Base):
64 __tablename__ = "treemap_acteco" 68 __tablename__ = "treemap_acteco"
65 __table_args__ = ( 69 __table_args__ = (
70 + Index("idx_treemap_acteco_acteco", "acteco"),
71 + Index("idx_treemap_acteco_acteco_gestion", "acteco", "gestion"),
66 Index("idx_treemap_acteco_gestion_entidad", "gestion", "entidad"), 72 Index("idx_treemap_acteco_gestion_entidad", "gestion", "entidad"),
67 Index("idx_treemap_acteco_nivel", "nivel"), 73 Index("idx_treemap_acteco_nivel", "nivel"),
68 {"schema": "web"}, 74 {"schema": "web"},
...@@ -87,6 +93,8 @@ class TreemapActeco(Base): ...@@ -87,6 +93,8 @@ class TreemapActeco(Base):
87 class TreemapRubro(Base): 93 class TreemapRubro(Base):
88 __tablename__ = "treemap_rubro" 94 __tablename__ = "treemap_rubro"
89 __table_args__ = ( 95 __table_args__ = (
96 + Index("idx_treemap_rubro_rubro", "rubro"),
97 + Index("idx_treemap_rubro_rubro_gestion", "rubro", "gestion"),
90 Index("idx_treemap_rubro_gestion_entidad", "gestion", "entidad"), 98 Index("idx_treemap_rubro_gestion_entidad", "gestion", "entidad"),
91 Index("idx_treemap_rubro_nivel", "nivel"), 99 Index("idx_treemap_rubro_nivel", "nivel"),
92 {"schema": "web"}, 100 {"schema": "web"},
......
...@@ -194,7 +194,7 @@ def fetch_acteco_ubigeo( ...@@ -194,7 +194,7 @@ def fetch_acteco_ubigeo(
194 query: schema.QueryParams = Depends(), 194 query: schema.QueryParams = Depends(),
195 ): 195 ):
196 gestion = query.gestion or 0 196 gestion = query.gestion or 0
197 - ClasObjetos = models.objeto.ActecoUbigeo 197 + ClasObjetos = models.acteco.ActecoUbigeo
198 198
199 return db.query(ClasObjetos).filter( 199 return db.query(ClasObjetos).filter(
200 ClasObjetos.acteco == acteco_id, 200 ClasObjetos.acteco == acteco_id,
......
...@@ -73,50 +73,74 @@ def fetch_entidades( ...@@ -73,50 +73,74 @@ def fetch_entidades(
73 @router.get('/timeline', response_model=List[schemas.daily.Timeline]) 73 @router.get('/timeline', response_model=List[schemas.daily.Timeline])
74 @not_found 74 @not_found
75 def fetch_timeline( 75 def fetch_timeline(
76 - db: Session = Depends(model.get_db), 76 + db: Session = Depends(model.get_db),
77 - query: schema.DailyParams = Depends(), 77 + query: schema.DailyParams = Depends(),
78 ): 78 ):
79 - Daily = models.daily.Timeline 79 + Daily = models.daily.Timeline
80 - page = query.page or 1 80 + page = query.page or 1
81 - page_size = 10 81 + page_size = 10
82 - 82 +
83 - if query.entidad is not None: 83 + if query.entidad is not None:
84 - return db.query(Daily).filter( 84 + stmt = (
85 - Daily.entidad == query.entidad 85 + select(Daily)
86 - ).order_by( 86 + .where(Daily.entidad == query.entidad)
87 - Daily.devengado.desc() 87 + .order_by(Daily.devengado.desc(), Daily.id)
88 - ).offset( 88 + .offset((page - 1) * page_size)
89 - (page - 1) * page_size 89 + .limit(page_size)
90 - ).limit(page_size).all() 90 + )
91 - 91 + return db.execute(stmt).scalars().all()
92 - # top 10 entidades 92 +
93 - top_entities = db.query( 93 + total_devengado = func.sum(Daily.devengado).label("total_devengado")
94 - Daily.entidad 94 +
95 - ).group_by( 95 + top_entities = (
96 - Daily.entidad 96 + select(
97 - ).order_by( 97 + Daily.entidad.label("entidad"),
98 - func.sum(Daily.devengado).desc() 98 + total_devengado,
99 - ).limit(10).subquery() 99 + )
100 - 100 + .group_by(Daily.entidad)
101 - # ordena las filas por devengado desc por cada una de las top 10 entidades 101 + .order_by(total_devengado.desc(), Daily.entidad)
102 - ranked = db.query( 102 + .limit(10)
103 - Daily, 103 + .subquery()
104 - func.row_number().over( 104 + )
105 - partition_by=Daily.entidad, 105 +
106 - order_by=Daily.devengado.desc() 106 + ranked = (
107 - ).label('row_num') 107 + select(
108 - ).filter( 108 + Daily.entidad.label("entidad"),
109 - Daily.entidad.in_(top_entities) 109 + Daily.devengado.label("devengado"),
110 - ).subquery() 110 + Daily.entidad_desc_area.label("entidad_desc_area"),
111 - 111 + Daily.entidad_desc_entidad.label("entidad_desc_entidad"),
112 - # top 10 filas por cada una de las top 10 entidades 112 + Daily.desc_programa.label("desc_programa"),
113 - RankedTimeline = aliased(Daily, ranked) 113 + Daily.desc_actividad_proyecto.label("desc_actividad_proyecto"),
114 - return db.query(RankedTimeline).filter( 114 + Daily.id.label("id"),
115 - ranked.c.row_num <= page_size 115 + top_entities.c.total_devengado,
116 - ).order_by( 116 + func.row_number().over(
117 - RankedTimeline.entidad, 117 + partition_by=Daily.entidad,
118 - RankedTimeline.devengado.desc() 118 + order_by=(Daily.devengado.desc(), Daily.id),
119 - ).all() 119 + ).label("row_num"),
120 + )
121 + .join(top_entities, top_entities.c.entidad == Daily.entidad)
122 + .subquery()
123 + )
124 +
125 + stmt = (
126 + select(
127 + ranked.c.entidad,
128 + ranked.c.devengado,
129 + ranked.c.entidad_desc_area,
130 + ranked.c.entidad_desc_entidad,
131 + ranked.c.desc_programa,
132 + ranked.c.desc_actividad_proyecto,
133 + )
134 + .where(ranked.c.row_num <= page_size)
135 + .order_by(
136 + ranked.c.total_devengado.desc(),
137 + ranked.c.entidad,
138 + ranked.c.devengado.desc(),
139 + ranked.c.id,
140 + )
141 + )
142 +
143 + return db.execute(stmt).mappings().all()
120 144
121 145
122 @router.get('/timeseries', response_model=List[schemas.daily.TimeSeries]) 146 @router.get('/timeseries', response_model=List[schemas.daily.TimeSeries])
......
...@@ -46,7 +46,6 @@ def fetch_ubigeo_classifier( ...@@ -46,7 +46,6 @@ def fetch_ubigeo_classifier(
46 return db.query(models.classifier.ClasGeografico).all() 46 return db.query(models.classifier.ClasGeografico).all()
47 47
48 48
49 -
50 # base 49 # base
51 50
52 @router_entidad.get('/{entity_id:str}', response_model=schemas.entity.ClasInstitucionalIngresosGastos) 51 @router_entidad.get('/{entity_id:str}', response_model=schemas.entity.ClasInstitucionalIngresosGastos)
......
1 +from fastapi import APIRouter, Query
2 +
3 +from presupuestov1 import metrics
4 +
5 +
6 +WINDOW_MINUTES_MAX = 7 * 24 * 60
7 +
8 +router = APIRouter(
9 + prefix='/_metrics',
10 + include_in_schema=False,
11 +)
12 +
13 +
14 +###############################################################################
15 +# GET
16 +###############################################################################
17 +
18 +@router.get('/summary')
19 +async def fetch_metrics_summary(
20 + minutes: int = Query(default=60, ge=1, le=WINDOW_MINUTES_MAX),
21 + include_ts: bool = Query(default=False),
22 +):
23 + return await metrics.read_summary(minutes, include_ts)
24 +
25 +
26 +@router.get('/routes')
27 +async def fetch_metrics_routes(
28 + minutes: int = Query(default=60, ge=1, le=WINDOW_MINUTES_MAX),
29 + include_ts: bool = Query(default=False),
30 +):
31 + return await metrics.read_routes(minutes, include_ts)
32 +
33 +
34 +@router.get('/ips')
35 +async def fetch_metrics_ips(
36 + minutes: int = Query(default=60, ge=1, le=WINDOW_MINUTES_MAX),
37 + include_ts: bool = Query(default=False),
38 +):
39 + return await metrics.read_ips(minutes, include_ts)
1 -import time 1 +import pydantic_core
2 2
3 from typing import List, Any, Optional, Union 3 from typing import List, Any, Optional, Union
4 4
5 from fastapi import APIRouter, Depends, Request, HTTPException, Body 5 from fastapi import APIRouter, Depends, Request, HTTPException, Body
6 from sqlalchemy.orm import Session 6 from sqlalchemy.orm import Session
7 +from sqlalchemy import and_, or_, select, func, desc
7 8
8 from presupuestov1 import model, schema, constant, schemas, models, search 9 from presupuestov1 import model, schema, constant, schemas, models, search
9 from presupuestov1.routers._decorators import not_found 10 from presupuestov1.routers._decorators import not_found
...@@ -19,23 +20,55 @@ router = APIRouter( ...@@ -19,23 +20,55 @@ router = APIRouter(
19 # GET 20 # GET
20 ############################################################################### 21 ###############################################################################
21 22
22 -@router.get('/{project_id}') 23 +@router.get('/{project_id}', response_model=List[schemas.project.ProyectoResumen])
23 @not_found 24 @not_found
24 def fetch_programa_proyecto( 25 def fetch_programa_proyecto(
25 - project_id: int, 26 + project_id: str,
26 db: Session = Depends(model.get_db), 27 db: Session = Depends(model.get_db),
27 ): 28 ):
28 - return [] 29 + ProyectoResumenes = models.project.ProyectoResumenes
30 + project_items = db.query(ProyectoResumenes).filter(
31 + ProyectoResumenes.codigo == project_id
32 + ).all()
33 +
34 + # TODO if devengado > mucho_dinero: cache
35 +
36 + for _ in ['entidad', 'objeto', 'finfun', 'acteco']:
37 + for __ in project_items:
38 + obj_ = getattr(__, _)
39 + setattr(__, _, pydantic_core.from_json(obj_))
40 +
41 + return project_items
29 42
30 43
31 @router.get('/{project_id}/proyectos') 44 @router.get('/{project_id}/proyectos')
32 @not_found 45 @not_found
33 def fetch_programa_proyecto_proyectos( 46 def fetch_programa_proyecto_proyectos(
34 - project_id: int, 47 + project_id: str,
35 db: Session = Depends(model.get_db), 48 db: Session = Depends(model.get_db),
36 - query: schema.QueryParams = Depends(), 49 + query: schema.SearchQueryParams = Depends(),
37 ): 50 ):
38 - return [] 51 + page = query.page or 1
52 + page_size = 20
53 +
54 + ProyectoResumenes = models.project.ProyectoResumenes
55 + return db.execute(
56 + select(
57 + ProyectoResumenes.nivel,
58 + ProyectoResumenes.codigo,
59 + ProyectoResumenes.desc,
60 + func.sum(ProyectoResumenes.devengado).label('devengado'),
61 + ).where(
62 + ProyectoResumenes.codigo_padre == project_id,
63 + ProyectoResumenes.nivel != 'programa',
64 + ).group_by(
65 + ProyectoResumenes.nivel,
66 + ProyectoResumenes.codigo,
67 + ProyectoResumenes.desc,
68 + ).order_by(desc('devengado')).offset(
69 + (page - 1) * page_size
70 + ).limit(page_size)
71 + ).mappings().all()
39 72
40 73
41 @router.get('/{project_id}/objetos') 74 @router.get('/{project_id}/objetos')
......
1 from typing import List, Any, Optional, Union 1 from typing import List, Any, Optional, Union
2 2
3 -from fastapi import APIRouter, Depends, Request, HTTPException, Body 3 +from fastapi import APIRouter, Depends, HTTPException
4 from sqlalchemy.orm import Session 4 from sqlalchemy.orm import Session
5 5
6 from presupuestov1 import model, schema, constant, schemas, models, search 6 from presupuestov1 import model, schema, constant, schemas, models, search
...@@ -28,6 +28,13 @@ def fetch_search( ...@@ -28,6 +28,13 @@ def fetch_search(
28 'hits': [], 28 'hits': [],
29 } 29 }
30 30
31 - return search.do_search( 31 + try:
32 - query.q, query.page, query.is_class, query.class_, query.order 32 + return search.do_search(
33 - ) 33 + query.q, query.page, query.is_class, query.class_, query.order
34 + )
35 + except search.SearchUnavailable as exc:
36 + raise HTTPException(
37 + status_code=503,
38 + detail='Search temporarily unavailable',
39 + headers={'Retry-After': str(exc.retry_after)},
40 + ) from exc
......
...@@ -19,14 +19,14 @@ class ActecoEstado(OrmBaseModel): ...@@ -19,14 +19,14 @@ class ActecoEstado(OrmBaseModel):
19 top1_entidad_desc: str | None 19 top1_entidad_desc: str | None
20 top1_monto: float | None 20 top1_monto: float | None
21 top1_pct: float | None 21 top1_pct: float | None
22 - top2_entidad: int | None | float 22 + top2_entidad: int | None | float | str
23 top2_entidad_desc: str | None 23 top2_entidad_desc: str | None
24 - top2_monto: float | None 24 + top2_monto: float | None | str
25 - top2_pct: float | None 25 + top2_pct: float | None | str
26 - top3_entidad: int | None | float 26 + top3_entidad: int | None | float | str
27 top3_entidad_desc: str | None 27 top3_entidad_desc: str | None
28 - top3_monto: float | None 28 + top3_monto: float | None | str
29 - top3_pct: float | None 29 + top3_pct: float | None | str
30 30
31 31
32 class ActecoEntidad(OrmBaseModel): 32 class ActecoEntidad(OrmBaseModel):
......
...@@ -26,14 +26,14 @@ class FinfunEstado(OrmBaseModel): ...@@ -26,14 +26,14 @@ class FinfunEstado(OrmBaseModel):
26 top1_entidad_desc: str | None 26 top1_entidad_desc: str | None
27 top1_monto: float | None 27 top1_monto: float | None
28 top1_pct: float | None 28 top1_pct: float | None
29 - top2_entidad: int | None | float 29 + top2_entidad: int | None | float | str
30 top2_entidad_desc: str | None 30 top2_entidad_desc: str | None
31 - top2_monto: float | None 31 + top2_monto: float | None | str
32 - top2_pct: float | None 32 + top2_pct: float | None | str
33 - top3_entidad: int | None | float 33 + top3_entidad: int | None | float | str
34 top3_entidad_desc: str | None 34 top3_entidad_desc: str | None
35 - top3_monto: float | None 35 + top3_monto: float | None | str
36 - top3_pct: float | None 36 + top3_pct: float | None | str
37 37
38 38
39 class FinfunEntidad(OrmBaseModel): 39 class FinfunEntidad(OrmBaseModel):
......
...@@ -26,14 +26,14 @@ class ObjetoEstado(OrmBaseModel): ...@@ -26,14 +26,14 @@ class ObjetoEstado(OrmBaseModel):
26 top1_entidad_desc: str | None 26 top1_entidad_desc: str | None
27 top1_monto: float | None 27 top1_monto: float | None
28 top1_pct: float | None 28 top1_pct: float | None
29 - top2_entidad: int | None 29 + top2_entidad: int | None | str
30 top2_entidad_desc: str | None 30 top2_entidad_desc: str | None
31 - top2_monto: float | None 31 + top2_monto: float | None | str
32 - top2_pct: float | None 32 + top2_pct: float | None | str
33 - top3_entidad: int | None 33 + top3_entidad: int | None | str
34 top3_entidad_desc: str | None 34 top3_entidad_desc: str | None
35 - top3_monto: float | None 35 + top3_monto: float | None | str
36 - top3_pct: float | None 36 + top3_pct: float | None | str
37 37
38 38
39 class ObjetoEntidad(OrmBaseModel): 39 class ObjetoEntidad(OrmBaseModel):
......
...@@ -20,14 +20,14 @@ class OrganismoEstado(OrmBaseModel): ...@@ -20,14 +20,14 @@ class OrganismoEstado(OrmBaseModel):
20 top1_entidad_desc: str | None 20 top1_entidad_desc: str | None
21 top1_monto: float | None 21 top1_monto: float | None
22 top1_pct: float | None 22 top1_pct: float | None
23 - top2_entidad: int | None | float 23 + top2_entidad: int | None | float | str
24 top2_entidad_desc: str | None 24 top2_entidad_desc: str | None
25 - top2_monto: float | None 25 + top2_monto: float | None | str
26 - top2_pct: float | None 26 + top2_pct: float | None | str
27 - top3_entidad: int | None | float 27 + top3_entidad: int | None | float | str
28 top3_entidad_desc: str | None 28 top3_entidad_desc: str | None
29 - top3_monto: float | None 29 + top3_monto: float | None | str
30 - top3_pct: float | None 30 + top3_pct: float | None | str
31 31
32 32
33 class OrganismoEntidad(OrmBaseModel): 33 class OrganismoEntidad(OrmBaseModel):
......
1 -from typing import List, Any, Dict 1 +from typing import List
2 2
3 -from presupuestov1.schema import BaseModel 3 +from presupuestov1.schema import BaseModel, OrmBaseModel
4 +
5 +
6 +class ProyectoResumenObj(BaseModel):
7 + codigo: int | str
8 + desc: str | None
9 + desc_padre: str | None
10 + devengado: float
11 +
12 +class ProyectoResumen(OrmBaseModel):
13 + gestion: int
14 + codigo: str
15 + codigo_padre: str | None
16 + devengado: float
17 + n_entidad: int
18 + n_objeto: int
19 + n_finfun: int
20 + n_acteco: int
21 + desc: str
22 + desc_padre: str | None
23 + entidad: str | List[ProyectoResumenObj]
24 + objeto: str | List[ProyectoResumenObj]
25 + finfun: str | List[ProyectoResumenObj]
26 + acteco: str | List[ProyectoResumenObj]
27 +
28 +
29 +class ProyectoProyectos(OrmBaseModel):
30 + nivel: str
31 + codigo: str
32 + desc: str
4 33
5 34
6 class ProjectSearch(BaseModel): 35 class ProjectSearch(BaseModel):
......
...@@ -20,14 +20,14 @@ class RubroEstado(OrmBaseModel): ...@@ -20,14 +20,14 @@ class RubroEstado(OrmBaseModel):
20 top1_entidad_desc: str | None 20 top1_entidad_desc: str | None
21 top1_monto: float | None 21 top1_monto: float | None
22 top1_pct: float | None 22 top1_pct: float | None
23 - top2_entidad: int | None | float 23 + top2_entidad: int | None | float | str
24 top2_entidad_desc: str | None 24 top2_entidad_desc: str | None
25 - top2_monto: float | None 25 + top2_monto: float | None | str
26 - top2_pct: float | None 26 + top2_pct: float | None | str
27 - top3_entidad: int | None | float 27 + top3_entidad: int | None | float | str
28 top3_entidad_desc: str | None 28 top3_entidad_desc: str | None
29 - top3_monto: float | None 29 + top3_monto: float | None | str
30 - top3_pct: float | None 30 + top3_pct: float | None | str
31 31
32 32
33 class RubroEntidad(OrmBaseModel): 33 class RubroEntidad(OrmBaseModel):
......
1 +import math
2 +import threading
3 +import time
4 +
1 import typesense 5 import typesense
6 +from typesense import exceptions as typesense_exceptions
2 7
3 TYPESENSE_API = open('./docs/typesense').read().strip() 8 TYPESENSE_API = open('./docs/typesense').read().strip()
4 client = typesense.Client({ 9 client = typesense.Client({
...@@ -8,7 +13,7 @@ client = typesense.Client({ ...@@ -8,7 +13,7 @@ client = typesense.Client({
8 'port': '8108', 13 'port': '8108',
9 'protocol': 'http' 14 'protocol': 'http'
10 }], 15 }],
11 - 'connection_timeout_seconds': 2 16 + 'connection_timeout_seconds': 0.5
12 }) 17 })
13 18
14 COLLECTION = 'presup_actpro_fts' 19 COLLECTION = 'presup_actpro_fts'
...@@ -34,9 +39,70 @@ search_parameters = { ...@@ -34,9 +39,70 @@ search_parameters = {
34 39
35 40
36 AVAILABLE_CLASSES = [ 41 AVAILABLE_CLASSES = [
37 - 'programa_proyecto', 'entidad', 'objeto', 'finfun', 'acteco' 42 + 'programa_proyecto',
43 + 'entidad',
44 + 'objeto',
45 + 'finfun',
46 + 'acteco',
47 + 'ubigeo',
48 + 'rubro',
49 + 'organismo',
38 ] 50 ]
51 +
52 +BACKEND_ERRORS = (
53 + typesense_exceptions.Timeout,
54 + typesense_exceptions.ServiceUnavailable,
55 + typesense_exceptions.ServerError,
56 + typesense_exceptions.HTTPStatus0Error,
57 + OSError,
58 +)
59 +FAILURE_THRESHOLD = 2
60 +OPEN_INTERVAL_SECONDS = 15.0
61 +
62 +
63 +class SearchUnavailable(Exception):
64 + def __init__(self, retry_after: int):
65 + self.retry_after = retry_after
66 + super().__init__('search backend unavailable')
67 +
68 +
69 +class SearchGuard:
70 + def __init__(self):
71 + self._lock = threading.Lock()
72 + self._failures = 0
73 + self._opened_until = 0.0
74 +
75 + def enter(self):
76 + now = time.monotonic()
77 +
78 + with self._lock:
79 + if self._opened_until > now:
80 + raise SearchUnavailable(max(1, math.ceil(self._opened_until - now)))
81 +
82 + def record_success(self):
83 + with self._lock:
84 + self._failures = 0
85 + self._opened_until = 0.0
86 +
87 + def record_failure(self):
88 + now = time.monotonic()
89 +
90 + with self._lock:
91 + self._failures += 1
92 +
93 + if self._failures >= FAILURE_THRESHOLD:
94 + self._opened_until = now + OPEN_INTERVAL_SECONDS
95 + return int(OPEN_INTERVAL_SECONDS)
96 +
97 + return 1
98 +
99 +
100 +search_guard = SearchGuard()
101 +
102 +
39 def do_search(query, page, is_class, class_=None, order=None): 103 def do_search(query, page, is_class, class_=None, order=None):
104 + search_guard.enter()
105 +
40 search_parameters_ = search_parameters.copy() 106 search_parameters_ = search_parameters.copy()
41 search_parameters_['q'] = query 107 search_parameters_['q'] = query
42 108
...@@ -60,6 +126,12 @@ def do_search(query, page, is_class, class_=None, order=None): ...@@ -60,6 +126,12 @@ def do_search(query, page, is_class, class_=None, order=None):
60 if order == 'asc': 126 if order == 'asc':
61 search_parameters_['sort_by'] = 'devengado:asc' 127 search_parameters_['sort_by'] = 'devengado:asc'
62 128
63 - return client.collections[COLLECTION].documents.search( 129 + try:
64 - search_parameters_ 130 + response = client.collections[COLLECTION].documents.search(
65 - ) 131 + search_parameters_
132 + )
133 + except BACKEND_ERRORS:
134 + raise SearchUnavailable(search_guard.record_failure())
135 +
136 + search_guard.record_success()
137 + return response
......