From f8383908bdf7045347e3065ad22f1490c5b68e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 8 Jun 2026 18:34:26 -0300 Subject: [PATCH 1/4] feat: improve ducklake api struture feat: global variables usage feat: include more orm models to manage .duckdb files --- docker/Dockerfile => Dockerfile | 4 +- ...docker-compose.yaml => docker-compose.yaml | 4 +- docker/notebooks/Welcome.ipynb | 89 - docker/scripts/poetry-install.sh | 6 - docker/scripts/entrypoint.sh => entrypoint.sh | 0 pysus/api/client.py | 16 +- pysus/api/dadosgov/models.py | 18 +- pysus/api/ducklake/catalog/adapters.py | 176 + pysus/api/ducklake/catalog/columns.py | 7235 ----------------- pysus/api/ducklake/catalog/orm/columns.py | 22 + pysus/api/ducklake/catalog/orm/dataset.py | 294 +- pysus/api/ducklake/catalog/orm/default.py | 3 + pysus/api/ducklake/catalog/parsers.py | 0 pysus/api/ducklake/client.py | 409 +- pysus/api/ducklake/functional.py | 179 + pysus/api/ducklake/models.py | 224 +- pysus/api/ftp/models.py | 2 +- pysus/api/models.py | 2 +- pysus/api/types.py | 21 + pysus/tests/api/ducklake/test_client.py | 68 +- 20 files changed, 565 insertions(+), 8207 deletions(-) rename docker/Dockerfile => Dockerfile (89%) rename docker/docker-compose.yaml => docker-compose.yaml (83%) delete mode 100644 docker/notebooks/Welcome.ipynb delete mode 100644 docker/scripts/poetry-install.sh rename docker/scripts/entrypoint.sh => entrypoint.sh (100%) create mode 100644 pysus/api/ducklake/catalog/adapters.py delete mode 100644 pysus/api/ducklake/catalog/columns.py create mode 100644 pysus/api/ducklake/catalog/orm/columns.py delete mode 100644 pysus/api/ducklake/catalog/parsers.py create mode 100644 pysus/api/ducklake/functional.py diff --git a/docker/Dockerfile b/Dockerfile similarity index 89% rename from docker/Dockerfile rename to Dockerfile index 5956df38..2e37f77b 100644 --- a/docker/Dockerfile +++ b/Dockerfile @@ -25,12 +25,10 @@ RUN useradd -ms /bin/bash pysus \ COPY pyproject.toml poetry.lock LICENSE README.md /usr/src/ COPY pysus /usr/src/pysus -COPY docker/scripts/entrypoint.sh /entrypoint.sh -COPY docker/notebooks/ /home/pysus/Notebooks/ +COPY entrypoint.sh /entrypoint.sh RUN pip install poetry \ && cd /usr/src && poetry config virtualenvs.create false && poetry install --with docs \ - && pip install 'httpx<0.28' \ && chown -R pysus:pysus /home/pysus USER pysus diff --git a/docker/docker-compose.yaml b/docker-compose.yaml similarity index 83% rename from docker/docker-compose.yaml rename to docker-compose.yaml index a854abe4..a9429ff5 100644 --- a/docker/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,8 +1,8 @@ services: jupyter: build: - context: ".." - dockerfile: docker/Dockerfile + context: "." + dockerfile: Dockerfile hostname: pysus-jupyter container_name: pysus-jupyter ports: diff --git a/docker/notebooks/Welcome.ipynb b/docker/notebooks/Welcome.ipynb deleted file mode 100644 index c1d645a3..00000000 --- a/docker/notebooks/Welcome.ipynb +++ /dev/null @@ -1,89 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Welcome to PySUS\n", - "\n", - "PySUS provides tools for dealing with Brazil's public health data (SINAN, SINASC, SIM, SIH, SIA, PNI, IBGE, CNES, CIHA).\n", - "\n", - "## Quick start\n", - "\n", - "List available datasets and files:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pysus\n", - "\n", - "pysus.list_files()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Fetch SINAN data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pysus import sinan\n", - "\n", - "df = sinan(\"deng\", year=2023)\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Fetch SINASC data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pysus import sinasc\n", - "\n", - "df = sinasc(state=\"RJ\", year=2023)\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Documentation\n", - "\n", - "- [PySUS GitHub](https://github.com/InfoDengue/PySUS)\n", - "- [PySUS Docs](https://pysus.readthedocs.io/)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12.0" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docker/scripts/poetry-install.sh b/docker/scripts/poetry-install.sh deleted file mode 100644 index 0499a555..00000000 --- a/docker/scripts/poetry-install.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -poetry config virtualenvs.create false -poetry install --with docs diff --git a/docker/scripts/entrypoint.sh b/entrypoint.sh similarity index 100% rename from docker/scripts/entrypoint.sh rename to entrypoint.sh diff --git a/pysus/api/client.py b/pysus/api/client.py index 28d486ea..95cf8f06 100644 --- a/pysus/api/client.py +++ b/pysus/api/client.py @@ -322,9 +322,9 @@ async def download( if timeout is not None: with anyio.fail_after(timeout): - await client._download_file(file, local_path, callback) + await client.download(file, local_path, callback) else: - await client._download_file(file, local_path, callback) + await client.download(file, local_path, callback) await self._update_state( local_path=local_path, @@ -517,9 +517,7 @@ async def query( all_datasets = await self._ducklake.datasets() if dataset: - matching = [ - d for d in all_datasets if d.name.lower() == dataset.lower() - ] + matching = [d for d in all_datasets if d.name.lower() == dataset.lower()] if not matching: return [] target = matching[0] @@ -618,9 +616,7 @@ def get_columns(path: Path) -> set[tuple[str, str]]: else: paths_str = ", ".join(f"'{p}'" for p in paths) - query = ( - f"SELECT * FROM read_parquet([{paths_str}], union_by_name=True)" - ) + query = f"SELECT * FROM read_parquet([{paths_str}], union_by_name=True)" if sql: if sql.upper().startswith("SELECT"): @@ -633,9 +629,7 @@ def get_columns(path: Path) -> set[tuple[str, str]]: if not add_dv: return base - geocode_cols = [ - col[0] for col in base.description if is_geocode_column(col[0]) - ] + geocode_cols = [col[0] for col in base.description if is_geocode_column(col[0])] if not geocode_cols: return base diff --git a/pysus/api/dadosgov/models.py b/pysus/api/dadosgov/models.py index bbe1d654..dab3d286 100644 --- a/pysus/api/dadosgov/models.py +++ b/pysus/api/dadosgov/models.py @@ -30,13 +30,9 @@ def _dedup_entries( if m: stem = filename[: m.start()] fmt = m.group(1).lower() - grouped.setdefault(stem, []).append( - (fmt, filename, recurso, metadata) - ) + grouped.setdefault(stem, []).append((fmt, filename, recurso, metadata)) else: - grouped.setdefault(filename, []).append( - ("", filename, recurso, metadata) - ) + grouped.setdefault(filename, []).append(("", filename, recurso, metadata)) result: list[tuple[str, Any, dict]] = [] for _, items in grouped.items(): @@ -210,7 +206,7 @@ async def _download( """Download the file to a local path.""" if not output: output = CACHEPATH / self.name - return await self.client._download_file(self, output, callback=callback) + return await self.client.download(self, output, callback=callback) async def fetch_size(self) -> int: """Fetch the remote file size and update the local record. @@ -249,9 +245,7 @@ class Group(BaseRemoteGroup): """A group of files within a dataset.""" record: ConjuntoDados - _formatter: Callable[[str], dict[str, Any]] | None = PrivateAttr( - default=None - ) + _formatter: Callable[[str], dict[str, Any]] | None = PrivateAttr(default=None) def __init__( self, @@ -319,9 +313,7 @@ async def _fetch_files(self) -> list[BaseRemoteFile]: """Build File objects from the underlying resources.""" entries: list[tuple[str, Any, dict]] = [] for recurso in self.record.resources: - filename = ( - recurso.file_name or recurso.url.split("/")[-1].split("?")[0] - ) + filename = recurso.file_name or recurso.url.split("/")[-1].split("?")[0] if filename.lower().endswith(".pdf") or filename.startswith("get_"): continue metadata = {} diff --git a/pysus/api/ducklake/catalog/adapters.py b/pysus/api/ducklake/catalog/adapters.py new file mode 100644 index 00000000..eb16883b --- /dev/null +++ b/pysus/api/ducklake/catalog/adapters.py @@ -0,0 +1,176 @@ +from abc import ABC +from pathlib import Path + +import httpx +from anyio import to_thread +from pydantic import BaseModel, SecretStr +from sqlalchemy.engine import Engine +from sqlalchemy import create_engine +from sqlalchemy.pool import StaticPool +from sqlalchemy.orm import sessionmaker, Session + +from pysus import CACHEPATH +from pysus.api import types +from pysus.api.ducklake.functional import download_s3, upload_s3 + + +class DuckLakeCredentials(BaseModel): + access_key: SecretStr + secret_key: SecretStr + + +class BaseAdapter(ABC): + cache_dir: Path = Path(CACHEPATH) / "ducklake" + db_local: Path + db_remote: Path + + def __init__( + self, engine=None, credentials: DuckLakeCredentials | None = None, **data + ) -> None: + self._engine = engine + self._session_factory = None + self.cache_dir.mkdir(parents=True, exist_ok=True) + self.credentials = credentials + + @property + def remote_url(self) -> str: + return f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{self.db_remote}" + + def get_session(self) -> Session: + if not self._session_factory: + raise RuntimeError("Database engine not initialized. Call connect() first.") + return self._session_factory() + + async def connect(self, force: bool = False) -> None: + if self._engine and not force: + if not self._session_factory: + self._session_factory = sessionmaker(bind=self._engine) + return + + await self._download_catalog( + self.db_local, + str(self.db_remote), + ) + self._engine = await to_thread.run_sync(self.setup_engine) + self._session_factory = sessionmaker(bind=self._engine) + + def setup_engine( + self, access_key: str | None = None, secret_key: str | None = None + ) -> Engine: + engine: Engine = create_engine( + f"duckdb:///{self.db_local}", + poolclass=StaticPool, + ) + + with engine.connect() as conn: + conn.exec_driver_sql("INSTALL ducklake; LOAD ducklake;") + + has_pysus = conn.exec_driver_sql( + "SELECT 1 FROM information_schema.schemata WHERE schema_name = 'pysus'" + ).fetchone() + + if has_pysus: + conn.exec_driver_sql("SET search_path='pysus,main';") + else: + conn.exec_driver_sql("SET search_path='main';") + + s3_cfg = { + "s3_endpoint": types.S3_ENDPOINT, + "s3_region": types.S3_REGION, + "s3_url_style": "path", + "s3_use_ssl": "true", + } + + if access_key and secret_key: + s3_cfg["s3_access_key_id"] = access_key + s3_cfg["s3_secret_access_key"] = secret_key + + for key, value in s3_cfg.items(): + conn.exec_driver_sql(f"SET {key}='{value}'") + + conn.commit() + + return engine + + async def _download_catalog(self, local_path: Path, remote_path: str) -> None: + url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote_path}" + + if local_path.exists(): + try: + local_size = local_path.stat().st_size + except OSError: + local_size = -1 + else: + local_size = -1 + + async with httpx.AsyncClient(follow_redirects=True) as client: + try: + head = await client.head(url) + head.raise_for_status() + remote_size = int(head.headers.get("content-length", 0)) + except Exception: + remote_size = 0 + + if remote_size == local_size: + return + + access_key = ( + self.credentials.access_key.get_secret_value() if self.credentials else None + ) + secret_key = ( + self.credentials.secret_key.get_secret_value() if self.credentials else None + ) + + await download_s3( + remote_path=remote_path, + local_path=local_path, + access_key=access_key, + secret_key=secret_key, + ) + + async def _upload_catalog(self) -> None: + if not self.credentials: + raise PermissionError( + "Admin credentials required to upload catalog.", + ) + + if not self.db_local.exists(): + raise FileNotFoundError("catalog file not found") + + await upload_s3( + local_path=self.db_local, + remote_path=str(self.db_remote), + access_key=self.credentials.access_key.get_secret_value(), + secret_key=self.credentials.secret_key.get_secret_value(), + ) + + async def close(self, update: bool = False) -> None: + if update: + await self._upload_catalog() + + if self._engine: + await to_thread.run_sync(self._engine.dispose) + self._engine = None + self._session_factory = None + + +class CatalogAdapter(BaseAdapter): + def __init__(self, engine=None, **data) -> None: + super().__init__(engine=engine, **data) + self.db_local: Path = self.cache_dir / "catalog.duckdb" + self.db_remote: str = "public/catalog.duckdb" + + +class DatasetAdapter(BaseAdapter): + def __init__(self, name: str, engine=None, **data) -> None: + super().__init__(engine=engine, **data) + self.dataset_name: str = name + self.db_local: Path = self.cache_dir / f"catalog_{name}.duckdb" + self.db_remote: str = f"datasets/catalog_{name}.duckdb" + + +class ColumnsAdapter(BaseAdapter): + def __init__(self, engine=None, **data) -> None: + super().__init__(engine=engine, **data) + self.db_local: Path = self.cache_dir / "columns.duckdb" + self.db_remote: str = "public/columns.duckdb" diff --git a/pysus/api/ducklake/catalog/columns.py b/pysus/api/ducklake/catalog/columns.py deleted file mode 100644 index b7cd3751..00000000 --- a/pysus/api/ducklake/catalog/columns.py +++ /dev/null @@ -1,7235 +0,0 @@ -"""Catalog column definitions extracted from catalog.db. - -Maps every column name to a dict of {dataset_name: description}. -""" - -ABAND = {"pni": ""} - -ABDOMINAL = {"sinan": ""} - -ABRANDAD = {"cnes": ""} - -AB_ANOACOM = {"sia": ""} - -AB_DTCIRG2 = {"sia": ""} - -AB_DTCIRUR = {"sia": ""} - -AB_IMC = {"sia": ""} - -AB_MESACOM = {"sia": ""} - -AB_NUMAIH = {"sia": ""} - -AB_NUMAIH2 = {"sia": ""} - -AB_PONTBAR = {"sia": ""} - -AB_PRCAIH2 = {"sia": ""} - -AB_PRCAIH3 = {"sia": ""} - -AB_PRCAIH4 = {"sia": ""} - -AB_PRCAIH5 = {"sia": ""} - -AB_PRCAIH6 = {"sia": ""} - -AB_PROCAIH = {"sia": ""} - -AB_TABBARR = {"sia": ""} - -AB_T_PRC2 = {"sia": ""} - -AB_T_PRC3 = {"sia": ""} - -AB_T_PRC4 = {"sia": ""} - -AB_T_PRC5 = {"sia": ""} - -AB_T_PRC6 = {"sia": ""} - -ACF_ARTDIA = {"sia": ""} - -ACF_DUPLEX = {"sia": ""} - -ACF_FLEBIT = {"sia": ""} - -ACF_FREMIT = {"sia": ""} - -ACF_HEMATO = {"sia": ""} - -ACF_PREFAV = {"sia": ""} - -ACF_PULSO = {"sia": ""} - -ACF_USOCAT = {"sia": ""} - -ACF_VEIAVI = {"sia": ""} - -ACF_VEIDIA = {"sia": ""} - -ACIDO_PEPT = {"sinan": ""} - -ACIDTRAB = {"sim": ""} - -ACONDIC = {"sinan": ""} - -ACUPUNTURA = {"sinan": ""} - -AEROFOBIA = {"sinan": ""} - -AFASTAMENT = {"sinan": ""} - -AFAST_DESG = {"sinan": ""} - -AFAST_RISC = {"sinan": ""} - -AFAST_TRAB = {"sinan": ""} - -AFIRMATIVO = {"sinan": ""} - -AGENTE = {"sinan": ""} - -AGENTE_1 = {"sinan": ""} - -AGENTE_2 = {"sinan": ""} - -AGENTE_3 = {"sinan": ""} - -AGENTE_DES = {"sinan": ""} - -AGENTE_ET0 = {"sinan": ""} - -AGENTE_ET1 = {"sinan": ""} - -AGENTE_ET2 = {"sinan": ""} - -AGENTE_ET3 = {"sinan": ""} - -AGENTE_ETI = {"sinan": ""} - -AGENTE_OUT = {"sinan": ""} - -AGENTE_TOX = {"sinan": ""} - -AGHBE = {"sinan": ""} - -AGHBS = {"sinan": ""} - -AGITACAO = {"sinan": ""} - -AGRAVAIDS = {"sinan": ""} - -AGRAVALCOO = {"sinan": ""} - -AGRAVDIABE = {"sinan": ""} - -AGRAVDOENC = {"sinan": ""} - -AGRAVDROGA = {"sinan": ""} - -AGRAVOUTDE = {"sinan": ""} - -AGRAVOUTRA = {"sinan": ""} - -AGRAVO_DES = {"sinan": ""} - -AGRAVTABAC = {"sinan": ""} - -AGRESSIVI = {"sinan": ""} - -AGUA_ALIME = {"sinan": ""} - -AG_AMEACA = {"sinan": ""} - -AG_CORTE = {"sinan": ""} - -AG_ENFOR = {"sinan": ""} - -AG_ENVEN = {"sinan": ""} - -AG_ESPEC = {"sinan": ""} - -AG_FOGO = {"sinan": ""} - -AG_FORCA = {"sinan": ""} - -AG_OBJETO = {"sinan": ""} - -AG_OUTROS = {"sinan": ""} - -AG_QUENTE = {"sinan": ""} - -AIH = {"sih": ""} - -ALCATRAO = {"sinan": ""} - -ALCOOL = {"sinan": ""} - -ALIMENTO_C = {"sinan": ""} - -ALRM_ABDOM = {"sinan": ""} - -ALRM_HEMAT = {"sinan": ""} - -ALRM_HEPAT = {"sinan": ""} - -ALRM_HIPOT = {"sinan": ""} - -ALRM_LETAR = {"sinan": ""} - -ALRM_LIQ = {"sinan": ""} - -ALRM_PLAQ = {"sinan": ""} - -ALRM_SANG = {"sinan": ""} - -ALRM_VOM = {"sinan": ""} - -ALTCAUSA = {"sim": ""} - -ALVARA = {"cnes": ""} - -AMALARIA = {"sinan": ""} - -AMBIENTE = {"sinan": ""} - -AMB_NSUS = {"cnes": ""} - -AMB_SUS = {"cnes": ""} - -AMINA = {"sinan": ""} - -AMOS_OUT = {"sinan": ""} - -AMOS_PCR = {"sinan": ""} - -AMPICILINA = {"sinan": ""} - -AMPOLAS = {"sinan": ""} - -AMP_ACEVAS = {"sia": ""} - -AMP_ALBUMI = {"sia": ""} - -AMP_CARACT = {"sia": ""} - -AMP_DTCLI = {"sia": ""} - -AMP_DTINI = {"sia": ""} - -AMP_FOSFOR = {"sia": ""} - -AMP_HB = {"sia": ""} - -AMP_HBSAG = {"sia": ""} - -AMP_HCV = {"sia": ""} - -AMP_HIV = {"sia": ""} - -AMP_INTERC = {"sia": ""} - -AMP_KTVSEM = {"sia": ""} - -AMP_MAISNE = {"sia": ""} - -AMP_PTH = {"sia": ""} - -AMP_SEAPTO = {"sia": ""} - -AMP_SEPERI = {"sia": ""} - -AMP_SITINI = {"sia": ""} - -AMP_SITTRA = {"sia": ""} - -AMP_TRU = {"sia": ""} - -AM_ALTURA = {"sia": ""} - -AM_GESTANT = {"sia": ""} - -AM_PESO = {"sia": ""} - -AM_QTDTRAN = {"sia": ""} - -AM_SANGUE = {"sinan": ""} - -AM_TRANSPL = {"sia": ""} - -ANIMAL = {"sinan": ""} - -ANIM_ESP = {"sinan": ""} - -ANI_ARANHA = {"sinan": ""} - -ANI_LAGART = {"sinan": ""} - -ANI_SERPEN = {"sinan": ""} - -ANI_TIPO_1 = {"sinan": ""} - -ANO = { - "ibge": "", - "pni": "", - "sih": "", - "sinan": "", -} - -ANOMES = {"pni": ""} - -ANOREXIA = {"sinan": ""} - -ANO_CMPT = { - "ciha": "", - "sih": "", -} - -ANO_DT_SIN = {"sinan": ""} - -ANO_NASC = {"sinan": ""} - -ANTDTTRANS = {"sinan": ""} - -ANTEC_POS = {"sinan": ""} - -ANTEC_PRE = {"sinan": ""} - -ANTIBIOTIC = {"sinan": ""} - -ANTIB_DES = {"sinan": ""} - -ANTIHAVIGM = {"sinan": ""} - -ANTIHBCIGM = {"sinan": ""} - -ANTIHBE = {"sinan": ""} - -ANTIHBS = {"sinan": ""} - -ANTIHCV = {"sinan": ""} - -ANTIHDV = {"sinan": ""} - -ANTIHDVIGM = {"sinan": ""} - -ANTIHEVIGM = {"sinan": ""} - -ANTI_HBS = {"sinan": ""} - -ANTI_HCV = {"sinan": ""} - -ANTI_HIV = {"sinan": ""} - -ANTI_RAB = {"sinan": ""} - -ANTMUNTRAN = {"sinan": ""} - -ANTRELSE_N = {"sinan": ""} - -ANTSIFIL_N = {"sinan": ""} - -ANTTRANS_M = {"sinan": ""} - -ANTUFTRANS = {"sinan": ""} - -ANT_30_DIA = {"sinan": ""} - -ANT_AC = {"sinan": ""} - -ANT_ACIDEN = {"sinan": ""} - -ANT_AIDS = {"sinan": ""} - -ANT_ANEMIA = {"sinan": ""} - -ANT_ANIMAI = {"sinan": ""} - -ANT_ARAGEM = {"sinan": ""} - -ANT_ARRANH = {"sinan": ""} - -ANT_ARRUMO = {"sinan": ""} - -ANT_ASTERI = {"sinan": ""} - -ANT_BC = {"sinan": ""} - -ANT_BCG = {"sinan": ""} - -ANT_CABECA = {"sinan": ""} - -ANT_CANCER = {"sinan": ""} - -ANT_CANDID = {"sinan": ""} - -ANT_CAQUEX = {"sinan": ""} - -ANT_CAT_EX = {"sinan": ""} - -ANT_CB_CAI = {"sinan": ""} - -ANT_CB_CAR = {"sinan": ""} - -ANT_CB_COR = {"sinan": ""} - -ANT_CB_CRI = {"sinan": ""} - -ANT_CB_FOS = {"sinan": ""} - -ANT_CB_GRA = {"sinan": ""} - -ANT_CB_LAM = {"sinan": ""} - -ANT_CB_LAV = {"sinan": ""} - -ANT_CB_LIM = {"sinan": ""} - -ANT_CB_LIX = {"sinan": ""} - -ANT_CB_OUT = {"sinan": ""} - -ANT_CB_PLA = {"sinan": ""} - -ANT_CB_ROE = {"sinan": ""} - -ANT_CB_SIN = {"sinan": ""} - -ANT_CB_TER = {"sinan": ""} - -ANT_CHAGAS = {"sinan": ""} - -ANT_CITO = {"sinan": ""} - -ANT_COLHEI = {"sinan": ""} - -ANT_CONJ_C = {"sinan": ""} - -ANT_CONTAG = {"sinan": ""} - -ANT_CONTAT = {"sinan": ""} - -ANT_CONT_N = {"sinan": ""} - -ANT_CORTE = {"sinan": ""} - -ANT_CRIPTO = {"sinan": ""} - -ANT_CRIP_1 = {"sinan": ""} - -ANT_DERMAT = {"sinan": ""} - -ANT_DESMAT = {"sinan": ""} - -ANT_DIARRE = {"sinan": ""} - -ANT_DILACE = {"sinan": ""} - -ANT_DISFUN = {"sinan": ""} - -ANT_DOSES = {"sinan": ""} - -ANT_DOSES_ = {"sinan": ""} - -ANT_DOSE_3 = {"sinan": ""} - -ANT_DOSE_4 = {"sinan": ""} - -ANT_DOSE_5 = {"sinan": ""} - -ANT_DOSE_7 = {"sinan": ""} - -ANT_DOSE_C = {"sinan": ""} - -ANT_DOSE_T = {"sinan": ""} - -ANT_DOS_N = {"sinan": ""} - -ANT_DROGA = {"sinan": ""} - -ANT_DTULT_ = {"sinan": ""} - -ANT_DTUL_3 = {"sinan": ""} - -ANT_DTUL_4 = {"sinan": ""} - -ANT_DTUL_5 = {"sinan": ""} - -ANT_DTUL_7 = {"sinan": ""} - -ANT_DTUL_8 = {"sinan": ""} - -ANT_DTUL_C = {"sinan": ""} - -ANT_DTUL_T = {"sinan": ""} - -ANT_DT_ACI = {"sinan": ""} - -ANT_DT_EXP = {"sinan": ""} - -ANT_DT_INV = {"sinan": ""} - -ANT_DT_VAC = {"sinan": ""} - -ANT_ESOF_N = {"sinan": ""} - -ANT_EVLABO = {"sinan": ""} - -ANT_EXPOSI = {"sinan": ""} - -ANT_FEBRE = {"sinan": ""} - -ANT_HEMOLF = {"sinan": ""} - -ANT_HEMO_T = {"sinan": ""} - -ANT_HERPES = {"sinan": ""} - -ANT_HISTO = {"sinan": ""} - -ANT_HUMANO = {"sinan": ""} - -ANT_H_SIMP = {"sinan": ""} - -ANT_IDADE = {"sinan": ""} - -ANT_IMUNO = {"sinan": ""} - -ANT_INF_HO = {"sinan": ""} - -ANT_INVEST = {"sinan": ""} - -ANT_IRA = {"sinan": ""} - -ANT_ISOPOR = {"sinan": ""} - -ANT_LAMBED = {"sinan": ""} - -ANT_LAZER = {"sinan": ""} - -ANT_LEUCO = {"sinan": ""} - -ANT_LIMPEZ = {"sinan": ""} - -ANT_LINFO = {"sinan": ""} - -ANT_LINFOM = {"sinan": ""} - -ANT_LINFO_ = {"sinan": ""} - -ANT_LOCA_1 = {"sinan": ""} - -ANT_MAOS = {"sinan": ""} - -ANT_MEMBRO = {"sinan": ""} - -ANT_MEMB_1 = {"sinan": ""} - -ANT_MICRO = {"sinan": ""} - -ANT_MOAGEM = {"sinan": ""} - -ANT_MORDED = {"sinan": ""} - -ANT_MUCOSA = {"sinan": ""} - -ANT_MUNIC_ = {"sinan": ""} - -ANT_MUNI_C = {"sinan": ""} - -ANT_OCUPAC = {"sinan": ""} - -ANT_OUTR = {"sinan": ""} - -ANT_OUTRA = {"sinan": ""} - -ANT_OUTRO = {"sinan": ""} - -ANT_OUTROS = {"sinan": ""} - -ANT_OUTRO_ = {"sinan": ""} - -ANT_OUTR_D = {"sinan": ""} - -ANT_OUT_D = {"sinan": ""} - -ANT_OU_DE = {"sinan": ""} - -ANT_OU_DES = {"sinan": ""} - -ANT_PAIS = {"sinan": ""} - -ANT_PERINA = {"sinan": ""} - -ANT_PLANTI = {"sinan": ""} - -ANT_PNEUMO = {"sinan": ""} - -ANT_PRE_NA = {"sinan": ""} - -ANT_PROFUN = {"sinan": ""} - -ANT_PULMON = {"sinan": ""} - -ANT_PULM_N = {"sinan": ""} - -ANT_RACA = {"sinan": ""} - -ANT_REL_CA = {"sinan": ""} - -ANT_REL_N = {"sinan": ""} - -ANT_RETRO = {"sinan": ""} - -ANT_ROEDOR = {"sinan": ""} - -ANT_SALMO = {"sinan": ""} - -ANT_SARCOM = {"sinan": ""} - -ANT_SECUND = {"sinan": ""} - -ANT_SENTIN = {"sinan": ""} - -ANT_SUPERF = {"sinan": ""} - -ANT_TEMPO_ = {"sinan": ""} - -ANT_TIPOCO = {"sinan": ""} - -ANT_TOSSE = {"sinan": ""} - -ANT_TOXO = {"sinan": ""} - -ANT_TRANS_ = {"sinan": ""} - -ANT_TRASMI = {"sinan": ""} - -ANT_TRATAD = {"sinan": ""} - -ANT_TRAUMA = {"sinan": ""} - -ANT_TRIPLI = {"sinan": ""} - -ANT_TRONCO = {"sinan": ""} - -ANT_TUBE = {"sinan": ""} - -ANT_TUBERC = {"sinan": ""} - -ANT_T_HEMO = {"sinan": ""} - -ANT_UF = {"sinan": ""} - -ANT_UF_1 = {"sinan": ""} - -ANT_UF_2 = {"sinan": ""} - -ANT_UF_3 = {"sinan": ""} - -ANT_UF_CRI = {"sinan": ""} - -ANT_ULTI_D = {"sinan": ""} - -ANT_VACINA = {"sinan": ""} - -AN_ACEVAS = {"sia": ""} - -AN_ALBUMI = {"sia": ""} - -AN_ALTURA = {"sia": ""} - -AN_CNCDO = {"sia": ""} - -AN_DIURES = {"sia": ""} - -AN_DTPDR = {"sia": ""} - -AN_GLICOS = {"sia": ""} - -AN_HB = {"sia": ""} - -AN_HBSAG = {"sia": ""} - -AN_HCV = {"sia": ""} - -AN_HIV = {"sia": ""} - -AN_INTFIS = {"sia": ""} - -AN_PESO = {"sia": ""} - -AN_QUALI = {"sinan": ""} - -AN_QUANT = {"sinan": ""} - -AN_TRU = {"sia": ""} - -AN_ULSOAB = {"sia": ""} - -AP01CV01 = {"cnes": ""} - -AP01CV02 = {"cnes": ""} - -AP01CV03 = {"cnes": ""} - -AP01CV04 = {"cnes": ""} - -AP01CV05 = {"cnes": ""} - -AP01CV06 = {"cnes": ""} - -AP01CV07 = {"cnes": ""} - -AP02CV01 = {"cnes": ""} - -AP02CV02 = {"cnes": ""} - -AP02CV03 = {"cnes": ""} - -AP02CV04 = {"cnes": ""} - -AP02CV05 = {"cnes": ""} - -AP02CV06 = {"cnes": ""} - -AP02CV07 = {"cnes": ""} - -AP03CV01 = {"cnes": ""} - -AP03CV02 = {"cnes": ""} - -AP03CV03 = {"cnes": ""} - -AP03CV04 = {"cnes": ""} - -AP03CV05 = {"cnes": ""} - -AP03CV06 = {"cnes": ""} - -AP03CV07 = {"cnes": ""} - -AP04CV01 = {"cnes": ""} - -AP04CV02 = {"cnes": ""} - -AP04CV03 = {"cnes": ""} - -AP04CV04 = {"cnes": ""} - -AP04CV05 = {"cnes": ""} - -AP04CV06 = {"cnes": ""} - -AP04CV07 = {"cnes": ""} - -AP05CV01 = {"cnes": ""} - -AP05CV02 = {"cnes": ""} - -AP05CV03 = {"cnes": ""} - -AP05CV04 = {"cnes": ""} - -AP05CV05 = {"cnes": ""} - -AP05CV06 = {"cnes": ""} - -AP05CV07 = {"cnes": ""} - -AP06CV01 = {"cnes": ""} - -AP06CV02 = {"cnes": ""} - -AP06CV03 = {"cnes": ""} - -AP06CV04 = {"cnes": ""} - -AP06CV05 = {"cnes": ""} - -AP06CV06 = {"cnes": ""} - -AP06CV07 = {"cnes": ""} - -AP07CV01 = {"cnes": ""} - -AP07CV02 = {"cnes": ""} - -AP07CV03 = {"cnes": ""} - -AP07CV04 = {"cnes": ""} - -AP07CV05 = {"cnes": ""} - -AP07CV06 = {"cnes": ""} - -AP07CV07 = {"cnes": ""} - -APGAR1 = {"sinasc": ""} - -APGAR5 = {"sinasc": ""} - -AP_ADESAO = {"sia": ""} - -AP_ALTA = {"sia": ""} - -AP_APACAN = {"sia": ""} - -AP_APACANT = {"sia": ""} - -AP_ATV_FIS = {"sia": ""} - -AP_AUTORIZ = {"sia": ""} - -AP_CATEND = {"sia": ""} - -AP_CEPPCN = {"sia": ""} - -AP_CIDCAS = {"sia": ""} - -AP_CIDPRI = {"sia": ""} - -AP_CIDSEC = {"sia": ""} - -AP_CID_C1 = {"sia": ""} - -AP_CID_C2 = {"sia": ""} - -AP_CID_C3 = {"sia": ""} - -AP_CID_C4 = {"sia": ""} - -AP_CID_C5 = {"sia": ""} - -AP_CID_CO = {"sia": ""} - -AP_CMP = {"sia": ""} - -AP_CNPJCPF = {"sia": ""} - -AP_CNPJMNT = {"sia": ""} - -AP_CNSPCN = {"sia": ""} - -AP_CODEMI = {"sia": ""} - -AP_CODUNI = {"sia": ""} - -AP_COIDADE = {"sia": ""} - -AP_COMORB = {"sia": ""} - -AP_CONDIC = {"sia": ""} - -AP_DTAUT = {"sia": ""} - -AP_DTFIM = {"sia": ""} - -AP_DTINIC = {"sia": ""} - -AP_DTOCOR = {"sia": ""} - -AP_DTOOCOR = {"sia": ""} - -AP_DTSOLIC = {"sia": ""} - -AP_ENCERR = {"sia": ""} - -AP_ETNIA = {"sia": ""} - -AP_GESTAO = {"sia": ""} - -AP_MEDICAM = {"sia": ""} - -AP_MNDIF = {"sia": ""} - -AP_MN_IND = {"sia": ""} - -AP_MOTSAI = {"sia": ""} - -AP_MUNPCN = {"sia": ""} - -AP_MVM = {"sia": ""} - -AP_NATJUR = {"sia": ""} - -AP_NUIDADE = {"sia": ""} - -AP_OBITO = {"sia": ""} - -AP_PERMAN = {"sia": ""} - -AP_POLIVIT = {"sia": ""} - -AP_PRIPAL = {"sia": ""} - -AP_RACACOR = {"sia": ""} - -AP_REG_PES = {"sia": ""} - -AP_SEXO = {"sia": ""} - -AP_TIPPRE = {"sia": ""} - -AP_TPAPAC = {"sia": ""} - -AP_TPATEN = {"sia": ""} - -AP_TPATEND = {"sia": ""} - -AP_TPPRE = {"sia": ""} - -AP_TPUPS = {"sia": ""} - -AP_TRANSF = {"sia": ""} - -AP_UFDIF = {"sia": ""} - -AP_UFMUN = {"sia": ""} - -AP_UFNACIO = {"sia": ""} - -AP_UNISOL = {"sia": ""} - -AP_VL_AP = {"sia": ""} - -AQ_CID10 = {"sia": ""} - -AQ_CIDINI1 = {"sia": ""} - -AQ_CIDINI2 = {"sia": ""} - -AQ_CIDINI3 = {"sia": ""} - -AQ_CONTTR = {"sia": ""} - -AQ_DTIDEN = {"sia": ""} - -AQ_DTINI1 = {"sia": ""} - -AQ_DTINI2 = {"sia": ""} - -AQ_DTINI3 = {"sia": ""} - -AQ_DTINTR = {"sia": ""} - -AQ_ESQU_P1 = {"sia": ""} - -AQ_ESQU_P2 = {"sia": ""} - -AQ_ESTADI = {"sia": ""} - -AQ_GRAHIS = {"sia": ""} - -AQ_LINFIN = {"sia": ""} - -AQ_MED01 = {"sia": ""} - -AQ_MED02 = {"sia": ""} - -AQ_MED03 = {"sia": ""} - -AQ_MED04 = {"sia": ""} - -AQ_MED05 = {"sia": ""} - -AQ_MED06 = {"sia": ""} - -AQ_MED07 = {"sia": ""} - -AQ_MED08 = {"sia": ""} - -AQ_MED09 = {"sia": ""} - -AQ_MED10 = {"sia": ""} - -AQ_TOTMAU = {"sia": ""} - -AQ_TOTMPL = {"sia": ""} - -AQ_TRANTE = {"sia": ""} - -AREA = {"sinasc": ""} - -AREARES = {"sim": ""} - -ARMAZ_FT = {"cnes": ""} - -ARRANHAO = {"sinan": ""} - -ARRITMIAS = {"sinan": ""} - -ARTEI = {"sinan": ""} - -ARTEM = {"sinan": ""} - -ARTEMI = {"sinan": ""} - -ARTESU = {"sinan": ""} - -ARTRALGIA = {"sinan": ""} - -ARTRITE = {"sinan": ""} - -AR_CID10 = {"sia": ""} - -AR_CIDINI1 = {"sia": ""} - -AR_CIDINI2 = {"sia": ""} - -AR_CIDINI3 = {"sia": ""} - -AR_CIDTR1 = {"sia": ""} - -AR_CIDTR2 = {"sia": ""} - -AR_CIDTR3 = {"sia": ""} - -AR_CONTTR = {"sia": ""} - -AR_DTIDEN = {"sia": ""} - -AR_DTINI1 = {"sia": ""} - -AR_DTINI2 = {"sia": ""} - -AR_DTINI3 = {"sia": ""} - -AR_DTINTR = {"sia": ""} - -AR_ESTADI = {"sia": ""} - -AR_FIMAR1 = {"sia": ""} - -AR_FIMAR2 = {"sia": ""} - -AR_FIMAR3 = {"sia": ""} - -AR_FINALI = {"sia": ""} - -AR_GRAHIS = {"sia": ""} - -AR_INIAR1 = {"sia": ""} - -AR_INIAR2 = {"sia": ""} - -AR_INIAR3 = {"sia": ""} - -AR_LINFIN = {"sia": ""} - -AR_NUMC1 = {"sia": ""} - -AR_NUMC2 = {"sia": ""} - -AR_NUMC3 = {"sia": ""} - -AR_SMRD = {"sia": ""} - -AR_TRANTE = {"sia": ""} - -ASBESTO = {"sinan": ""} - -ASCITE = {"sinan": ""} - -ASMA = {"sinan": ""} - -ASSENTAD = {"cnes": ""} - -ASSINTOM = {"sinan": ""} - -ASSINTOMA = {"sinan": ""} - -ASSINTOMAT = {"sinan": ""} - -ASSISTMED = {"sim": ""} - -ASSIST_SOC = {"sinan": ""} - -ASTENIA = {"sinan": ""} - -ATD_ACEVAS = {"sia": ""} - -ATD_ALBUMI = {"sia": ""} - -ATD_CARACT = {"sia": ""} - -ATD_DTCLI = {"sia": ""} - -ATD_DTPDR = {"sia": ""} - -ATD_FOSFOR = {"sia": ""} - -ATD_HB = {"sia": ""} - -ATD_HBSAG = {"sia": ""} - -ATD_HCV = {"sia": ""} - -ATD_HIV = {"sia": ""} - -ATD_INTERC = {"sia": ""} - -ATD_KTVSEM = {"sia": ""} - -ATD_MAISNE = {"sia": ""} - -ATD_PTH = {"sia": ""} - -ATD_SEAPTO = {"sia": ""} - -ATD_SEPERI = {"sia": ""} - -ATD_SITINI = {"sia": ""} - -ATD_SITTRA = {"sia": ""} - -ATD_TRU = {"sia": ""} - -ATENDAMB = {"cnes": ""} - -ATENDE_MED = {"sinan": ""} - -ATENDHOS = {"cnes": ""} - -ATENDIMENT = {"sinan": ""} - -ATEND_MULH = {"sinan": ""} - -ATEND_PR = {"cnes": ""} - -ATESTADO = {"sim": ""} - -ATESTANTE = {"sim": ""} - -ATE_DT_ALT = {"sinan": ""} - -ATE_DT_INT = {"sinan": ""} - -ATE_HIPOTE = {"sinan": ""} - -ATE_HOSP = {"sinan": ""} - -ATE_HOSPIT = {"sinan": ""} - -ATE_INTERN = {"sinan": ""} - -ATE_MUNICI = {"sinan": ""} - -ATE_UF = {"sinan": ""} - -ATE_UF_HOS = {"sinan": ""} - -ATE_UF_INT = {"sinan": ""} - -ATIVIDAD = {"cnes": ""} - -ATIVIDA_1 = {"sinan": ""} - -ATIVIDA_2 = {"sinan": ""} - -ATIVIDA_3 = {"sinan": ""} - -AT_ATIVIDA = {"sinan": ""} - -AT_LAMINA = {"sinan": ""} - -AT_SINTOMA = {"sinan": ""} - -AUDITIVA = {"sinan": ""} - -AUD_JUST = {"sih": ""} - -AUMENTO = {"sinan": ""} - -AUTORIZ = {"sia": ""} - -AUTOR_ALCO = {"sinan": ""} - -AUTOR_SEXO = {"sinan": ""} - -AUTO_IMUNE = {"sinan": ""} - -AVALIA_N = {"sinan": ""} - -AVAL_ATU_N = {"sinan": ""} - -AVENTAL = {"sinan": ""} - -AV_ACRED = {"cnes": ""} - -AV_PNASS = {"cnes": ""} - -AZT3TC = {"sinan": ""} - -AZT3TC_IND = {"sinan": ""} - -AZT3TC_NFV = {"sinan": ""} - -BACILOSCOP = {"sinan": ""} - -BACILOSC_1 = {"sinan": ""} - -BACILOSC_2 = {"sinan": ""} - -BACILOSC_3 = {"sinan": ""} - -BACILOSC_4 = {"sinan": ""} - -BACILOSC_5 = {"sinan": ""} - -BACILOSC_6 = {"sinan": ""} - -BACILOSC_E = {"sinan": ""} - -BACILOSC_O = {"sinan": ""} - -BACILOS_E2 = {"sinan": ""} - -BACO = {"sinan": ""} - -BACTERIA = {"sinan": ""} - -BAC_APOS_6 = {"sinan": ""} - -BAIRES = {"sim": ""} - -BAIRRO_MAE = {"sinasc": ""} - -BANCOSANGU = {"sinan": ""} - -BENEF_GOV = {"sinan": ""} - -BENZENO = {"sinan": ""} - -BERILIO = {"sinan": ""} - -BIOPSIA = {"sinan": ""} - -BIOSSEG = {"sinan": ""} - -BLOCOPER = {"cnes": ""} - -BLOQUEIO = {"sinan": ""} - -BOTA = {"sinan": ""} - -BOVINO = {"sinan": ""} - -BUSCA_ATIV = {"sinan": ""} - -CABECA = {"sinan": ""} - -CADMIO = {"sinan": ""} - -CALAFRIO = {"sinan": ""} - -CANCER = {"sinan": ""} - -CAO_GATO = {"sinan": ""} - -CAPES = {"sinan": ""} - -CAPIVARA = {"sinan": ""} - -CARACTER = {"cnes": ""} - -CARDIOPATI = {"sinan": ""} - -CARRAPATO = {"sinan": ""} - -CARTORIO = { - "sim": "", - "sinasc": "", -} - -CARVAO = {"sinan": ""} - -CAR_INT = { - "ciha": "", - "sih": "", -} - -CASO = {"sinan": ""} - -CASO_ISOLA = {"sinan": ""} - -CAT = {"sinan": ""} - -CATARATA = {"sinan": ""} - -CATEND = {"sia": ""} - -CAUSABAS = {"sim": ""} - -CAUSABAS_O = {"sim": ""} - -CAUSAMAT = {"sim": ""} - -CBO = {"cnes": ""} - -CBOPROF = {"sia": ""} - -CBOR = {"sih": ""} - -CBOUNICO = {"cnes": ""} - -CB_PRE = {"sim": ""} - -CD_OUTRO = {"sinan": ""} - -CEFALEIA = {"sinan": ""} - -CENTRCIR = {"cnes": ""} - -CENTRNEO = {"cnes": ""} - -CENTROBS = {"cnes": ""} - -CEP = {"sih": ""} - -CGC_CONSOR = {"ciha": ""} - -CGC_HOSP = { - "ciha": "", - "sih": "", -} - -CGC_MANT = {"sih": ""} - -CHAGOMA = {"sinan": ""} - -CHOQUE = {"sinan": ""} - -CICL_VID = {"sinan": ""} - -CIDASSOC = {"sia": ""} - -CIDPRI = {"sia": ""} - -CID_ACID = {"sinan": ""} - -CID_ASSO = {"sih": ""} - -CID_LESAO = {"sinan": ""} - -CID_MORTE = {"sih": ""} - -CID_NOTIF = {"sih": ""} - -CIRCOBITO = {"sim": ""} - -CIRCUNSTAN = {"sinan": ""} - -CIRCUN_DES = {"sinan": ""} - -CIRC_LESAO = {"sinan": ""} - -CIRURGIA = {"sim": ""} - -CIRURGICO = {"sinan": ""} - -CLASAVAL = {"cnes": ""} - -CLASSATUAL = {"sinan": ""} - -CLASSI_FIN = {"sinan": ""} - -CLASSOPERA = {"sinan": ""} - -CLASS_SR = {"cnes": ""} - -CLAS_FORMA = {"sinan": ""} - -CLAS_TIPO_ = {"sinan": ""} - -CLA_ME_ASS = {"sinan": ""} - -CLA_ME_BAC = {"sinan": ""} - -CLA_ME_ETI = {"sinan": ""} - -CLA_SOROGR = {"sinan": ""} - -CLA_TIPO_N = {"sinan": ""} - -CLICDCCA_N = {"sinan": ""} - -CLIENTEL = {"cnes": ""} - -CLINC_CHIK = {"sinan": ""} - -CLIND = {"sinan": ""} - -CLINDI = {"sinan": ""} - -CLI_ABAULA = {"sinan": ""} - -CLI_ABDOMI = {"sinan": ""} - -CLI_AGUDA = {"sinan": ""} - -CLI_AMIGDA = {"sinan": ""} - -CLI_ANEMIA = {"sinan": ""} - -CLI_ANGUST = {"sinan": ""} - -CLI_AQ_D_N = {"sinan": ""} - -CLI_AQ_E_N = {"sinan": ""} - -CLI_ARRITM = {"sinan": ""} - -CLI_ASCEND = {"sinan": ""} - -CLI_ASSIME = {"sinan": ""} - -CLI_ASTENI = {"sinan": ""} - -CLI_A_FMID = {"sinan": ""} - -CLI_A_FMIE = {"sinan": ""} - -CLI_A_FMSD = {"sinan": ""} - -CLI_A_FMSE = {"sinan": ""} - -CLI_A_SMID = {"sinan": ""} - -CLI_A_SMIE = {"sinan": ""} - -CLI_A_SMSD = {"sinan": ""} - -CLI_A_SMSE = {"sinan": ""} - -CLI_A_S_FA = {"sinan": ""} - -CLI_A_TMID = {"sinan": ""} - -CLI_A_TMIE = {"sinan": ""} - -CLI_A_TMSD = {"sinan": ""} - -CLI_A_TMSE = {"sinan": ""} - -CLI_A_T_CE = {"sinan": ""} - -CLI_A_T_FA = {"sinan": ""} - -CLI_BICD_N = {"sinan": ""} - -CLI_BICE_N = {"sinan": ""} - -CLI_BRUDZ = {"sinan": ""} - -CLI_CANDIA = {"sinan": ""} - -CLI_CARDIA = {"sinan": ""} - -CLI_CAVIDA = {"sinan": ""} - -CLI_CDCCRE = {"sinan": ""} - -CLI_CDCLIH = {"sinan": ""} - -CLI_CDC_CI = {"sinan": ""} - -CLI_CDC_CR = {"sinan": ""} - -CLI_CDC_EN = {"sinan": ""} - -CLI_CDC_GE = {"sinan": ""} - -CLI_CDC_HE = {"sinan": ""} - -CLI_CDC_HI = {"sinan": ""} - -CLI_CDC_IN = {"sinan": ""} - -CLI_CDC_IS = {"sinan": ""} - -CLI_CDC_LE = {"sinan": ""} - -CLI_CDC_LI = {"sinan": ""} - -CLI_CDC_ME = {"sinan": ""} - -CLI_CDC_MI = {"sinan": ""} - -CLI_CDC_PC = {"sinan": ""} - -CLI_CDC_PN = {"sinan": ""} - -CLI_CDC_SA = {"sinan": ""} - -CLI_CDC_SI = {"sinan": ""} - -CLI_CDC_SK = {"sinan": ""} - -CLI_CDC_TO = {"sinan": ""} - -CLI_CEFALE = {"sinan": ""} - -CLI_CERVIC = {"sinan": ""} - -CLI_CHOQUE = {"sinan": ""} - -CLI_CICATR = {"sinan": ""} - -CLI_COMA = {"sinan": ""} - -CLI_CONDUT = {"sinan": ""} - -CLI_CONGES = {"sinan": ""} - -CLI_CONJUN = {"sinan": ""} - -CLI_CONTAT = {"sinan": ""} - -CLI_CONVUL = {"sinan": ""} - -CLI_CON_ES = {"sinan": ""} - -CLI_CORDAO = {"sinan": ""} - -CLI_CO_HIV = {"sinan": ""} - -CLI_CRONIC = {"sinan": ""} - -CLI_CUTANE = {"sinan": ""} - -CLI_CUT_DI = {"sinan": ""} - -CLI_DERMA = {"sinan": ""} - -CLI_DESCEN = {"sinan": ""} - -CLI_DESC_O = {"sinan": ""} - -CLI_DIARRE = {"sinan": ""} - -CLI_DISPNE = {"sinan": ""} - -CLI_DISSEM = {"sinan": ""} - -CLI_DOR = {"sinan": ""} - -CLI_DORES = {"sinan": ""} - -CLI_DT = {"sinan": ""} - -CLI_DT_ATE = {"sinan": ""} - -CLI_DT_EXA = {"sinan": ""} - -CLI_EDEMA = {"sinan": ""} - -CLI_EDEMAG = {"sinan": ""} - -CLI_EQUIMO = {"sinan": ""} - -CLI_ESPECI = {"sinan": ""} - -CLI_ESPLEN = {"sinan": ""} - -CLI_EXT_D = {"sinan": ""} - -CLI_EXT_E = {"sinan": ""} - -CLI_FACE = {"sinan": ""} - -CLI_FARING = {"sinan": ""} - -CLI_FEBRE = {"sinan": ""} - -CLI_FLACID = {"sinan": ""} - -CLI_FLE_D = {"sinan": ""} - -CLI_FLE_E = {"sinan": ""} - -CLI_F_MID = {"sinan": ""} - -CLI_F_MIE = {"sinan": ""} - -CLI_F_MSD = {"sinan": ""} - -CLI_F_MSE = {"sinan": ""} - -CLI_GARGAN = {"sinan": ""} - -CLI_H = {"sinan": ""} - -CLI_HEMO = {"sinan": ""} - -CLI_HEMOPU = {"sinan": ""} - -CLI_HEMORR = {"sinan": ""} - -CLI_HEPATI = {"sinan": ""} - -CLI_HEPATO = {"sinan": ""} - -CLI_HERPEG = {"sinan": ""} - -CLI_HERPES = {"sinan": ""} - -CLI_HIPOTE = {"sinan": ""} - -CLI_H_DESC = {"sinan": ""} - -CLI_ICTERI = {"sinan": ""} - -CLI_INFCIT = {"sinan": ""} - -CLI_INJECA = {"sinan": ""} - -CLI_KERNIG = {"sinan": ""} - -CLI_LARING = {"sinan": ""} - -CLI_LEIOMI = {"sinan": ""} - -CLI_LINFA = {"sinan": ""} - -CLI_LINFO = {"sinan": ""} - -CLI_LOCAL = {"sinan": ""} - -CLI_LOCAL_ = {"sinan": ""} - -CLI_LOCA_1 = {"sinan": ""} - -CLI_LOMBAR = {"sinan": ""} - -CLI_MENING = {"sinan": ""} - -CLI_MIALGI = {"sinan": ""} - -CLI_MIAL_D = {"sinan": ""} - -CLI_MIAL_G = {"sinan": ""} - -CLI_MIOCAR = {"sinan": ""} - -CLI_MIOLIT = {"sinan": ""} - -CLI_MUCOSA = {"sinan": ""} - -CLI_MUNICI = {"sinan": ""} - -CLI_NECROS = {"sinan": ""} - -CLI_NEFRIT = {"sinan": ""} - -CLI_NEFRO = {"sinan": ""} - -CLI_NEURO = {"sinan": ""} - -CLI_NEUROL = {"sinan": ""} - -CLI_NOCAR = {"sinan": ""} - -CLI_NUCA = {"sinan": ""} - -CLI_OBSTIP = {"sinan": ""} - -CLI_ORGAOS = {"sinan": ""} - -CLI_OSTEO = {"sinan": ""} - -CLI_OTITE = {"sinan": ""} - -CLI_OTRDES = {"sinan": ""} - -CLI_OUTRAS = {"sinan": ""} - -CLI_OUTRO = {"sinan": ""} - -CLI_OUTROS = {"sinan": ""} - -CLI_OUTR_2 = {"sinan": ""} - -CLI_OUTR_3 = {"sinan": ""} - -CLI_OUT_D = {"sinan": ""} - -CLI_PALATO = {"sinan": ""} - -CLI_PALIDE = {"sinan": ""} - -CLI_PANTUR = {"sinan": ""} - -CLI_PARALB = {"sinan": ""} - -CLI_PARALM = {"sinan": ""} - -CLI_PARALP = {"sinan": ""} - -CLI_PAROTI = {"sinan": ""} - -CLI_PATD_N = {"sinan": ""} - -CLI_PATE_N = {"sinan": ""} - -CLI_PELE = {"sinan": ""} - -CLI_PESCOC = {"sinan": ""} - -CLI_PETEQU = {"sinan": ""} - -CLI_PROGRE = {"sinan": ""} - -CLI_PROST = {"sinan": ""} - -CLI_PROSTR = {"sinan": ""} - -CLI_PSEUDO = {"sinan": ""} - -CLI_PULMAO = {"sinan": ""} - -CLI_RENAL = {"sinan": ""} - -CLI_RESPI = {"sinan": ""} - -CLI_RESPIR = {"sinan": ""} - -CLI_RIGIDE = {"sinan": ""} - -CLI_RINITE = {"sinan": ""} - -CLI_RINORR = {"sinan": ""} - -CLI_SINTOM = {"sinan": ""} - -CLI_TEMPER = {"sinan": ""} - -CLI_TEMPO_ = {"sinan": ""} - -CLI_TONTUR = {"sinan": ""} - -CLI_TORACI = {"sinan": ""} - -CLI_TOSSE = {"sinan": ""} - -CLI_TOX1M = {"sinan": ""} - -CLI_TRAQUE = {"sinan": ""} - -CLI_TRID_N = {"sinan": ""} - -CLI_TRIE_N = {"sinan": ""} - -CLI_TUBERC = {"sinan": ""} - -CLI_TUPULM = {"sinan": ""} - -CLI_VAGAIS = {"sinan": ""} - -CLI_VARICE = {"sinan": ""} - -CLI_VOMITO = {"sinan": ""} - -CLORAFEN = {"sinan": ""} - -CLOROQ = {"sinan": ""} - -CLOROQI = {"sinan": ""} - -CMPT = {"sih": ""} - -CMPT_FIM = {"cnes": ""} - -CMPT_INI = {"cnes": ""} - -CNAE = {"sinan": ""} - -CNAER = {"sih": ""} - -CNAE_PRIN = {"sinan": ""} - -CNES = { - "ciha": "", - "cnes": "", - "sih": "", -} - -CNESTERC = {"cnes": ""} - -CNES_ESF = {"sia": ""} - -CNES_EXEC = {"sia": ""} - -CNPJCPF = {"sia": ""} - -CNPJMNT = {"sia": ""} - -CNPJ_CC = {"sia": ""} - -CNPJ_MAN = {"cnes": ""} - -CNPJ_MANT = {"sih": ""} - -CNSPROF = {"sia": ""} - -CNS_ADM = {"cnes": ""} - -CNS_CONC = {"cnes": ""} - -CNS_CRES = {"cnes": ""} - -CNS_FNUC = {"cnes": ""} - -CNS_HMTL = {"cnes": ""} - -CNS_HMTR = {"cnes": ""} - -CNS_MRAD = {"cnes": ""} - -CNS_NEFR = {"cnes": ""} - -CNS_OCLIN = {"cnes": ""} - -CNS_OPED = {"cnes": ""} - -CNS_PAC = {"sia": ""} - -CNS_PROF = {"cnes": ""} - -CNS_RTEC = {"cnes": ""} - -COAGTOXMA1 = {"sinan": ""} - -COAGTOXMA2 = {"sinan": ""} - -COAGTOXMA3 = {"sinan": ""} - -COBERT = {"pni": ""} - -COBRANCA = { - "ciha": "", - "sih": "", -} - -COB_ESF = {"sia": ""} - -CODANOMAL = {"sinasc": ""} - -CODBAINASC = {"sinasc": ""} - -CODBAIOCOR = {"sim": ""} - -CODBAIRES = { - "sim": "", - "sinasc": "", -} - -CODCART = { - "sim": "", - "sinasc": "", -} - -CODEQUIP = {"cnes": ""} - -CODESTAB = { - "sim": "", - "sinasc": "", -} - -CODIFICADO = {"sim": ""} - -CODIGO = { - "sim": "", - "sinasc": "", -} - -CODINST = {"sinasc": ""} - -CODISINF = {"sinan": ""} - -CODLEITO = {"cnes": ""} - -CODMUNCART = { - "sim": "", - "sinasc": "", -} - -CODMUNNASC = {"sinasc": ""} - -CODMUNNATU = { - "sim": "", - "sinasc": "", -} - -CODMUNOCOR = {"sim": ""} - -CODMUNRES = { - "sim": "", - "sinasc": "", -} - -CODOCUPMAE = {"sinasc": ""} - -CODPAISRES = {"sinasc": ""} - -CODUFMUN = {"cnes": ""} - -CODUFNATU = {"sinasc": ""} - -CODUNI = {"sia": ""} - -COD_ARQ = {"sih": ""} - -COD_CEP = {"cnes": ""} - -COD_IDADE = { - "ciha": "", - "sih": "", -} - -COD_IR = {"cnes": ""} - -COD_MUN_HO = {"sinan": ""} - -COD_SEG = {"sih": ""} - -COD_UF_HOS = {"sinan": ""} - -COLETAMARC = {"sinan": ""} - -COLETIVA = {"sinan": ""} - -COLETRES = {"cnes": ""} - -COLET_COMU = {"sinan": ""} - -COMA = {"sinan": ""} - -COMISS01 = {"cnes": ""} - -COMISS02 = {"cnes": ""} - -COMISS03 = {"cnes": ""} - -COMISS04 = {"cnes": ""} - -COMISS05 = {"cnes": ""} - -COMISS06 = {"cnes": ""} - -COMISS07 = {"cnes": ""} - -COMISS08 = {"cnes": ""} - -COMISS09 = {"cnes": ""} - -COMISS10 = {"cnes": ""} - -COMISS11 = {"cnes": ""} - -COMISS12 = {"cnes": ""} - -COMISSAO = {"cnes": ""} - -COMPET = {"sih": ""} - -COMPETEN = {"cnes": ""} - -COMPLEX = { - "sia": "", - "sih": "", -} - -COMPLICA = {"sinan": ""} - -COMP_OUT = {"sinan": ""} - -COMP_OUT_D = {"sinan": ""} - -COMUNHOSP = {"sinan": ""} - -COMUNINF = {"sinan": ""} - -COMUNSVOIM = {"sim": ""} - -COM_APUTAC = {"sinan": ""} - -COM_CHOQUE = {"sinan": ""} - -COM_COMPOR = {"sinan": ""} - -COM_DEFICT = {"sinan": ""} - -COM_EDEMA = {"sinan": ""} - -COM_LOC = {"sinan": ""} - -COM_NECROS = {"sinan": ""} - -COM_PEST = {"sinan": ""} - -COM_RENAL = {"sinan": ""} - -COM_SECUND = {"sinan": ""} - -COM_SEPTIC = {"sinan": ""} - -COM_SISTEM = {"sinan": ""} - -CONDIC = {"sia": ""} - -CONDIC_ANI = {"sinan": ""} - -CONDUTA = {"sinan": ""} - -CONDUTA_DE = {"sinan": ""} - -CONDUT_DES = {"sinan": ""} - -CONFIRMA = {"sinan": ""} - -CONFIRMAD = {"sinan": ""} - -CONFPESO = {"sinasc": ""} - -CONF_INF_M = {"sinan": ""} - -CONF_INF_U = {"sinan": ""} - -CONF_MAS = {"cnes": ""} - -CONJUNTVIT = {"sinan": ""} - -CONSELHO = {"cnes": ""} - -CONSPRENAT = {"sinasc": ""} - -CONSTIPA = {"sinan": ""} - -CONSULTAS = {"sinasc": ""} - -CONS_ABORT = {"sinan": ""} - -CONS_COMP = {"sinan": ""} - -CONS_DST = {"sinan": ""} - -CONS_ESPEC = {"sinan": ""} - -CONS_ESTRE = {"sinan": ""} - -CONS_GRAV = {"sinan": ""} - -CONS_IDO = {"sinan": ""} - -CONS_MENT = {"sinan": ""} - -CONS_OUTR = {"sinan": ""} - -CONS_SUIC = {"sinan": ""} - -CONS_TUTEL = {"sinan": ""} - -CONT = {"sih": ""} - -CONTADOR = { - "sim": "", - "sinasc": "", -} - -CONTATO = {"sinan": ""} - -CONTEXAM = {"sinan": ""} - -CONTRACEP1 = {"sih": ""} - -CONTRACEP2 = {"sih": ""} - -CONTRATE = {"cnes": ""} - -CONTRATM = {"cnes": ""} - -CONTREG = {"sinan": ""} - -CONTROLE = {"sinan": ""} - -CONTSRVU = {"cnes": ""} - -CONT_OUT = {"sinan": ""} - -CONVULSAO = {"sinan": ""} - -CON_ALIMEN = {"sinan": ""} - -CON_AMBIEN = {"sinan": ""} - -CON_AMB_DE = {"sinan": ""} - -CON_ANIMAI = {"sinan": ""} - -CON_AREA = {"sinan": ""} - -CON_AUTOPS = {"sinan": ""} - -CON_AUTO_M = {"sinan": ""} - -CON_AUTO_U = {"sinan": ""} - -CON_CLASSI = {"sinan": ""} - -CON_CLASS_ = {"sinan": ""} - -CON_CLAS_E = {"sinan": ""} - -CON_CONFIR = {"sinan": ""} - -CON_CRITER = {"sinan": ""} - -CON_DESCAR = {"sinan": ""} - -CON_DIAGES = {"sinan": ""} - -CON_DIAGNO = {"sinan": ""} - -CON_DIAG_D = {"sinan": ""} - -CON_DOENCA = {"sinan": ""} - -CON_DT_ENC = {"sinan": ""} - -CON_DT_OBI = {"sinan": ""} - -CON_ENCHEN = {"sinan": ""} - -CON_ENTULH = {"sinan": ""} - -CON_ESGOTO = {"sinan": ""} - -CON_EVOLUC = {"sinan": ""} - -CON_FHD = {"sinan": ""} - -CON_FORMA = {"sinan": ""} - -CON_GRAVID = {"sinan": ""} - -CON_IMPORT = {"sinan": ""} - -CON_INFECC = {"sinan": ""} - -CON_INF_BA = {"sinan": ""} - -CON_INF_DI = {"sinan": ""} - -CON_INF_MU = {"sinan": ""} - -CON_INF_OU = {"sinan": ""} - -CON_INF_PA = {"sinan": ""} - -CON_INF_UF = {"sinan": ""} - -CON_LOCAL = {"sinan": ""} - -CON_LOCAL2 = {"sinan": ""} - -CON_LOCALI = {"sinan": ""} - -CON_MUNICI = {"sinan": ""} - -CON_OUTRA = {"sinan": ""} - -CON_PAIS = {"sinan": ""} - -CON_PROVAV = {"sinan": ""} - -CON_RIO = {"sinan": ""} - -CON_ROEDOR = {"sinan": ""} - -CON_SOROTE = {"sinan": ""} - -CON_TERREN = {"sinan": ""} - -CON_TRIAT = {"sinan": ""} - -CON_UF = {"sinan": ""} - -COPAISINF = {"sinan": ""} - -COPRO_D_1 = {"sinan": ""} - -COPRO_D_2 = {"sinan": ""} - -COPRO_D_3 = {"sinan": ""} - -COPRO_R1 = {"sinan": ""} - -COPRO_R2 = {"sinan": ""} - -COPRO_R3 = {"sinan": ""} - -CORRACA = {"ibge": ""} - -COUFHOSP = {"sinan": ""} - -COUFINF = {"sinan": ""} - -COUNIDINF = {"sinan": ""} - -CO_AGENC = {"cnes": ""} - -CO_BANCO = {"cnes": ""} - -CO_CIDPRIM = {"sia": ""} - -CO_CIDSEC = {"sia": ""} - -CO_ERRO = {"sih": ""} - -CO_FOCAL = {"sinan": ""} - -CO_INE = {"sia": ""} - -CO_MUN_EX2 = {"sinan": ""} - -CO_MUN_EX3 = {"sinan": ""} - -CO_MUN_EXP = {"sinan": ""} - -CO_MUN_R1 = {"sinan": ""} - -CO_MUN_R2 = {"sinan": ""} - -CO_MUN_R3 = {"sinan": ""} - -CO_MUN_R4 = {"sinan": ""} - -CO_PAIS_1 = {"sinan": ""} - -CO_PAIS_2 = {"sinan": ""} - -CO_PAIS_3 = {"sinan": ""} - -CO_RISCO = {"sinan": ""} - -CO_UF_1 = {"sinan": ""} - -CO_UF_2 = {"sinan": ""} - -CO_UF_3 = {"sinan": ""} - -CO_UF_DES1 = {"sinan": ""} - -CO_UF_DES2 = {"sinan": ""} - -CO_UF_DES3 = {"sinan": ""} - -CO_UF_EX2 = {"sinan": ""} - -CO_UF_EX3 = {"sinan": ""} - -CO_UF_EXP = {"sinan": ""} - -CO_UF_R1 = {"sinan": ""} - -CO_UF_R2 = {"sinan": ""} - -CO_UF_R3 = {"sinan": ""} - -CO_UF_R4 = {"sinan": ""} - -CPFUNICO = {"cnes": ""} - -CPF_AUT = {"sih": ""} - -CPF_CNPJ = {"cnes": ""} - -CPF_PROF = {"cnes": ""} - -CPF_UNICO = {"cnes": ""} - -CRITERIO = {"sinan": ""} - -CRITICA = { - "sim": "", - "sinasc": "", -} - -CRI_1000 = {"sinan": ""} - -CRI_1500 = {"sinan": ""} - -CRI_500 = {"sinan": ""} - -CRM = {"sim": ""} - -CROMO = {"sinan": ""} - -CRSOCOR = {"sim": ""} - -CRSRES = {"sim": ""} - -CRS_MAE = {"sinasc": ""} - -CRS_OCOR = {"sinasc": ""} - -CS_ABDOMEN = {"sinan": ""} - -CS_ABDOMIN = {"sinan": ""} - -CS_ANALISE = {"sinan": ""} - -CS_ANTIB = {"sinan": ""} - -CS_ANTIBIO = {"sinan": ""} - -CS_ANTIB_T = {"sinan": ""} - -CS_APNEIA = {"sinan": ""} - -CS_ASSINTO = {"sinan": ""} - -CS_ATEND_N = {"sinan": ""} - -CS_BUSCAAT = {"sinan": ""} - -CS_CADASTR = {"sinan": ""} - -CS_CAIMBRA = {"sinan": ""} - -CS_CHOQUE = {"sinan": ""} - -CS_CHORO = {"sinan": ""} - -CS_CIANOSE = {"sinan": ""} - -CS_COBERTU = {"sinan": ""} - -CS_COLETA = {"sinan": ""} - -CS_CRISE = {"sinan": ""} - -CS_CRISES = {"sinan": ""} - -CS_CULTURA = {"sinan": ""} - -CS_DESCART = {"sinan": ""} - -CS_DESIT = {"sinan": ""} - -CS_DESITRA = {"sinan": ""} - -CS_DESNUTR = {"sinan": ""} - -CS_DIARRE = {"sinan": ""} - -CS_DIVULGA = {"sinan": ""} - -CS_DOR = {"sinan": ""} - -CS_ENCEFAL = {"sinan": ""} - -CS_ESCOLAR = {"sinan": ""} - -CS_ESCOL_N = {"sinan": ""} - -CS_FEBRE = {"sinan": ""} - -CS_FLXRET = {"sinan": ""} - -CS_FONTE = {"sinan": ""} - -CS_FREQUEN = {"sinan": ""} - -CS_GESTANT = {"sinan": ""} - -CS_HOSPITA = {"sinan": ""} - -CS_INF_COT = {"sinan": ""} - -CS_INQUERI = {"sinan": ""} - -CS_LIQUOR = {"sinan": ""} - -CS_LOCAL = {"sinan": ""} - -CS_MAMAR = {"sinan": ""} - -CS_MATERIA = {"sinan": ""} - -CS_MEMBROS = {"sinan": ""} - -CS_MENING = {"sinan": ""} - -CS_MUCO = {"sinan": ""} - -CS_NASCIDO = {"sinan": ""} - -CS_NEG_ESP = {"sinan": ""} - -CS_NUCA = {"sinan": ""} - -CS_OPISTOT = {"sinan": ""} - -CS_ORIENTA = {"sinan": ""} - -CS_ORIGEM = {"sinan": ""} - -CS_OTITE = {"sinan": ""} - -CS_OUTRAS = {"sinan": ""} - -CS_OUTROS = {"sinan": ""} - -CS_OUT_COM = {"sinan": ""} - -CS_OUT_SIN = {"sinan": ""} - -CS_PNEUMON = {"sinan": ""} - -CS_POSITIV = {"sinan": ""} - -CS_RACA = {"sinan": ""} - -CS_REIDRAT = {"sinan": ""} - -CS_RESULTA = {"sinan": ""} - -CS_RISO = {"sinan": ""} - -CS_SANGUE = {"sinan": ""} - -CS_SECRECA = {"sinan": ""} - -CS_SEXO = {"sinan": ""} - -CS_SIN_OUT = {"sinan": ""} - -CS_SUGOU = {"sinan": ""} - -CS_SUSPEIT = {"sinan": ""} - -CS_TEMP37 = {"sinan": ""} - -CS_TEMP_38 = {"sinan": ""} - -CS_TIPO = {"sinan": ""} - -CS_TOSSE_E = {"sinan": ""} - -CS_TOSSE_P = {"sinan": ""} - -CS_TRANS = {"sinan": ""} - -CS_TRISMO = {"sinan": ""} - -CS_URINA = {"sinan": ""} - -CS_VACINA = {"sinan": ""} - -CS_VACINAC = {"sinan": ""} - -CS_VACINAL = {"sinan": ""} - -CS_VACTETA = {"sinan": ""} - -CS_VAC_N = {"sinan": ""} - -CS_VOMITO = {"sinan": ""} - -CS_VOMITOS = {"sinan": ""} - -CS_ZONA = {"sinan": ""} - -CULTURA_ES = {"sinan": ""} - -CULTURA_OU = {"sinan": ""} - -C_CORREN = {"cnes": ""} - -C_D = {"sinan": ""} - -C_M = {"sinan": ""} - -DATANASC = {"sim": ""} - -DATAOBITO = {"sim": ""} - -DATAREG = {"sim": ""} - -DATA_CART = {"sinasc": ""} - -DATA_NASC = {"sinasc": ""} - -DE15A39ANO = {"sinan": ""} - -DE5A14ANOS = {"sinan": ""} - -DEFEN_PUBL = {"sinan": ""} - -DEF_AUDITI = {"sinan": ""} - -DEF_DIAGNO = {"sinan": ""} - -DEF_ESPEC = {"sinan": ""} - -DEF_FISICA = {"sinan": ""} - -DEF_MENTAL = {"sinan": ""} - -DEF_OUT = {"sinan": ""} - -DEF_TRANS = {"sinan": ""} - -DEF_VISUAL = {"sinan": ""} - -DEIONIZA = {"cnes": ""} - -DELEG = {"sinan": ""} - -DELEG_CRIA = {"sinan": ""} - -DELEG_IDOS = {"sinan": ""} - -DELEG_MULH = {"sinan": ""} - -DENCRIREND = {"ibge": ""} - -DENDESOCUP = {"ibge": ""} - -DENGUE = {"sinan": ""} - -DENRENDA = {"ibge": ""} - -DENTARIO = {"sinan": ""} - -DENTRABINF = {"ibge": ""} - -DESCSEGM = {"cnes": ""} - -DESMATA_N = {"sinan": ""} - -DESTINOPAC = {"sia": ""} - -DEXAME = {"sinan": ""} - -DG_OUT_N = {"sinan": ""} - -DIABETES = {"sinan": ""} - -DIAGNO_LAB = {"sinan": ""} - -DIAGSEC1 = {"sih": ""} - -DIAGSEC2 = {"sih": ""} - -DIAGSEC3 = {"sih": ""} - -DIAGSEC4 = {"sih": ""} - -DIAGSEC5 = {"sih": ""} - -DIAGSEC6 = {"sih": ""} - -DIAGSEC7 = {"sih": ""} - -DIAGSEC8 = {"sih": ""} - -DIAGSEC9 = {"sih": ""} - -DIAG_CONF = {"sinan": ""} - -DIAG_DESCA = {"sinan": ""} - -DIAG_ESP = {"sinan": ""} - -DIAG_MAE = {"sinan": ""} - -DIAG_PARA = {"sinan": ""} - -DIAG_PAR_N = {"sinan": ""} - -DIAG_PRINC = { - "ciha": "", - "sih": "", -} - -DIAG_SEC = {"sih": ""} - -DIAG_SECUN = { - "ciha": "", - "sih": "", -} - -DIALISE = {"cnes": ""} - -DIARREIA = {"sinan": ""} - -DIAR_ACOM = {"sih": ""} - -DIAS = {"sinan": ""} - -DIAS_PERM = { - "ciha": "", - "sih": "", -} - -DIFDATA = { - "sim": "", - "sinasc": "", -} - -DIFER = {"pni": ""} - -DILACERANT = {"sinan": ""} - -DINTERNA = {"sinan": ""} - -DIR_HUMAN = {"sinan": ""} - -DISFAGIA = {"sinan": ""} - -DISTRADM = {"cnes": ""} - -DISTRSAN = {"cnes": ""} - -DOENCA_TRA = {"sinan": ""} - -DOMICILI = {"sinan": ""} - -DOR = {"sinan": ""} - -DORMIU_N = {"sinan": ""} - -DOR_COSTAS = {"sinan": ""} - -DOR_RETRO = {"sinan": ""} - -DOSAGEM = {"sinan": ""} - -DOSE = { - "pni": "", - "sinan": "", -} - -DOSE1 = {"pni": ""} - -DOSEN = {"pni": ""} - -DOSES = {"sinan": ""} - -DOSES_A = {"sinan": ""} - -DOSE_RECEB = {"sinan": ""} - -DOXOCI = {"sinan": ""} - -DROGA = {"sinan": ""} - -DROGAS = {"sinan": ""} - -DSALIMENTO = {"sinan": ""} - -DSCARDIOP = {"sinan": ""} - -DSCAUSALIM = {"sinan": ""} - -DSFONTE = {"sinan": ""} - -DSMOTIVO = {"sinan": ""} - -DSTITULO1 = {"sinan": ""} - -DSTRAESQUE = {"sinan": ""} - -DS_ALI1 = {"sinan": ""} - -DS_ALI1OUT = {"sinan": ""} - -DS_ALI2 = {"sinan": ""} - -DS_ALI2OUT = {"sinan": ""} - -DS_ESQUEMA = {"sinan": ""} - -DS_FIM_GES = {"sinan": ""} - -DS_FORMA = {"sinan": ""} - -DS_F_OUTRO = {"sinan": ""} - -DS_INDUS = {"sinan": ""} - -DS_INF_LOC = {"sinan": ""} - -DS_INF_OUT = {"sinan": ""} - -DS_INGEST = {"sinan": ""} - -DS_INI_GES = {"sinan": ""} - -DS_LOCAL1 = {"sinan": ""} - -DS_LOCAL2 = {"sinan": ""} - -DS_MUN_1 = {"sinan": ""} - -DS_MUN_2 = {"sinan": ""} - -DS_MUN_3 = {"sinan": ""} - -DS_OUTRO = {"sinan": ""} - -DS_OUTROSI = {"sinan": ""} - -DS_OUTR_LO = {"sinan": ""} - -DS_OUT_AMB = {"sinan": ""} - -DS_PARES = {"sinan": ""} - -DS_RESU_OU = {"sinan": ""} - -DS_TRANS1 = {"sinan": ""} - -DS_TRANS2 = {"sinan": ""} - -DS_TRANS3 = {"sinan": ""} - -DS_TRANS_1 = {"sinan": ""} - -DS_TRANS_2 = {"sinan": ""} - -DS_TRAT = {"sinan": ""} - -DTALTA = {"sinan": ""} - -DTALTA_N = {"sinan": ""} - -DTATEND = {"sinan": ""} - -DTATESTADO = {"sim": ""} - -DTCADASTRO = { - "sim": "", - "sinasc": "", -} - -DTCADINF = {"sim": ""} - -DTCADINV = {"sim": ""} - -DTCONCASO = {"sim": ""} - -DTCONFIRMA = {"sinan": ""} - -DTCONINV = {"sim": ""} - -DTDECLARAC = {"sinasc": ""} - -DTDIASINAC = {"sinan": ""} - -DTELETRO = {"sinan": ""} - -DTFEZESCOL = {"sinan": ""} - -DTIMUNO = {"sinan": ""} - -DTINICTRAT = {"sinan": ""} - -DTINTERNA = {"sinan": ""} - -DTINVESTIG = {"sim": ""} - -DTISOLA = {"sinan": ""} - -DTMICRO1 = {"sinan": ""} - -DTMICRO2 = {"sinan": ""} - -DTMUDESQ = {"sinan": ""} - -DTNASC = { - "sia": "", - "sim": "", - "sinasc": "", -} - -DTNASCMAE = {"sinasc": ""} - -DTOBITO = {"sim": ""} - -DTPORTAR = {"cnes": ""} - -DTPRICONS = {"sinan": ""} - -DTRAPIDO1 = {"sinan": ""} - -DTRATA = {"sinan": ""} - -DTRECEBIM = { - "sim": "", - "sinasc": "", -} - -DTRECORIG = { - "sim": "", - "sinasc": "", -} - -DTRECORIGA = { - "sim": "", - "sinasc": "", -} - -DTREGCART = { - "sim": "", - "sinasc": "", -} - -DTS1 = {"sinan": ""} - -DTS2 = {"sinan": ""} - -DTSORO = {"sinan": ""} - -DTSOROCOL = {"sinan": ""} - -DTSUSPEIC = {"sinan": ""} - -DTTESTE1 = {"sinan": ""} - -DTTRANSDM = {"sinan": ""} - -DTTRANSFU = {"sinan": ""} - -DTTRANSRM = {"sinan": ""} - -DTTRANSRS = {"sinan": ""} - -DTTRANSSE = {"sinan": ""} - -DTTRANSSM = {"sinan": ""} - -DTTRANSUS = {"sinan": ""} - -DTTRAT = {"sinan": ""} - -DTTRIAGEM = {"sinan": ""} - -DTULTCOMP = {"sinan": ""} - -DTULTMENST = {"sinasc": ""} - -DT_1VAC = {"sinan": ""} - -DT_1_DOSE = {"sinan": ""} - -DT_2VAC = {"sinan": ""} - -DT_2_DOSE = {"sinan": ""} - -DT_3_DOSE = {"sinan": ""} - -DT_ACID = {"sinan": ""} - -DT_ACIDENT = {"sinan": ""} - -DT_ACRED = {"cnes": ""} - -DT_ADM_ANT = {"sinan": ""} - -DT_ALI1COL = {"sinan": ""} - -DT_ALI2COL = {"sinan": ""} - -DT_ALRM = {"sinan": ""} - -DT_APLI_SO = {"sinan": ""} - -DT_ATEND = { - "ciha": "", - "sia": "", -} - -DT_ATENDE = {"sinan": ""} - -DT_ATENDIM = {"sinan": ""} - -DT_ATIVA = {"cnes": ""} - -DT_ATUAL = {"cnes": ""} - -DT_CATARRA = {"sinan": ""} - -DT_CHIK_S1 = {"sinan": ""} - -DT_CHIK_S2 = {"sinan": ""} - -DT_CHOQUE = {"sinan": ""} - -DT_COL1 = {"sinan": ""} - -DT_COL2 = {"sinan": ""} - -DT_COL3 = {"sinan": ""} - -DT_COLETA = {"sinan": ""} - -DT_COLOUT = {"sinan": ""} - -DT_COL_1 = {"sinan": ""} - -DT_COL_2 = {"sinan": ""} - -DT_COL_DIR = {"sinan": ""} - -DT_COL_HE2 = {"sinan": ""} - -DT_COL_HEM = {"sinan": ""} - -DT_COL_IGM = {"sinan": ""} - -DT_COL_IND = {"sinan": ""} - -DT_COL_PL2 = {"sinan": ""} - -DT_COL_PLQ = {"sinan": ""} - -DT_COL_S1 = {"sinan": ""} - -DT_COL_S2 = {"sinan": ""} - -DT_CONFIRM = {"sinan": ""} - -DT_COPRO = {"sinan": ""} - -DT_COPRO1 = {"sinan": ""} - -DT_COPRO2 = {"sinan": ""} - -DT_COPRO3 = {"sinan": ""} - -DT_DESAT = {"cnes": ""} - -DT_DESC1 = {"sinan": ""} - -DT_DESC2 = {"sinan": ""} - -DT_DESC3 = {"sinan": ""} - -DT_DESLC1 = {"sinan": ""} - -DT_DESLC2 = {"sinan": ""} - -DT_DESLC3 = {"sinan": ""} - -DT_DIAG = {"sinan": ""} - -DT_DIGITA = {"sinan": ""} - -DT_DOSE = {"sinan": ""} - -DT_DOSE_1 = {"sinan": ""} - -DT_DOSE_2 = {"sinan": ""} - -DT_DOSE_3 = {"sinan": ""} - -DT_DOSE_4 = {"sinan": ""} - -DT_DOSE_5 = {"sinan": ""} - -DT_DOSE_N = {"sinan": ""} - -DT_ENCERRA = {"sinan": ""} - -DT_ENVIO = {"sinan": ""} - -DT_EVOLUC = {"sinan": ""} - -DT_EXPED = {"cnes": ""} - -DT_EXPO = {"sinan": ""} - -DT_FEBRE = {"sinan": ""} - -DT_FEZES = {"sinan": ""} - -DT_FIM = {"sia": ""} - -DT_GRAV = {"sinan": ""} - -DT_HEMO1 = {"sinan": ""} - -DT_HEMO2 = {"sinan": ""} - -DT_HEMO3 = {"sinan": ""} - -DT_INICIO = {"sia": ""} - -DT_INICIO_ = {"sinan": ""} - -DT_INIC_TR = {"sinan": ""} - -DT_INI_EPI = {"sinan": ""} - -DT_INTER = {"sih": ""} - -DT_INTERNA = {"sinan": ""} - -DT_INVEST = {"sinan": ""} - -DT_LIQUOR = {"sinan": ""} - -DT_MOTCOB = {"sia": ""} - -DT_MUDANCA = {"sinan": ""} - -DT_NASC = {"sinan": ""} - -DT_NOTIFIC = {"sinan": ""} - -DT_NOTI_AT = {"sinan": ""} - -DT_NS1 = {"sinan": ""} - -DT_OBITO = {"sinan": ""} - -DT_OCOR = {"sinan": ""} - -DT_OUTR1 = {"sinan": ""} - -DT_OUTR2 = {"sinan": ""} - -DT_OUTR3 = {"sinan": ""} - -DT_PCR = {"sinan": ""} - -DT_PCR_1 = {"sinan": ""} - -DT_PCR_2 = {"sinan": ""} - -DT_PCR_3 = {"sinan": ""} - -DT_PNASS = {"cnes": ""} - -DT_PRNT = {"sinan": ""} - -DT_PROCESS = {"sia": ""} - -DT_PUBLE = {"cnes": ""} - -DT_PUBLM = {"cnes": ""} - -DT_RAPIDO = {"sinan": ""} - -DT_REFORCO = {"sinan": ""} - -DT_RESU3 = {"sinan": ""} - -DT_RISCO1 = {"sinan": ""} - -DT_RISCO2 = {"sinan": ""} - -DT_RISCO3 = {"sinan": ""} - -DT_RISCO4 = {"sinan": ""} - -DT_RTPCR = {"sinan": ""} - -DT_R_TRA = {"sinan": ""} - -DT_S1 = {"sinan": ""} - -DT_S2 = {"sinan": ""} - -DT_SAIDA = { - "ciha": "", - "sih": "", -} - -DT_SIN_PRI = {"sinan": ""} - -DT_SORO = {"sinan": ""} - -DT_SORO1 = {"sinan": ""} - -DT_SORO2 = {"sinan": ""} - -DT_SOROR1 = {"sinan": ""} - -DT_SOROR2 = {"sinan": ""} - -DT_TRANSDM = {"sinan": ""} - -DT_TRANSRM = {"sinan": ""} - -DT_TRANSRS = {"sinan": ""} - -DT_TRANSSE = {"sinan": ""} - -DT_TRANSSM = {"sinan": ""} - -DT_TRANSUS = {"sinan": ""} - -DT_TRIA_11 = {"sinan": ""} - -DT_TRISMO = {"sinan": ""} - -DT_TRNASRM = {"sinan": ""} - -DT_TRNASRS = {"sinan": ""} - -DT_TR_RAB = {"sinan": ""} - -DT_ULT_DOS = {"sinan": ""} - -DT_URO = {"sinan": ""} - -DT_URO2 = {"sinan": ""} - -DT_URO3 = {"sinan": ""} - -DT_VAC1 = {"sinan": ""} - -DT_VACINA = {"sinan": ""} - -DT_VAC_1 = {"sinan": ""} - -DT_VAC_2 = {"sinan": ""} - -DT_VAC_3 = {"sinan": ""} - -DT_VAC_4 = {"sinan": ""} - -DT_VAC_5 = {"sinan": ""} - -DT_VAC_ULT = {"sinan": ""} - -DT_VENCIM = {"sinan": ""} - -DT_VIRAL = {"sinan": ""} - -DT_VOP = {"sinan": ""} - -DURACAO = {"sinan": ""} - -D_DIAR = {"sinan": ""} - -D_VOMITO = {"sinan": ""} - -ECG = {"sinan": ""} - -ECG_RESULT = {"sinan": ""} - -EDEMA = {"sinan": ""} - -ELISA = {"sinan": ""} - -ELISA1 = {"sinan": ""} - -ELISA2 = {"sinan": ""} - -ELI_IGG_S1 = {"sinan": ""} - -ELI_IGG_S2 = {"sinan": ""} - -ELI_IGM_S1 = {"sinan": ""} - -ELI_IGM_S2 = {"sinan": ""} - -EMAGRA = {"sinan": ""} - -ENCAMINHA = {"sinan": ""} - -ENC_ABRIGO = {"sinan": ""} - -ENC_CREAS = {"sinan": ""} - -ENC_DEAM = {"sinan": ""} - -ENC_DELEG = {"sinan": ""} - -ENC_DPCA = {"sinan": ""} - -ENC_ESPEC = {"sinan": ""} - -ENC_IML = {"sinan": ""} - -ENC_MPU = {"sinan": ""} - -ENC_MULHER = {"sinan": ""} - -ENC_OUTR = {"sinan": ""} - -ENC_SAUDE = {"sinan": ""} - -ENC_SENTIN = {"sinan": ""} - -ENC_TUTELA = {"sinan": ""} - -ENC_VARA = {"sinan": ""} - -ENDEMICO = {"sinan": ""} - -ENDRES = {"sinasc": ""} - -ENTERO = {"sinan": ""} - -ENTO_ANIMA = {"sinan": ""} - -ENTO_CAO = {"sinan": ""} - -ENTO_CAPTU = {"sinan": ""} - -ENTO_EQUIN = {"sinan": ""} - -ENTO_EXIST = {"sinan": ""} - -ENTO_EXI_1 = {"sinan": ""} - -ENTO_EXI_2 = {"sinan": ""} - -ENTO_EXI_3 = {"sinan": ""} - -ENTO_EXI_4 = {"sinan": ""} - -ENTO_EXTRA = {"sinan": ""} - -ENTO_FLEBO = {"sinan": ""} - -ENTO_INSET = {"sinan": ""} - -ENTO_INTRA = {"sinan": ""} - -ENTO_LOCAL = {"sinan": ""} - -ENTO_OUTRO = {"sinan": ""} - -ENTO_PERID = {"sinan": ""} - -ENTO_PROXI = {"sinan": ""} - -ENTO_TRANS = {"sinan": ""} - -ENTRADA = {"sinan": ""} - -EPICUTA = {"sinan": ""} - -EPISTAXE = {"sinan": ""} - -EPIS_RACIO = {"sinan": ""} - -EPI_PESTE = {"sinan": ""} - -EQBRALTA = {"cnes": ""} - -EQBRBAIX = {"cnes": ""} - -EQBRMEDI = {"cnes": ""} - -EQDOSCLI = {"cnes": ""} - -EQFONSEL = {"cnes": ""} - -EQSISPLN = {"cnes": ""} - -EQUINOS = {"sinan": ""} - -EQ_MAREA = {"cnes": ""} - -EQ_MINDI = {"cnes": ""} - -ESC = {"sim": ""} - -ESC2010 = {"sim": ""} - -ESCFALAGR1 = {"sim": ""} - -ESCMAE = { - "sim": "", - "sinasc": "", -} - -ESCMAE2010 = { - "sim": "", - "sinasc": "", -} - -ESCMAEAGR1 = { - "sim": "", - "sinasc": "", -} - -ESCOLA = {"cnes": ""} - -ESCOLARID = {"ibge": ""} - -ESCOLMAE = {"sinan": ""} - -ESCOLMAE_N = {"sinan": ""} - -ESC_MAE_N = {"sinan": ""} - -ESFERA_A = {"cnes": ""} - -ESPEC = { - "ciha": "", - "sih": "", -} - -ESPECIE = {"sinan": ""} - -ESPECIE_N = {"sinan": ""} - -ESPECIFICO = {"sinan": ""} - -ESPLENO = {"sinan": ""} - -ESPLENOM = {"sinan": ""} - -ESP_OUT = {"sinan": ""} - -ESQ_ATU_N = {"sinan": ""} - -ESQ_INI_N = {"sinan": ""} - -ESTABDESCR = {"sim": ""} - -ESTAB_OCOR = {"sinasc": ""} - -ESTCIV = {"sim": ""} - -ESTCIVIL = {"sim": ""} - -ESTCIVMAE = {"sinasc": ""} - -ESTREPTOMI = {"sinan": ""} - -ETAMBUTOL = {"sinan": ""} - -ETIOL_OUTR = {"sinan": ""} - -ETIONAMIDA = {"sinan": ""} - -ETNIA = { - "sia": "", - "sih": "", - "sim": "", - "sinasc": "", -} - -EVIDENCIA = {"sinan": ""} - -EVOLUCAO = {"sinan": ""} - -EVOL_AFAST = {"sinan": ""} - -EVOR1_DT_R = {"sinan": ""} - -EVOR_A_MID = {"sinan": ""} - -EVOR_A_MIE = {"sinan": ""} - -EVOR_A_MSD = {"sinan": ""} - -EVOR_A_MSE = {"sinan": ""} - -EVOR_DT_RE = {"sinan": ""} - -EVOR_F_MID = {"sinan": ""} - -EVOR_F_MIE = {"sinan": ""} - -EVOR_F_MSD = {"sinan": ""} - -EVOR_F_MSE = {"sinan": ""} - -EVOR_RC_ED = {"sinan": ""} - -EVOR_RC_EE = {"sinan": ""} - -EVOR_RC_FD = {"sinan": ""} - -EVOR_RC_FE = {"sinan": ""} - -EVOR_S_FAC = {"sinan": ""} - -EVOR_S_MID = {"sinan": ""} - -EVOR_S_MIE = {"sinan": ""} - -EVOR_S_MSD = {"sinan": ""} - -EVOR_S_MSE = {"sinan": ""} - -EVO_DIAG = {"sinan": ""} - -EVO_DIAG_N = {"sinan": ""} - -EVO_DT_OBI = {"sinan": ""} - -EVO_OUTR = {"sinan": ""} - -EXAME = { - "sim": "", - "sinan": "", -} - -EXANTEMA = {"sinan": ""} - -EXPDIFDATA = {"sim": ""} - -EXPO_N = {"sinan": ""} - -EXTRAPU1_N = {"sinan": ""} - -EXTRAPU2_N = {"sinan": ""} - -EXTRAPUL_O = {"sinan": ""} - -FACIAL = {"sinan": ""} - -FAEC_TP = {"sih": ""} - -FALA = {"sinan": ""} - -FALENCIA = {"sinan": ""} - -FC_CONTATO = {"sinan": ""} - -FC_CONT_DE = {"sinan": ""} - -FEBRE = {"sinan": ""} - -FEN_HEMORR = {"sinan": ""} - -FERIMENTO = {"sinan": ""} - -FERIMENT_N = {"sinan": ""} - -FEZES = {"sinan": ""} - -FIGADO = {"sinan": ""} - -FILHMORT = {"sim": ""} - -FILHVIVOS = {"sim": ""} - -FIL_ABORT = {"sinasc": ""} - -FIL_MORTOS = {"sinasc": ""} - -FIL_VIVOS = {"sinasc": ""} - -FIM = {"sia": ""} - -FIM_ANIMAL = {"sinan": ""} - -FINANC = {"sih": ""} - -FISCALIZA = {"sinan": ""} - -FLOGISTICO = {"sinan": ""} - -FLUXO_AERE = {"sinan": ""} - -FLXRECEBI = {"sinan": ""} - -FOI_MATA = {"sinan": ""} - -FONTE = { - "ciha": "", - "sim": "", - "sinan": "", -} - -FONTEINV = {"sim": ""} - -FONTES = {"sim": ""} - -FONTESINF = {"sim": ""} - -FONTE_ORC = {"sih": ""} - -FONTINFO = {"sim": ""} - -FORMA = {"sinan": ""} - -FORMACLINI = {"sinan": ""} - -FORMA_CO = {"sinan": ""} - -FORMA_TF = {"sinan": ""} - -FORMA_TI = {"sinan": ""} - -FORMA_TS = {"sinan": ""} - -FORMA_TT = {"sinan": ""} - -FO_ANT_HBC = {"sinan": ""} - -FO_ANT_HCV = {"sinan": ""} - -FO_ANT_HIV = {"sinan": ""} - -FO_HBSAG = {"sinan": ""} - -FRAQUEZA = {"sinan": ""} - -FUMA = {"sinan": ""} - -FXETARIA = {"ibge": ""} - -FX_ETARIA = {"pni": ""} - -F_AREIA = {"cnes": ""} - -F_CARVAO = {"cnes": ""} - -GANGLIOS = {"sinan": ""} - -GASES = {"sinan": ""} - -GENGIVO = {"sinan": ""} - -GENOT_G = {"sinan": ""} - -GENOT_P = {"sinan": ""} - -GEN_VHC = {"sinan": ""} - -GESPRG1E = {"cnes": ""} - -GESPRG1M = {"cnes": ""} - -GESPRG2E = {"cnes": ""} - -GESPRG2M = {"cnes": ""} - -GESPRG3E = {"cnes": ""} - -GESPRG3M = {"cnes": ""} - -GESPRG4E = {"cnes": ""} - -GESPRG4M = {"cnes": ""} - -GESPRG5E = {"cnes": ""} - -GESPRG5M = {"cnes": ""} - -GESPRG6E = {"cnes": ""} - -GESPRG6M = {"cnes": ""} - -GESTACAO = { - "sim": "", - "sinasc": "", -} - -GESTANTE = {"sinan": ""} - -GESTAO = { - "ciha": "", - "sia": "", - "sih": "", -} - -GESTOR_COD = {"sih": ""} - -GESTOR_CPF = {"sih": ""} - -GESTOR_DT = {"sih": ""} - -GESTOR_TP = {"sih": ""} - -GESTRISCO = {"sih": ""} - -GLAUCOMA = {"sinan": ""} - -GRAVIDEZ = { - "sim": "", - "sinasc": "", -} - -GRAV_AST = {"sinan": ""} - -GRAV_CONSC = {"sinan": ""} - -GRAV_CONV = {"sinan": ""} - -GRAV_ENCH = {"sinan": ""} - -GRAV_EXTRE = {"sinan": ""} - -GRAV_HEMAT = {"sinan": ""} - -GRAV_HIPOT = {"sinan": ""} - -GRAV_INSUF = {"sinan": ""} - -GRAV_MELEN = {"sinan": ""} - -GRAV_METRO = {"sinan": ""} - -GRAV_MIOC = {"sinan": ""} - -GRAV_ORGAO = {"sinan": ""} - -GRAV_PULSO = {"sinan": ""} - -GRAV_SANG = {"sinan": ""} - -GRAV_TAQUI = {"sinan": ""} - -G_D = {"sinan": ""} - -G_M = {"sinan": ""} - -HANSENIASE = {"sinan": ""} - -HAV = {"sinan": ""} - -HA_PAUSA = {"sinan": ""} - -HBC_TOTAL = {"sinan": ""} - -HBSAG = {"sinan": ""} - -HBV = {"sinan": ""} - -HCV = {"sinan": ""} - -HDV = {"sinan": ""} - -HEMATOLOG = {"sinan": ""} - -HEMATURA = {"sinan": ""} - -HEMA_MAIOR = {"sinan": ""} - -HEMA_MENOR = {"sinan": ""} - -HEMO = {"sinan": ""} - -HEMOCULT = {"sinan": ""} - -HEMODIALIS = {"sinan": ""} - -HEMORRAG = {"sinan": ""} - -HEMORRAGI = {"sinan": ""} - -HEMOTERA = {"cnes": ""} - -HEMO_D_1 = {"sinan": ""} - -HEMO_D_2 = {"sinan": ""} - -HEMO_D_3 = {"sinan": ""} - -HEMO_IGG = {"sinan": ""} - -HEMO_IGM = {"sinan": ""} - -HEMO_R1 = {"sinan": ""} - -HEMO_R2 = {"sinan": ""} - -HEMO_R3 = {"sinan": ""} - -HEM_IGG_S1 = {"sinan": ""} - -HEM_IGG_S2 = {"sinan": ""} - -HEM_IGM_S1 = {"sinan": ""} - -HEM_IGM_S2 = {"sinan": ""} - -HEPAESPLE = {"sinan": ""} - -HEPATITA = {"sinan": ""} - -HEPATITB = {"sinan": ""} - -HEPATITE_N = {"sinan": ""} - -HEPATO = {"sinan": ""} - -HEPATOME = {"sinan": ""} - -HEPATOPAT = {"sinan": ""} - -HEPA_ESP = {"sinan": ""} - -HERBIV_DES = {"sinan": ""} - -HEV = {"sinan": ""} - -HIDROCARBO = {"sinan": ""} - -HIDROFOBI = {"sinan": ""} - -HIPEREMIA = {"sinan": ""} - -HIPERTEN = {"sinan": ""} - -HIPERTENSA = {"sinan": ""} - -HIPOREXIA = {"sinan": ""} - -HIPOTENSAO = {"sinan": ""} - -HISTOLOG_N = {"sinan": ""} - -HISTOPA = {"sinan": ""} - -HISTOPATO = {"sinan": ""} - -HISTOPATOL = {"sinan": ""} - -HISTOPA_N = {"sinan": ""} - -HISTORIA = {"sinan": ""} - -HIV = {"sinan": ""} - -HOMONIMO = { - "ciha": "", - "sih": "", -} - -HORAHOSP = {"cnes": ""} - -HORANASC = {"sinasc": ""} - -HORAOBITO = {"sim": ""} - -HORAOUTR = {"cnes": ""} - -HORA_ACID = {"sinan": ""} - -HORA_AMB = {"cnes": ""} - -HORA_JOR = {"sinan": ""} - -HORA_OCOR = {"sinan": ""} - -HORMONIO = {"sinan": ""} - -HOSPITAL = {"sinan": ""} - -HOSPITALIZ = {"sinan": ""} - -HOSP_NSUS = {"cnes": ""} - -HOSP_SUS = {"cnes": ""} - -ICTERICIA = {"sinan": ""} - -IDADE = { - "ciha": "", - "ibge": "", - "sih": "", - "sim": "", -} - -IDADEMAE = { - "sim": "", - "sinan": "", - "sinasc": "", -} - -IDADEMAX = {"sia": ""} - -IDADEMIN = {"sia": ""} - -IDADEPAC = {"sia": ""} - -IDADEPAI = {"sinasc": ""} - -IDADE_MAE = { - "sinan": "", - "sinasc": "", -} - -IDANOMAL = {"sinasc": ""} - -IDENT = {"sih": ""} - -IDENT_GEN = {"sinan": ""} - -IDENT_MICR = {"sinan": ""} - -IDEQUIPE = {"cnes": ""} - -ID_AGRAVO = {"sinan": ""} - -ID_AREA = {"cnes": ""} - -ID_ARTRALG = {"sinan": ""} - -ID_CNS_SUS = {"sinan": ""} - -ID_CONJUNT = {"sinan": ""} - -ID_CORIZA = {"sinan": ""} - -ID_DG_DES = {"sinan": ""} - -ID_DG_NOT = {"sinan": ""} - -ID_DT_RESI = {"sinan": ""} - -ID_ETIOLOG = {"sinan": ""} - -ID_EV_NOT = {"sinan": ""} - -ID_GANGLIO = {"sinan": ""} - -ID_HOSPIT = {"sinan": ""} - -ID_LIQUOR = {"sinan": ""} - -ID_MN_OCOR = {"sinan": ""} - -ID_MN_RESI = {"sinan": ""} - -ID_MUNICIP = {"sinan": ""} - -ID_MUNIC_2 = {"sinan": ""} - -ID_MUNIC_A = {"sinan": ""} - -ID_MUNI_AT = {"sinan": ""} - -ID_MUNI_RE = {"sinan": ""} - -ID_NOTIFIC = {"sinan": ""} - -ID_OCUPACA = {"sinan": ""} - -ID_OCUPA_N = {"sinan": ""} - -ID_OCUP_MA = {"sinan": ""} - -ID_PAIS = {"sinan": ""} - -ID_REGIONA = {"sinan": ""} - -ID_RETRO = {"sinan": ""} - -ID_RE_IGG = {"sinan": ""} - -ID_RE_IGG_ = {"sinan": ""} - -ID_RE_IGM = {"sinan": ""} - -ID_RE_IGM_ = {"sinan": ""} - -ID_RE_IG_1 = {"sinan": ""} - -ID_RE_IG_2 = {"sinan": ""} - -ID_RG_RESI = {"sinan": ""} - -ID_S1_IGG = {"sinan": ""} - -ID_S1_IGG_ = {"sinan": ""} - -ID_S1_IGM = {"sinan": ""} - -ID_S1_IGM_ = {"sinan": ""} - -ID_S1_IG_1 = {"sinan": ""} - -ID_S1_IG_2 = {"sinan": ""} - -ID_S2_IGG = {"sinan": ""} - -ID_S2_IGG_ = {"sinan": ""} - -ID_S2_IGM = {"sinan": ""} - -ID_S2_IGM_ = {"sinan": ""} - -ID_S2_IG_1 = {"sinan": ""} - -ID_S2_IG_2 = {"sinan": ""} - -ID_SANGUE = {"sinan": ""} - -ID_SECRECA = {"sinan": ""} - -ID_SEGM = {"cnes": ""} - -ID_TOSSE = {"sinan": ""} - -ID_UNIDADE = {"sinan": ""} - -ID_UNID_AT = {"sinan": ""} - -ID_URINA = {"sinan": ""} - -IFI = {"sinan": ""} - -IGG_S1 = {"sinan": ""} - -IGG_S2 = {"sinan": ""} - -IGG_T2 = {"sinan": ""} - -IGM_S1 = {"sinan": ""} - -IGM_S2 = {"sinan": ""} - -IGM_T1 = {"sinan": ""} - -IMPLANTA = {"sinan": ""} - -IMUNO = { - "pni": "", - "sinan": "", -} - -IMUNOH = {"sinan": ""} - -IMUNOHIST = {"sinan": ""} - -IMUNOH_N = {"sinan": ""} - -IMUNO_DIRE = {"sinan": ""} - -IMUNO_INDI = {"sinan": ""} - -IMU_HEP_B = {"sinan": ""} - -IMU_IGG_S1 = {"sinan": ""} - -IMU_IGG_S2 = {"sinan": ""} - -IMU_IGM_S1 = {"sinan": ""} - -IMU_IGM_S2 = {"sinan": ""} - -INAL_CRACK = {"sinan": ""} - -INDIGENA = {"cnes": ""} - -INDIVIDUAL = {"sinan": ""} - -IND_NSUS = {"cnes": ""} - -IND_SUS = {"cnes": ""} - -IND_VDRL = {"sih": ""} - -INESPECIF = {"sinan": ""} - -INFAN_JUV = {"sinan": ""} - -INFECCIOSO = {"sinan": ""} - -INFEHOSP = {"sih": ""} - -INFERIORES = {"sinan": ""} - -INFILTRA = {"sinan": ""} - -INICIO = {"sia": ""} - -INJETAVEIS = {"sinan": ""} - -INSC_PN = {"sih": ""} - -INSTITUCIO = {"sinan": ""} - -INSTRMAE = {"sim": ""} - -INSTRPAI = {"sim": ""} - -INSTRU = {"sih": ""} - -INSTRUCAO = {"sim": ""} - -INSTR_MAE = {"sinasc": ""} - -INSUFICIEN = {"sinan": ""} - -INTOX_CHUM = {"sinan": ""} - -INTOX_MERC = {"sinan": ""} - -INTOX_META = {"sinan": ""} - -INT_TEMPO = {"sinan": ""} - -IN_AIDS = {"sinan": ""} - -IN_TP_VAL = {"sih": ""} - -IN_VINCULA = {"sinan": ""} - -IONIZANTES = {"sinan": ""} - -ISOLAMENTO = {"sinan": ""} - -ISONIAZIDA = {"sinan": ""} - -KOTELCHUCK = {"sinasc": ""} - -LABC_DT = {"sinan": ""} - -LABC_DT_1 = {"sinan": ""} - -LABC_DT_2 = {"sinan": ""} - -LABC_EVIDE = {"sinan": ""} - -LABC_IGG = {"sinan": ""} - -LABC_LIQUO = {"sinan": ""} - -LABC_LIQ_1 = {"sinan": ""} - -LABC_SANGU = {"sinan": ""} - -LABC_TITUL = {"sinan": ""} - -LABC_TIT_1 = {"sinan": ""} - -LABC_TIT_2 = {"sinan": ""} - -LAB_AGLIQU = {"sinan": ""} - -LAB_AGSANG = {"sinan": ""} - -LAB_ASPECT = {"sinan": ""} - -LAB_ATIPIC = {"sinan": ""} - -LAB_BCESCA = {"sinan": ""} - -LAB_BCLESA = {"sinan": ""} - -LAB_BCLIQU = {"sinan": ""} - -LAB_BCSANG = {"sinan": ""} - -LAB_BD = {"sinan": ""} - -LAB_BI = {"sinan": ""} - -LAB_BILATE = {"sinan": ""} - -LAB_BT = {"sinan": ""} - -LAB_CELEBR = {"sinan": ""} - -LAB_CILIQU = {"sinan": ""} - -LAB_CISANG = {"sinan": ""} - -LAB_CLOR = {"sinan": ""} - -LAB_COLHEU = {"sinan": ""} - -LAB_CONF = {"sinan": ""} - -LAB_CONFIR = {"sinan": ""} - -LAB_CON_F = {"sinan": ""} - -LAB_CREATI = {"sinan": ""} - -LAB_CTESCA = {"sinan": ""} - -LAB_CTLESA = {"sinan": ""} - -LAB_CTLIQU = {"sinan": ""} - -LAB_CTSANG = {"sinan": ""} - -LAB_CULTUR = {"sinan": ""} - -LAB_DATA_C = {"sinan": ""} - -LAB_DERRAM = {"sinan": ""} - -LAB_DIFUSO = {"sinan": ""} - -LAB_DT3 = {"sinan": ""} - -LAB_DTPUNC = {"sinan": ""} - -LAB_DT_1 = {"sinan": ""} - -LAB_DT_2 = {"sinan": ""} - -LAB_DT_3 = {"sinan": ""} - -LAB_DT_C1 = {"sinan": ""} - -LAB_DT_CEN = {"sinan": ""} - -LAB_DT_E_1 = {"sinan": ""} - -LAB_DT_F1 = {"sinan": ""} - -LAB_DT_L_1 = {"sinan": ""} - -LAB_DT_L_2 = {"sinan": ""} - -LAB_DT_NLE = {"sinan": ""} - -LAB_DT_R1 = {"sinan": ""} - -LAB_DT_RE1 = {"sinan": ""} - -LAB_ELIS_1 = {"sinan": ""} - -LAB_ELIS_2 = {"sinan": ""} - -LAB_EOSI = {"sinan": ""} - -LAB_ESFR = {"sinan": ""} - -LAB_E_D_1 = {"sinan": ""} - -LAB_GLICO = {"sinan": ""} - -LAB_HEMA = {"sinan": ""} - -LAB_HEMATO = {"sinan": ""} - -LAB_HEMA_N = {"sinan": ""} - -LAB_HEMO = {"sinan": ""} - -LAB_HISTOP = {"sinan": ""} - -LAB_IGG = {"sinan": ""} - -LAB_IGG_R = {"sinan": ""} - -LAB_IGM = {"sinan": ""} - -LAB_IGM_R = {"sinan": ""} - -LAB_IMUNO = {"sinan": ""} - -LAB_INTEST = {"sinan": ""} - -LAB_IRM = {"sinan": ""} - -LAB_ISFEZE = {"sinan": ""} - -LAB_ISLIQU = {"sinan": ""} - -LAB_LEUCO = {"sinan": ""} - -LAB_LEUC_N = {"sinan": ""} - -LAB_LINFO = {"sinan": ""} - -LAB_LOCAL = {"sinan": ""} - -LAB_L_CEL1 = {"sinan": ""} - -LAB_L_CEL2 = {"sinan": ""} - -LAB_L_CL1 = {"sinan": ""} - -LAB_L_CL2 = {"sinan": ""} - -LAB_L_C_DE = {"sinan": ""} - -LAB_L_GLI1 = {"sinan": ""} - -LAB_L_GLI2 = {"sinan": ""} - -LAB_L_LIN1 = {"sinan": ""} - -LAB_L_LIN2 = {"sinan": ""} - -LAB_L_OUT = {"sinan": ""} - -LAB_L_PRO1 = {"sinan": ""} - -LAB_L_PRO2 = {"sinan": ""} - -LAB_L_S_DE = {"sinan": ""} - -LAB_MACRO = {"sinan": ""} - -LAB_MATE_N = {"sinan": ""} - -LAB_MEDULA = {"sinan": ""} - -LAB_METODO = {"sinan": ""} - -LAB_MET_D = {"sinan": ""} - -LAB_MICRO = {"sinan": ""} - -LAB_MICRON = {"sinan": ""} - -LAB_MICR_1 = {"sinan": ""} - -LAB_MICR_2 = {"sinan": ""} - -LAB_MONO = {"sinan": ""} - -LAB_NEUTRO = {"sinan": ""} - -LAB_OUTRO = {"sinan": ""} - -LAB_OUT_D = {"sinan": ""} - -LAB_OUT_E = {"sinan": ""} - -LAB_PARASI = {"sinan": ""} - -LAB_PARTO = {"sinan": ""} - -LAB_PCESCA = {"sinan": ""} - -LAB_PCLESA = {"sinan": ""} - -LAB_PCLIQU = {"sinan": ""} - -LAB_PCR_1 = {"sinan": ""} - -LAB_PCR_2 = {"sinan": ""} - -LAB_PCR_3 = {"sinan": ""} - -LAB_PCSANG = {"sinan": ""} - -LAB_PLAQUE = {"sinan": ""} - -LAB_POTASS = {"sinan": ""} - -LAB_PROD1 = {"sinan": ""} - -LAB_PROD2 = {"sinan": ""} - -LAB_PROT = {"sinan": ""} - -LAB_PROVAS = {"sinan": ""} - -LAB_PUNCAO = {"sinan": ""} - -LAB_Q_F = {"sinan": ""} - -LAB_RADIOL = {"sinan": ""} - -LAB_REALIZ = {"sinan": ""} - -LAB_RESULT = {"sinan": ""} - -LAB_RES_B = {"sinan": ""} - -LAB_RES_F1 = {"sinan": ""} - -LAB_RES_F2 = {"sinan": ""} - -LAB_RES_F3 = {"sinan": ""} - -LAB_RTPCR = {"sinan": ""} - -LAB_R_1 = {"sinan": ""} - -LAB_R_2 = {"sinan": ""} - -LAB_SORO = {"sinan": ""} - -LAB_SOROAG = {"sinan": ""} - -LAB_SOR_DE = {"sinan": ""} - -LAB_S_1 = {"sinan": ""} - -LAB_S_2 = {"sinan": ""} - -LAB_S_3 = {"sinan": ""} - -LAB_S_4 = {"sinan": ""} - -LAB_S_5 = {"sinan": ""} - -LAB_TGO = {"sinan": ""} - -LAB_TGO_D = {"sinan": ""} - -LAB_TGP = {"sinan": ""} - -LAB_TGP_D = {"sinan": ""} - -LAB_TITU_2 = {"sinan": ""} - -LAB_TRIAGE = {"sinan": ""} - -LAB_TROMBO = {"sinan": ""} - -LAB_UF = {"sinan": ""} - -LAB_UREIA = {"sinan": ""} - -LAB_VACINA = {"sinan": ""} - -LAB_VAC_DE = {"sinan": ""} - -LACO = {"sinan": ""} - -LACO_N = {"sinan": ""} - -LAMBEDURA = {"sinan": ""} - -LAVOURA = {"sinan": ""} - -LEITE = {"sinan": ""} - -LEITHOSP = {"cnes": ""} - -LESAO = {"sinan": ""} - -LESAO_CORP = {"sinan": ""} - -LESAO_DES = {"sinan": ""} - -LESAO_ESPE = {"sinan": ""} - -LESAO_NAT = {"sinan": ""} - -LESOES = {"sinan": ""} - -LES_AUTOP = {"sinan": ""} - -LEUCOPENIA = {"sinan": ""} - -LIMITA_MOV = {"sinan": ""} - -LINFADENO = {"sinan": ""} - -LINHAA = {"sim": ""} - -LINHAB = {"sim": ""} - -LINHAC = {"sim": ""} - -LINHAD = {"sim": ""} - -LINHAII = {"sim": ""} - -LOCACID = {"sim": ""} - -LOCAL_ACID = {"sinan": ""} - -LOCAL_ESPE = {"sinan": ""} - -LOCAL_OCOR = { - "sinan": "", - "sinasc": "", -} - -LOCA_MID_N = {"sinan": ""} - -LOCA_MIE_N = {"sinan": ""} - -LOCA_MSD_N = {"sinan": ""} - -LOCA_MSE_N = {"sinan": ""} - -LOCNASC = {"sinasc": ""} - -LOCOCOR = {"sim": ""} - -LOC_EXPO = {"sinan": ""} - -LOC_EXP_DE = {"sinan": ""} - -LOC_INF = {"sinan": ""} - -LOC_REALIZ = {"sia": ""} - -LOTE1 = {"sinan": ""} - -LOTE2 = {"sinan": ""} - -LOTE_VAC = {"sinan": ""} - -LUVA = {"sinan": ""} - -MAECHAGA = {"sinan": ""} - -MAIS_6HS = {"sinan": ""} - -MAIS_TRAB = {"sinan": ""} - -MANIFESTA = {"sinan": ""} - -MANIPULA = {"sinan": ""} - -MANI_HEMOR = {"sinan": ""} - -MAOS_N = {"sinan": ""} - -MAPORTAR = {"cnes": ""} - -MAQ_OUTR = {"cnes": ""} - -MAQ_PROP = {"cnes": ""} - -MARCA_UCI = {"sih": ""} - -MARCA_UTI = {"sih": ""} - -MASCARA = {"sinan": ""} - -MATBIOLOGI = {"sinan": ""} - -MATERIAL = {"sinan": ""} - -MAT_ORG = {"sinan": ""} - -MAT_ORG_DE = {"sinan": ""} - -MAX_INC = {"sinan": ""} - -MAX_ST_INC = {"sinan": ""} - -MCLI_LOCAL = {"sinan": ""} - -MCLI_SIST = {"sinan": ""} - -MEDICA = {"sinan": ""} - -MEDICAMENT = {"sinan": ""} - -MED_BLOQUE = {"sinan": ""} - -MED_CASO_S = {"sinan": ""} - -MED_CONTR = {"sinan": ""} - -MED_DT_EVO = {"sinan": ""} - -MED_DT_QUI = {"sinan": ""} - -MED_IDEN_C = {"sinan": ""} - -MED_MATERI = {"sinan": ""} - -MED_NUCOMU = {"sinan": ""} - -MED_OUTRO = {"sinan": ""} - -MED_PREVEN = {"sinan": ""} - -MED_QUAN_C = {"sinan": ""} - -MED_QUAN_M = {"sinan": ""} - -MED_QUAN_P = {"sinan": ""} - -MED_QUIMIO = {"sinan": ""} - -MEFLOQ = {"sinan": ""} - -MENINGO = {"sinan": ""} - -MENINGOE = {"sinan": ""} - -MENOR_5ANO = {"sinan": ""} - -MENOS_MOV = {"sinan": ""} - -MENTAL = {"sinan": ""} - -MES = { - "pni": "", - "sih": "", -} - -MESPRENAT = {"sinasc": ""} - -MES_CMPT = { - "ciha": "", - "sih": "", -} - -METAL = {"sinan": ""} - -METRO = {"sinan": ""} - -MIALGIA = {"sinan": ""} - -MICRO1_S1 = {"sinan": ""} - -MICRO1_S_2 = {"sinan": ""} - -MICRO1_T_1 = {"sinan": ""} - -MICRO1_T_2 = {"sinan": ""} - -MICRO2_S1 = {"sinan": ""} - -MICRO2_S_2 = {"sinan": ""} - -MICRO2_T_1 = {"sinan": ""} - -MICRO2_T_2 = {"sinan": ""} - -MICROCEFA = {"sinan": ""} - -MICRO_HEMA = {"sinan": ""} - -MICR_REG = {"cnes": ""} - -MIGRADO_W = {"sinan": ""} - -MINTERNA = {"sinan": ""} - -MIN_ACID = {"sinan": ""} - -MIN_JOR = {"sinan": ""} - -MIOCARDI = {"sinan": ""} - -MNDIF = {"sia": ""} - -MN_IND = {"sia": ""} - -MOAGEM_N = {"sinan": ""} - -MODALIDADE = {"ciha": ""} - -MODODETECT = {"sinan": ""} - -MODOENTR = {"sinan": ""} - -MORDEDURA = {"sinan": ""} - -MORTE = { - "ciha": "", - "sih": "", -} - -MORTEPARTO = {"sim": ""} - -MOTDESAT = {"cnes": ""} - -MOT_COB = {"sia": ""} - -MPU = {"sinan": ""} - -MTRANSFU = {"sinan": ""} - -MUCOSA = {"sinan": ""} - -MUDA_TRAB = {"sinan": ""} - -MUNCOD = {"ibge": ""} - -MUNIC = {"pni": ""} - -MUNICIPIO = {"sinan": ""} - -MUNIC_LOC = {"sih": ""} - -MUNIC_MOV = { - "ciha": "", - "sih": "", -} - -MUNIC_RES = { - "ciha": "", - "ibge": "", - "sih": "", -} - -MUNIOCOR = {"sim": ""} - -MUNIRES = {"sim": ""} - -MUNIRESAT = {"sinan": ""} - -MUNI_MAE = {"sinasc": ""} - -MUNI_OCOR = {"sinasc": ""} - -MUNPAC = {"sia": ""} - -MUN_1 = {"sinan": ""} - -MUN_2 = {"sinan": ""} - -MUN_3 = {"sinan": ""} - -MUN_ACID = {"sinan": ""} - -MUN_ATENDE = {"sinan": ""} - -MUN_DES1 = {"sinan": ""} - -MUN_DES2 = {"sinan": ""} - -MUN_DES3 = {"sinan": ""} - -MUN_EMP = {"sinan": ""} - -MUN_HOSP = {"sinan": ""} - -MUN_ING = {"sinan": ""} - -MUN_MOV = {"sih": ""} - -MUN_PRE_NA = {"sinan": ""} - -MUN_RES = {"sih": ""} - -MUN_TRANSF = {"sinan": ""} - -MUSCULAR = {"sinan": ""} - -NACIONAL = { - "ciha": "", - "sih": "", -} - -NACION_PAC = {"sia": ""} - -NAO_IONIZA = {"sinan": ""} - -NASC = { - "ciha": "", - "sih": "", -} - -NATURAL = {"sim": ""} - -NATURALMAE = {"sinasc": ""} - -NATUREZA = { - "ciha": "", - "cnes": "", - "sih": "", -} - -NAT_JUR = { - "cnes": "", - "sia": "", - "sih": "", -} - -NAUSEA = {"sinan": ""} - -NAUSEAS = {"sinan": ""} - -NDUPLIC = {"sinan": ""} - -NDUPLIC_N = {"sinan": ""} - -NECROPSIA = {"sim": ""} - -NECROSE = {"sinan": ""} - -NENHUM = {"sinan": ""} - -NEOPLASICO = {"sinan": ""} - -NERVOSAFET = {"sinan": ""} - -NIQUEL = {"sinan": ""} - -NIVATE_A = {"cnes": ""} - -NIVATE_H = {"cnes": ""} - -NIV_DEP = {"cnes": ""} - -NIV_HIER = {"cnes": ""} - -NM_ANTIBIO = {"sinan": ""} - -NM_MUNIC_H = {"sinan": ""} - -NM_MUN_HOS = {"sinan": ""} - -NM_OUT_COM = {"sinan": ""} - -NM_OUT_SIN = {"sinan": ""} - -NM_SIN_OUT = {"sinan": ""} - -NOCOLINF = {"sinan": ""} - -NOMEAREA = {"cnes": ""} - -NOMEFANT = {"sih": ""} - -NOMEPROF = {"cnes": ""} - -NOME_BACT = {"sinan": ""} - -NOME_EQP = {"cnes": ""} - -NOME_PARAS = {"sinan": ""} - -NOME_VIRUS = {"sinan": ""} - -NOPROPIN = {"sinan": ""} - -NOVO = {"sinasc": ""} - -NO_ATENOUT = {"sinan": ""} - -NO_COBOUTR = {"sinan": ""} - -NO_OUPARTO = {"sinan": ""} - -NO_OUTRAS = {"sinan": ""} - -NU10_19_N = {"sinan": ""} - -NU1_4_F_NU = {"sinan": ""} - -NU5_9_F_NU = {"sinan": ""} - -NUATEND = {"sinan": ""} - -NUCONSOME = {"sinan": ""} - -NUDIASINF = {"sim": ""} - -NUDIASOBCO = {"sim": ""} - -NUDIASOBIN = {"sim": ""} - -NULEITOS = {"cnes": ""} - -NUMCRIPOB = {"ibge": ""} - -NUMCRIPOBX = {"ibge": ""} - -NUMDESOCUP = {"ibge": ""} - -NUMERODN = { - "sim": "", - "sinasc": "", -} - -NUMERODV = {"sinasc": ""} - -NUMEROLOTE = { - "sim": "", - "sinasc": "", -} - -NUMEXPORT = {"sim": ""} - -NUMPOBRES = {"ibge": ""} - -NUMPOBRESX = {"ibge": ""} - -NUMREGCART = { - "sim": "", - "sinasc": "", -} - -NUMRENDA = {"ibge": ""} - -NUMTRABINF = {"ibge": ""} - -NUM_CON_N = {"sinan": ""} - -NUM_DOSES = {"sinan": ""} - -NUM_ENVOLV = {"sinan": ""} - -NUM_EXPORT = {"sinasc": ""} - -NUM_FILHOS = {"sih": ""} - -NUM_PROC = {"sih": ""} - -NUTEMPO = {"sinan": ""} - -NUTEMPORIS = {"sinan": ""} - -NU_10_19 = {"sinan": ""} - -NU_10_19IG = {"sinan": ""} - -NU_10_19_M = {"sinan": ""} - -NU_1_4_IGN = {"sinan": ""} - -NU_1_4_NU = {"sinan": ""} - -NU_1_4_TOT = {"sinan": ""} - -NU_1_F_NU = {"sinan": ""} - -NU_1_IGN = {"sinan": ""} - -NU_1_M_NU = {"sinan": ""} - -NU_1_TOT_N = {"sinan": ""} - -NU_20_49 = {"sinan": ""} - -NU_20_49IG = {"sinan": ""} - -NU_20_49_F = {"sinan": ""} - -NU_20_49_N = {"sinan": ""} - -NU_50_F_NU = {"sinan": ""} - -NU_50_IGN = {"sinan": ""} - -NU_50_M_NU = {"sinan": ""} - -NU_50_TOT = {"sinan": ""} - -NU_5_9_IGN = {"sinan": ""} - -NU_5_9_NU = {"sinan": ""} - -NU_5_9_TOT = {"sinan": ""} - -NU_ABDOM_N = {"sinan": ""} - -NU_AFAST = {"sinan": ""} - -NU_AMPOLAS = {"sinan": ""} - -NU_AMPOL_1 = {"sinan": ""} - -NU_AMPOL_3 = {"sinan": ""} - -NU_AMPOL_4 = {"sinan": ""} - -NU_AMPOL_6 = {"sinan": ""} - -NU_AMPOL_8 = {"sinan": ""} - -NU_AMPOL_9 = {"sinan": ""} - -NU_AMPO_5 = {"sinan": ""} - -NU_AMPO_7 = {"sinan": ""} - -NU_ANO = {"sinan": ""} - -NU_A_ALIM = {"sinan": ""} - -NU_A_CLINI = {"sinan": ""} - -NU_A_NUM_1 = {"sinan": ""} - -NU_A_NUM_2 = {"sinan": ""} - -NU_A_NUM_3 = {"sinan": ""} - -NU_CASO = {"sinan": ""} - -NU_CASOEXA = {"sinan": ""} - -NU_CASOPOS = {"sinan": ""} - -NU_CEFAL_N = {"sinan": ""} - -NU_CELULA = {"sinan": ""} - -NU_CLI_NUM = {"sinan": ""} - -NU_COMU_EX = {"sinan": ""} - -NU_CONTATO = {"sinan": ""} - -NU_DIARR_N = {"sinan": ""} - -NU_DOSE = {"sinan": ""} - -NU_ENTR = {"sinan": ""} - -NU_ENT_DOE = {"sinan": ""} - -NU_FEBRE_N = {"sinan": ""} - -NU_F_TOT = {"sinan": ""} - -NU_F_TOT_N = {"sinan": ""} - -NU_GESTA = {"sinan": ""} - -NU_IDADE = {"sinan": ""} - -NU_IDADE_N = {"sinan": ""} - -NU_IGN_NU = {"sinan": ""} - -NU_IGRA_NU = {"sinan": ""} - -NU_IG_F_NU = {"sinan": ""} - -NU_IG_IGN = {"sinan": ""} - -NU_INCUB_M = {"sinan": ""} - -NU_INC_ME = {"sinan": ""} - -NU_LESOES = {"sinan": ""} - -NU_LOTE = {"sinan": ""} - -NU_LOTE_H = {"sinan": ""} - -NU_LOTE_I = {"sinan": ""} - -NU_LOTE_IA = {"sinan": ""} - -NU_LOTE_V = {"sinan": ""} - -NU_NAUSE_P = {"sinan": ""} - -NU_NEURO_N = {"sinan": ""} - -NU_NOTIFIC = {"sinan": ""} - -NU_NUM_2 = {"sinan": ""} - -NU_NUM_3 = {"sinan": ""} - -NU_OBITO = {"sinan": ""} - -NU_OUTRO_N = {"sinan": ""} - -NU_PA_TOT = {"sia": ""} - -NU_PROTEI = {"sinan": ""} - -NU_RESU_3 = {"sinan": ""} - -NU_SEMA_EP = {"sinan": ""} - -NU_TOT = {"sinan": ""} - -NU_TOT_HOS = {"sinan": ""} - -NU_TOT_IGN = {"sinan": ""} - -NU_TO_F_NU = {"sinan": ""} - -NU_TRAB = {"sinan": ""} - -NU_VOMTO_N = {"sinan": ""} - -NU_VPA_TOT = {"sia": ""} - -N_AIH = {"sih": ""} - -N_DIAR = {"sinan": ""} - -N_VOMITO = {"sinan": ""} - -OBITOFE1 = {"sim": ""} - -OBITOFE2 = {"sim": ""} - -OBITOGRAV = {"sim": ""} - -OBITOPARTO = {"sim": ""} - -OBITOPUERP = {"sim": ""} - -OBSERVACAO = {"sinan": ""} - -OCULOS = {"sinan": ""} - -OCUP = {"sim": ""} - -OCUPACAO = { - "sim": "", - "sinan": "", -} - -OCUPACIO = {"sinan": ""} - -OCUPMAE = {"sim": ""} - -OCUPPAI = {"sim": ""} - -OLEOS = {"sinan": ""} - -OLIGURIA = {"sinan": ""} - -ORAL = {"sinan": ""} - -ORGEXPED = {"cnes": ""} - -ORIENT_SEX = {"sinan": ""} - -ORIGEM = { - "sim": "", - "sinan": "", - "sinasc": "", -} - -ORIGEM_PAC = {"sia": ""} - -ORTV1050 = {"cnes": ""} - -ORV50150 = {"cnes": ""} - -OSMOSE_R = {"cnes": ""} - -OSSEA = {"sinan": ""} - -OUTRAS = {"sinan": ""} - -OUTRAS_DES = {"sinan": ""} - -OUTRA_ATIV = {"sinan": ""} - -OUTRA_DST = {"sinan": ""} - -OUTRO = {"sinan": ""} - -OUTROANI = {"sinan": ""} - -OUTROS = {"sinan": ""} - -OUTROS_DES = {"sinan": ""} - -OUTROS_ESP = {"sinan": ""} - -OUTROS_M = {"sinan": ""} - -OUTROS_M_D = {"sinan": ""} - -OUTRO_ARV = {"sinan": ""} - -OUTRO_DES = {"sinan": ""} - -OUTRO_DOE = {"sinan": ""} - -OUTRO_ESP = {"sinan": ""} - -OUTRO_EX = {"sinan": ""} - -OUTRO_EXP = {"sinan": ""} - -OUTRO_S = {"sinan": ""} - -OUTRO_SIN = {"sinan": ""} - -OUTRO_S_D = {"sinan": ""} - -OUTR_ATI_D = {"sinan": ""} - -OUTR_D1 = {"sinan": ""} - -OUTR_D2 = {"sinan": ""} - -OUTR_D3 = {"sinan": ""} - -OUTR_R1 = {"sinan": ""} - -OUTR_R2 = {"sinan": ""} - -OUTR_R3 = {"sinan": ""} - -OUT_AGENTE = {"sinan": ""} - -OUT_AGRAVO = {"sinan": ""} - -OUT_ARV_ES = {"sinan": ""} - -OUT_CONTAT = {"sinan": ""} - -OUT_DOE_DE = {"sinan": ""} - -OUT_EXAME = {"sinan": ""} - -OUT_EXP_DE = {"sinan": ""} - -OUT_MEDIC = {"sinan": ""} - -OUT_TRAT = {"cnes": ""} - -OUT_VEZES = {"sinan": ""} - -OUT_VINCUL = {"sinan": ""} - -OUT_VIRUS = {"sinan": ""} - -OV150500 = {"cnes": ""} - -PAIS_EXP = {"sinan": ""} - -PALIDEZ = {"sinan": ""} - -PALQ_MAIOR = {"sinan": ""} - -PARALISIA = {"sinan": ""} - -PARASITA = {"sinan": ""} - -PARASITO = {"sinan": ""} - -PARESTESI = {"sinan": ""} - -PARIDADE = {"sinasc": ""} - -PARTO = { - "sim": "", - "sinasc": "", -} - -PART_CORP1 = {"sinan": ""} - -PART_CORP2 = {"sinan": ""} - -PART_CORP3 = {"sinan": ""} - -PAR_ANTIDU = {"sinan": ""} - -PAR_DT_PAR = {"sinan": ""} - -PAR_EVOLUC = {"sinan": ""} - -PAR_INICPR = {"sinan": ""} - -PAR_TIPO = {"sinan": ""} - -PAR_UFPART = {"sinan": ""} - -PA_ALTA = {"sia": ""} - -PA_AUTORIZ = {"sia": ""} - -PA_CATEND = {"sia": ""} - -PA_CBOCOD = {"sia": ""} - -PA_CID = {"sia": ""} - -PA_CIDCAS = {"sia": ""} - -PA_CIDPRI = {"sia": ""} - -PA_CIDSEC = {"sia": ""} - -PA_CLASS_S = {"sia": ""} - -PA_CMP = {"sia": ""} - -PA_CNPJCPF = {"sia": ""} - -PA_CNPJMNT = {"sia": ""} - -PA_CNPJ_CC = {"sia": ""} - -PA_CNSMED = {"sia": ""} - -PA_CODESP = {"sia": ""} - -PA_CODOCO = {"sia": ""} - -PA_CODPRO = {"sia": ""} - -PA_CODUNI = {"sia": ""} - -PA_CONDIC = {"sia": ""} - -PA_DATPR = {"sia": ""} - -PA_DATREF = {"sia": ""} - -PA_DES1 = {"sinan": ""} - -PA_DES2 = {"sinan": ""} - -PA_DES3 = {"sinan": ""} - -PA_DIF_VAL = {"sia": ""} - -PA_DOCORIG = {"sia": ""} - -PA_ENCERR = {"sia": ""} - -PA_EQUIPE = {"sia": ""} - -PA_ETNIA = {"sia": ""} - -PA_FLER = {"sia": ""} - -PA_FLIDADE = {"sia": ""} - -PA_FLQT = {"sia": ""} - -PA_FNTORC = {"sia": ""} - -PA_FXETAR = {"sia": ""} - -PA_GESTAO = {"sia": ""} - -PA_IDADE = {"sia": ""} - -PA_INCOUT = {"sia": ""} - -PA_INCURG = {"sia": ""} - -PA_INDICA = {"sia": ""} - -PA_INE = {"sia": ""} - -PA_MNDIF = {"sia": ""} - -PA_MN_IND = {"sia": ""} - -PA_MORFOL = {"sia": ""} - -PA_MOTSAI = {"sia": ""} - -PA_MUNAT = {"sia": ""} - -PA_MUNPCN = {"sia": ""} - -PA_MVM = {"sia": ""} - -PA_NAT_JUR = {"sia": ""} - -PA_NH = {"sia": ""} - -PA_NIVCPL = {"sia": ""} - -PA_NUMAPA = {"sia": ""} - -PA_OBITO = {"sia": ""} - -PA_PERMAN = {"sia": ""} - -PA_PROC_ID = {"sia": ""} - -PA_QTDAPR = {"sia": ""} - -PA_QTDPRO = {"sia": ""} - -PA_RACACOR = {"sia": ""} - -PA_RCB = {"sia": ""} - -PA_RCBDF = {"sia": ""} - -PA_REGCT = {"sia": ""} - -PA_SEXO = {"sia": ""} - -PA_SRV = {"sia": ""} - -PA_SRV_C = {"sia": ""} - -PA_SUBFIN = {"sia": ""} - -PA_TIPATE = {"sia": ""} - -PA_TIPPRE = {"sia": ""} - -PA_TIPPRO = {"sia": ""} - -PA_TPFIN = {"sia": ""} - -PA_TPUPS = {"sia": ""} - -PA_TP_EQP = {"sia": ""} - -PA_TRANSF = {"sia": ""} - -PA_UFDIF = {"sia": ""} - -PA_UFMUN = {"sia": ""} - -PA_VALAPR = {"sia": ""} - -PA_VALPRO = {"sia": ""} - -PA_VL_CF = {"sia": ""} - -PA_VL_CL = {"sia": ""} - -PA_VL_INC = {"sia": ""} - -PCRUZ = {"sinan": ""} - -PELE_INTEG = {"sinan": ""} - -PELE_NAO_I = {"sinan": ""} - -PEN_ANAL = {"sinan": ""} - -PEN_ORAL = {"sinan": ""} - -PEN_VAGINA = {"sinan": ""} - -PERCUTANEA = {"sinan": ""} - -PERFURA = {"sinan": ""} - -PERICARDI = {"sinan": ""} - -PERIODO = {"sinan": ""} - -PERMANEN = {"sia": ""} - -PES = {"sinan": ""} - -PESCOU_N = {"sinan": ""} - -PESO = { - "sim": "", - "sinan": "", - "sinasc": "", -} - -PESONASC = {"sim": ""} - -PETEQUIAS = {"sinan": ""} - -PETEQUIA_N = {"sinan": ""} - -PF_PJ = {"cnes": ""} - -PIRAZINAMI = {"sinan": ""} - -PLANJ_RD = {"cnes": ""} - -PLAQ_MENOR = {"sinan": ""} - -PLASMATICO = {"sinan": ""} - -PLEURAL = {"sinan": ""} - -PMALARIA = {"sinan": ""} - -PMM = {"sinan": ""} - -POEIRAS = {"sinan": ""} - -POE_ABRASI = {"sinan": ""} - -POE_MISTA = {"sinan": ""} - -POE_ORGANI = {"sinan": ""} - -POLIADENO = {"sinan": ""} - -POP = {"pni": ""} - -POPALFAB = {"ibge": ""} - -POPDEPEND = {"ibge": ""} - -POPGERAL = {"cnes": ""} - -POPNALFAB = {"ibge": ""} - -POPTOT = {"ibge": ""} - -POPULACAO = {"ibge": ""} - -POP_IMIG = {"sinan": ""} - -POP_LIBER = {"sinan": ""} - -POP_RUA = {"sinan": ""} - -POP_SAUDE = {"sinan": ""} - -PORTARIA = {"cnes": ""} - -POS_EXPOS = {"sinan": ""} - -PREFIXODN = {"sinasc": ""} - -PREMIOS = {"sinan": ""} - -PRESENCA = {"sinan": ""} - -PRE_ANTRET = {"sinan": ""} - -PRE_DT_RET = {"sinan": ""} - -PRE_EXPOS = {"sinan": ""} - -PRE_MUNIPA = {"sinan": ""} - -PRE_MUNIRE = {"sinan": ""} - -PRE_NATAL = {"sinasc": ""} - -PRE_PRENAT = {"sinan": ""} - -PRE_UFREL = {"sinan": ""} - -PRIMAQ = {"sinan": ""} - -PROC_ABORT = {"sinan": ""} - -PROC_CONTR = {"sinan": ""} - -PROC_DST = {"sinan": ""} - -PROC_HEPB = {"sinan": ""} - -PROC_HIV = {"sinan": ""} - -PROC_ID = {"sia": ""} - -PROC_REA = { - "ciha": "", - "sih": "", -} - -PROC_SANG = {"sinan": ""} - -PROC_SEMEN = {"sinan": ""} - -PROC_SOLIC = {"sih": ""} - -PROC_VAGIN = {"sinan": ""} - -PROFNSUS = {"cnes": ""} - -PROFUNDO = {"sinan": ""} - -PROF_SUS = {"cnes": ""} - -PRONASCI = {"cnes": ""} - -PROSTACAO = {"sinan": ""} - -PROVA_BIOL = {"sinan": ""} - -PSICO_FARM = {"sinan": ""} - -PTRANSFU = {"sinan": ""} - -PULSO = {"sinan": ""} - -PURPURA = {"sinan": ""} - -PUSUARIO = {"sinan": ""} - -P_ATIVO_1 = {"sinan": ""} - -P_ATIVO_2 = {"sinan": ""} - -P_ATIVO_3 = {"sinan": ""} - -QTDATE = {"sia": ""} - -QTDFILMORT = { - "sim": "", - "sinasc": "", -} - -QTDFILVIVO = { - "sim": "", - "sinasc": "", -} - -QTDGESTANT = {"sinasc": ""} - -QTDPARTCES = {"sinasc": ""} - -QTDPARTNOR = {"sinasc": ""} - -QTDPCN = {"sia": ""} - -QTINST01 = {"cnes": ""} - -QTINST02 = {"cnes": ""} - -QTINST03 = {"cnes": ""} - -QTINST04 = {"cnes": ""} - -QTINST05 = {"cnes": ""} - -QTINST06 = {"cnes": ""} - -QTINST07 = {"cnes": ""} - -QTINST08 = {"cnes": ""} - -QTINST09 = {"cnes": ""} - -QTINST10 = {"cnes": ""} - -QTINST11 = {"cnes": ""} - -QTINST12 = {"cnes": ""} - -QTINST13 = {"cnes": ""} - -QTINST14 = {"cnes": ""} - -QTINST15 = {"cnes": ""} - -QTINST16 = {"cnes": ""} - -QTINST17 = {"cnes": ""} - -QTINST18 = {"cnes": ""} - -QTINST19 = {"cnes": ""} - -QTINST20 = {"cnes": ""} - -QTINST21 = {"cnes": ""} - -QTINST22 = {"cnes": ""} - -QTINST23 = {"cnes": ""} - -QTINST24 = {"cnes": ""} - -QTINST25 = {"cnes": ""} - -QTINST26 = {"cnes": ""} - -QTINST27 = {"cnes": ""} - -QTINST28 = {"cnes": ""} - -QTINST29 = {"cnes": ""} - -QTINST30 = {"cnes": ""} - -QTINST31 = {"cnes": ""} - -QTINST32 = {"cnes": ""} - -QTINST33 = {"cnes": ""} - -QTINST34 = {"cnes": ""} - -QTINST35 = {"cnes": ""} - -QTINST36 = {"cnes": ""} - -QTINST37 = {"cnes": ""} - -QTLEIT05 = {"cnes": ""} - -QTLEIT06 = {"cnes": ""} - -QTLEIT07 = {"cnes": ""} - -QTLEIT08 = {"cnes": ""} - -QTLEIT09 = {"cnes": ""} - -QTLEIT19 = {"cnes": ""} - -QTLEIT20 = {"cnes": ""} - -QTLEIT21 = {"cnes": ""} - -QTLEIT22 = {"cnes": ""} - -QTLEIT23 = {"cnes": ""} - -QTLEIT32 = {"cnes": ""} - -QTLEIT34 = {"cnes": ""} - -QTLEIT38 = {"cnes": ""} - -QTLEIT39 = {"cnes": ""} - -QTLEIT40 = {"cnes": ""} - -QTLEITP1 = {"cnes": ""} - -QTLEITP2 = {"cnes": ""} - -QTLEITP3 = {"cnes": ""} - -QT_AGIPL = {"cnes": ""} - -QT_AGLTN = {"cnes": ""} - -QT_APRES = {"sia": ""} - -QT_APROV = {"sia": ""} - -QT_CADRE = {"cnes": ""} - -QT_CAPFL = {"cnes": ""} - -QT_CENRE = {"cnes": ""} - -QT_CONRA = {"cnes": ""} - -QT_CONTR = {"cnes": ""} - -QT_DIARIAS = {"sih": ""} - -QT_DOSE = {"pni": ""} - -QT_EXIST = {"cnes": ""} - -QT_EXTPL = {"cnes": ""} - -QT_FRE18 = {"cnes": ""} - -QT_FRE30 = {"cnes": ""} - -QT_IRRHE = {"cnes": ""} - -QT_MAQAF = {"cnes": ""} - -QT_NSUS = {"cnes": ""} - -QT_PROC = {"ciha": ""} - -QT_REFAS = {"cnes": ""} - -QT_REFRE = {"cnes": ""} - -QT_REFSA = {"cnes": ""} - -QT_SELAD = {"cnes": ""} - -QT_SUS = {"cnes": ""} - -QT_TOTAL_C = {"sinan": ""} - -QT_USO = {"cnes": ""} - -QUANTID = {"sinan": ""} - -QUANTOS = {"sinan": ""} - -QUAN_COMUN = {"sinan": ""} - -QUAN_POSIT = {"sinan": ""} - -QUILOMBO = {"cnes": ""} - -QUIMRADI = {"cnes": ""} - -QUININO = {"sinan": ""} - -QUININOI = {"sinan": ""} - -QUINOLONA = {"sinan": ""} - -RACACOR = { - "sia": "", - "sim": "", - "sinasc": "", -} - -RACACORMAE = {"sinasc": ""} - -RACACORN = {"sinasc": ""} - -RACACOR_RN = {"sinasc": ""} - -RACA_COR = {"sih": ""} - -RACA_MAE = {"sinan": ""} - -RACCOR = {"sinasc": ""} - -RAIOX = {"sinan": ""} - -RAIOX_TORA = {"sinan": ""} - -RAI_RESULT = {"sinan": ""} - -RAZAO = {"sih": ""} - -REACAO_SOR = {"sinan": ""} - -REACAO_VAC = {"sinan": ""} - -RECEMNASC = {"sinan": ""} - -RECEM_NASC = {"sinan": ""} - -RECUSA_QUI = {"sinan": ""} - -REDE_EDUCA = {"sinan": ""} - -REDE_SAU = {"sinan": ""} - -REFR_AQD_N = {"sinan": ""} - -REFR_AQE_N = {"sinan": ""} - -REFR_BID_N = {"sinan": ""} - -REFR_BIE_N = {"sinan": ""} - -REFR_PAD_N = {"sinan": ""} - -REFR_PAE_N = {"sinan": ""} - -REFR_TRD_N = {"sinan": ""} - -REFR_TRE_N = {"sinan": ""} - -REGCT = {"sih": ""} - -REGIME = {"sinan": ""} - -REGISTRO = { - "cnes": "", - "sim": "", -} - -REGSAUDE = {"cnes": ""} - -REL_CAT = {"sinan": ""} - -REL_CONHEC = {"sinan": ""} - -REL_CONJ = {"sinan": ""} - -REL_CUIDA = {"sinan": ""} - -REL_DESCO = {"sinan": ""} - -REL_ESPEC = {"sinan": ""} - -REL_EXCON = {"sinan": ""} - -REL_EXNAM = {"sinan": ""} - -REL_FILHO = {"sinan": ""} - -REL_INST = {"sinan": ""} - -REL_IRMAO = {"sinan": ""} - -REL_MAD = {"sinan": ""} - -REL_MAE = {"sinan": ""} - -REL_NAMO = {"sinan": ""} - -REL_OUTROS = {"sinan": ""} - -REL_PAD = {"sinan": ""} - -REL_PAI = {"sinan": ""} - -REL_PATRAO = {"sinan": ""} - -REL_POL = {"sinan": ""} - -REL_PROPRI = {"sinan": ""} - -REL_SEXUAL = {"sinan": ""} - -REL_TRAB = {"sinan": ""} - -REMESSA = {"sih": ""} - -RENAL = {"sinan": ""} - -REPETITIVO = {"sinan": ""} - -RESALIM1 = {"sinan": ""} - -RESALIMOUT = {"sinan": ""} - -RESPIRATO = {"sinan": ""} - -RESULT = {"sinan": ""} - -RESUL_HIS = {"sinan": ""} - -RESUL_NS1 = {"sinan": ""} - -RESUL_OUT = {"sinan": ""} - -RESUL_PCR = {"sinan": ""} - -RESUL_PCR_ = {"sinan": ""} - -RESUL_PRNT = {"sinan": ""} - -RESUL_SORO = {"sinan": ""} - -RESUL_VIRA = {"sinan": ""} - -RESUL_VI_N = {"sinan": ""} - -RES_BIOL = {"cnes": ""} - -RES_CHIKS1 = {"sinan": ""} - -RES_CHIKS2 = {"sinan": ""} - -RES_COMU = {"cnes": ""} - -RES_HBSAG = {"sinan": ""} - -RES_HIST = {"sinan": ""} - -RES_IMUNO = {"sinan": ""} - -RES_ISOL = {"sinan": ""} - -RES_PCR = {"sinan": ""} - -RES_QUIM = {"cnes": ""} - -RES_RADI = {"cnes": ""} - -RETAR_PM = {"sinan": ""} - -RETENCAO = {"cnes": ""} - -RETINOPA = {"sinan": ""} - -RE_ANTIHBC = {"sinan": ""} - -RE_ANTIHCV = {"sinan": ""} - -RIFAMPICIN = {"sinan": ""} - -ROEDOR_N = {"sinan": ""} - -ROTA_R = {"sinan": ""} - -RUBRICA = {"sih": ""} - -RUIDO_OUT = {"sinan": ""} - -RUI_OUTDES = {"sinan": ""} - -S1_IGG = {"sinan": ""} - -S1_IGM = {"sinan": ""} - -S1_TIT1 = {"sinan": ""} - -S2_IGG = {"sinan": ""} - -S2_IGM = {"sinan": ""} - -S2_TIT1 = {"sinan": ""} - -S3_IGG = {"sinan": ""} - -S3_IGM = {"sinan": ""} - -SALA_MOL = {"cnes": ""} - -SANG = {"sinan": ""} - -SANGRAM = {"sinan": ""} - -SANGUE = {"sinan": ""} - -SEMAGESTAC = { - "sim": "", - "sinasc": "", -} - -SEMANGEST = {"sim": ""} - -SEMIPLEN = {"sih": ""} - -SEM_ACID = {"sinan": ""} - -SEM_DIAG = {"sinan": ""} - -SEM_NOT = {"sinan": ""} - -SEM_PRI = {"sinan": ""} - -SEM_QUIMIO = {"sinan": ""} - -SENSIBILI = {"sinan": ""} - -SEQUENCIA = {"sih": ""} - -SEQ_AIH5 = {"sih": ""} - -SERAP01P = {"cnes": ""} - -SERAP01T = {"cnes": ""} - -SERAP02P = {"cnes": ""} - -SERAP02T = {"cnes": ""} - -SERAP03P = {"cnes": ""} - -SERAP03T = {"cnes": ""} - -SERAP04P = {"cnes": ""} - -SERAP04T = {"cnes": ""} - -SERAP05P = {"cnes": ""} - -SERAP05T = {"cnes": ""} - -SERAP06P = {"cnes": ""} - -SERAP06T = {"cnes": ""} - -SERAP07P = {"cnes": ""} - -SERAP07T = {"cnes": ""} - -SERAP08P = {"cnes": ""} - -SERAP08T = {"cnes": ""} - -SERAP09P = {"cnes": ""} - -SERAP09T = {"cnes": ""} - -SERAP10P = {"cnes": ""} - -SERAP10T = {"cnes": ""} - -SERAP11P = {"cnes": ""} - -SERAP11T = {"cnes": ""} - -SERAPOIO = {"cnes": ""} - -SERIESCFAL = {"sim": ""} - -SERIESCMAE = { - "sim": "", - "sinasc": "", -} - -SERV_CLA = {"sih": ""} - -SERV_ESP = {"cnes": ""} - -SEXO = { - "ciha": "", - "ibge": "", - "sih": "", - "sim": "", - "sinasc": "", -} - -SEXOPAC = {"sia": ""} - -SEXUAL = {"sinan": ""} - -SEX_ASSEDI = {"sinan": ""} - -SEX_ESPEC = {"sinan": ""} - -SEX_ESTUPR = {"sinan": ""} - -SEX_EXPLO = {"sinan": ""} - -SEX_OUTRO = {"sinan": ""} - -SEX_PORNO = {"sinan": ""} - -SEX_PUDOR = {"sinan": ""} - -SGRUPHAB = {"cnes": ""} - -SG_UF = {"sinan": ""} - -SG_UF_2 = {"sinan": ""} - -SG_UF_AT = {"sinan": ""} - -SG_UF_INTE = {"sinan": ""} - -SG_UF_NOT = {"sinan": ""} - -SG_UF_OCOR = {"sinan": ""} - -SILICA = {"sinan": ""} - -SIMUL_RD = {"cnes": ""} - -SINAIS = {"sinan": ""} - -SINAIS_ICC = {"sinan": ""} - -SINTOMATIC = {"sinan": ""} - -SINTO_DES = {"sinan": ""} - -SIN_GANG = {"sinan": ""} - -SIN_OUT = {"sinan": ""} - -SIN_OUTR_E = {"sinan": ""} - -SIN_PULM = {"sinan": ""} - -SIS_JUST = {"sih": ""} - -SITUACAO = {"ibge": ""} - -SITUA_12_M = {"sinan": ""} - -SITUA_9_M = {"sinan": ""} - -SITUA_ENCE = {"sinan": ""} - -SIT_CONJUG = {"sinan": ""} - -SIT_RUA = {"sia": ""} - -SIT_TRAB = {"sinan": ""} - -SOLVENTE = {"sinan": ""} - -SORO1 = {"sinan": ""} - -SORO2 = {"sinan": ""} - -SOROTIPO = {"sinan": ""} - -SOUTROS = {"sinan": ""} - -SP_AA = {"sih": ""} - -SP_ATOPROF = {"sih": ""} - -SP_CGCHOSP = {"sih": ""} - -SP_CIDPRI = {"sih": ""} - -SP_CIDSEC = {"sih": ""} - -SP_CNES = {"sih": ""} - -SP_COMPLEX = {"sih": ""} - -SP_CO_FAEC = {"sih": ""} - -SP_CPFCGC = {"sih": ""} - -SP_DES_HOS = {"sih": ""} - -SP_DES_PAC = {"sih": ""} - -SP_DTINTER = {"sih": ""} - -SP_DTSAIDA = {"sih": ""} - -SP_FINANC = {"sih": ""} - -SP_GESTOR = {"sih": ""} - -SP_MM = {"sih": ""} - -SP_M_HOSP = {"sih": ""} - -SP_M_PAC = {"sih": ""} - -SP_NAIH = {"sih": ""} - -SP_NF = {"sih": ""} - -SP_NUM_PR = {"sih": ""} - -SP_PF_CBO = {"sih": ""} - -SP_PF_DOC = {"sih": ""} - -SP_PJ_DOC = {"sih": ""} - -SP_PROCREA = {"sih": ""} - -SP_PTSP = {"sih": ""} - -SP_PTSP_NF = {"sih": ""} - -SP_QTD_ATO = {"sih": ""} - -SP_QT_PROC = {"sih": ""} - -SP_TIPO = {"sih": ""} - -SP_TP_ATO = {"sih": ""} - -SP_UF = {"sih": ""} - -SP_U_AIH = {"sih": ""} - -SP_VALATO = {"sih": ""} - -SRVUNICO = {"cnes": ""} - -STALIMENTO = {"sinan": ""} - -STANTIBIO = {"sinan": ""} - -STANTIBOTU = {"sinan": ""} - -STAVALIA = {"sinan": ""} - -STBOCA = {"sinan": ""} - -STBROMATO = {"sinan": ""} - -STBULBAR = {"sinan": ""} - -STCARDIACA = {"sinan": ""} - -STCASEIRA = {"sinan": ""} - -STCEFALEIA = {"sinan": ""} - -STCESPARTO = {"sinasc": ""} - -STCLINICA = {"sinan": ""} - -STCODIFICA = {"sim": ""} - -STCOMA = {"sinan": ""} - -STCOMERCIO = {"sinan": ""} - -STCONSTIPA = {"sinan": ""} - -STCURA1 = {"sinan": ""} - -STCURA2 = {"sinan": ""} - -STCURA3 = {"sinan": ""} - -STDESCENDE = {"sinan": ""} - -STDIARREIA = {"sinan": ""} - -STDIPLOPIA = {"sinan": ""} - -STDISARTRI = {"sinan": ""} - -STDISFAGIA = {"sinan": ""} - -STDISFONIA = {"sinan": ""} - -STDISPNEIA = {"sinan": ""} - -STDNEPIDEM = {"sinasc": ""} - -STDNNOVA = {"sinasc": ""} - -STDOEPIDEM = {"sim": ""} - -STDOMICILI = {"sinan": ""} - -STDONOVA = {"sim": ""} - -STELETRO = {"sinan": ""} - -STESCOLA = {"sinan": ""} - -STEXPALIM = {"sinan": ""} - -STFACIAL = {"sinan": ""} - -STFEBRE = {"sinan": ""} - -STFERIMENT = {"sinan": ""} - -STFESTA = {"sinan": ""} - -STFEZESMAT = {"sinan": ""} - -STFEZESRES = {"sinan": ""} - -STFLACIDEZ = {"sinan": ""} - -STHOSPITAL = {"sinan": ""} - -STMEMINF = {"sinan": ""} - -STMEMSUP = {"sinan": ""} - -STMIDRIASE = {"sinan": ""} - -STNAUSEA = {"sinan": ""} - -STOFTALMO = {"sinan": ""} - -STOUTROLOC = {"sinan": ""} - -STOUTROSIN = {"sinan": ""} - -STOUTROTRA = {"sinan": ""} - -STPARESTES = {"sinan": ""} - -STPTOSE = {"sinan": ""} - -STRESPIRA = {"sinan": ""} - -STRESS = {"sinan": ""} - -STRESTAURA = {"sinan": ""} - -STRESULTA = {"sinan": ""} - -STSENSIVEL = {"sinan": ""} - -STSIMETRIC = {"sinan": ""} - -STSORO = {"sinan": ""} - -STSOROMAT = {"sinan": ""} - -STSORORES = {"sinan": ""} - -STTONTURA = {"sinan": ""} - -STTRABALHO = {"sinan": ""} - -STTRABPART = {"sinasc": ""} - -STVENTILA = {"sinan": ""} - -STVISAO = {"sinan": ""} - -STVOMITO = {"sinan": ""} - -ST_ALI1COL = {"sinan": ""} - -ST_ALI2COL = {"sinan": ""} - -ST_ALI2RES = {"sinan": ""} - -ST_ALIMEN = {"sinan": ""} - -ST_A_CLINI = {"sinan": ""} - -ST_BLOQ = {"sih": ""} - -ST_F_OUTRO = {"sinan": ""} - -ST_IMPRO = {"sinan": ""} - -ST_IMPRO_ = {"sinan": ""} - -ST_INAD = {"sinan": ""} - -ST_INCUB_M = {"sinan": ""} - -ST_INC_ME = {"sinan": ""} - -ST_MANIP = {"sinan": ""} - -ST_MOT_BLO = {"sih": ""} - -ST_SITUAC = {"sih": ""} - -SUBFIN = {"sia": ""} - -SUDORESE = {"sinan": ""} - -SUGE_VINCU = {"sinan": ""} - -SULFA = {"sinan": ""} - -SUPERFICIA = {"sinan": ""} - -SUPERIORES = {"sinan": ""} - -SURTO = {"sinan": ""} - -SUSPEITOS = {"sinan": ""} - -S_ACELL6 = {"cnes": ""} - -S_AFERES = {"cnes": ""} - -S_ALCOME = {"cnes": ""} - -S_ALSEME = {"cnes": ""} - -S_ARMAZE = {"cnes": ""} - -S_BIOMOL = {"cnes": ""} - -S_COLETA = {"cnes": ""} - -S_CONTRQ = {"cnes": ""} - -S_CPFLUX = {"cnes": ""} - -S_DISTRI = {"cnes": ""} - -S_DPAC = {"cnes": ""} - -S_DPI = {"cnes": ""} - -S_ESTOQU = {"cnes": ""} - -S_HBSAGN = {"cnes": ""} - -S_HBSAGP = {"cnes": ""} - -S_HEMOST = {"cnes": ""} - -S_IMUNFE = {"cnes": ""} - -S_IMUNOH = {"cnes": ""} - -S_PREEST = {"cnes": ""} - -S_PREPAR = {"cnes": ""} - -S_PRETRA = {"cnes": ""} - -S_PROCES = {"cnes": ""} - -S_QCDURA = {"cnes": ""} - -S_QLDURA = {"cnes": ""} - -S_REAGN = {"cnes": ""} - -S_REAGP = {"cnes": ""} - -S_RECEPC = {"cnes": ""} - -S_REHCV = {"cnes": ""} - -S_SGDOAD = {"cnes": ""} - -S_SIMULA = {"cnes": ""} - -S_SOROLO = {"cnes": ""} - -S_TRANSF = {"cnes": ""} - -S_TRICLI = {"cnes": ""} - -S_TRIHMT = {"cnes": ""} - -TAREFAS = {"sinan": ""} - -TATU_PIER = {"sinan": ""} - -TECIDOS = {"sinan": ""} - -TECNICA = {"sinan": ""} - -TEMPO = {"sinan": ""} - -TEMPO_FUMA = {"sinan": ""} - -TERCEIRIZA = {"sinan": ""} - -TERCEIRO = {"cnes": ""} - -TESTE_TUBE = {"sinan": ""} - -TEST_MOLEC = {"sinan": ""} - -TEST_SENSI = {"sinan": ""} - -TETRAC = {"sinan": ""} - -TIFICA = {"sinan": ""} - -TIPEQUIP = {"cnes": ""} - -TIPOACID = {"sim": ""} - -TIPOBITO = {"sim": ""} - -TIPOGRAV = {"sim": ""} - -TIPOPARTO = {"sim": ""} - -TIPOSEGM = {"cnes": ""} - -TIPOVIOL = {"sim": ""} - -TIPO_ACID = {"sinan": ""} - -TIPO_EQP = {"cnes": ""} - -TIPO_GRAV = {"sinasc": ""} - -TIPO_INVES = {"sinan": ""} - -TIPO_LEITE = {"sinan": ""} - -TIPO_PARTO = {"sinasc": ""} - -TIPPRE = {"sia": ""} - -TIPPRE = {"sia": ""} - -TIP_DIARRE = {"sinan": ""} - -TIP_SORO = {"sinan": ""} - -TIREOIDITE = {"sinan": ""} - -TIT_IGG_S1 = {"sinan": ""} - -TIT_IGG_S2 = {"sinan": ""} - -TIT_IGM_S1 = {"sinan": ""} - -TIT_IGM_S2 = {"sinan": ""} - -TOMOGRAFIA = {"sinan": ""} - -TONR_CER_N = {"sinan": ""} - -TONR_FAC_N = {"sinan": ""} - -TONR_MID_N = {"sinan": ""} - -TONR_MIE_N = {"sinan": ""} - -TONR_MSD_N = {"sinan": ""} - -TONR_MSE_N = {"sinan": ""} - -TONTURA = {"sinan": ""} - -TOSSE = {"sinan": ""} - -TOT_PT_SP = {"sih": ""} - -TPALTA_N = {"sinan": ""} - -TPAPRESENT = {"sinasc": ""} - -TPASSINA = {"sim": ""} - -TPATENDE = {"sinan": ""} - -TPAUTOCTO = {"sinan": ""} - -TPBOTULISM = {"sinan": ""} - -TPBROMATO = {"sinan": ""} - -TPCLINICA = {"sinan": ""} - -TPCONFIRMA = {"sinan": ""} - -TPDISEC1 = {"sih": ""} - -TPDISEC2 = {"sih": ""} - -TPDISEC3 = {"sih": ""} - -TPDISEC4 = {"sih": ""} - -TPDISEC5 = {"sih": ""} - -TPDISEC6 = {"sih": ""} - -TPDISEC7 = {"sih": ""} - -TPDISEC8 = {"sih": ""} - -TPDISEC9 = {"sih": ""} - -TPDOCRESP = {"sinasc": ""} - -TPESQPAR = {"sinan": ""} - -TPESQUEMA = {"sinan": ""} - -TPEVIDENCI = {"sinan": ""} - -TPEXANTE = {"sinan": ""} - -TPEXP = {"sinan": ""} - -TPFEZESTOX = {"sinan": ""} - -TPFIN = {"sia": ""} - -TPFUNCRESP = {"sinasc": ""} - -TPGESTAO = {"cnes": ""} - -TPIDADEPAC = {"sia": ""} - -TPMETESTIM = {"sinasc": ""} - -TPMORTEOCO = {"sim": ""} - -TPMOTPARC = {"sinan": ""} - -TPNASCASSI = {"sinasc": ""} - -TPNEURO = {"sinan": ""} - -TPNIVELINV = {"sim": ""} - -TPOBITOCOR = {"sim": ""} - -TPPOS = {"sim": ""} - -TPRAPIDO1 = {"sinan": ""} - -TPRAPIDO2 = {"sinan": ""} - -TPRAPIDO3 = {"sinan": ""} - -TPRESGINFO = {"sim": ""} - -TPROBSON = {"sinasc": ""} - -TPRUIDO = {"sinan": ""} - -TPSOROTOX = {"sinan": ""} - -TPTEMPO = {"sinan": ""} - -TPTEMPORIS = {"sinan": ""} - -TPTESTE1 = {"sinan": ""} - -TPUNINOT = {"sinan": ""} - -TPUPS = {"sia": ""} - -TP_ACIDENT = {"sinan": ""} - -TP_AFAST = {"sinan": ""} - -TP_ALI1TOX = {"sinan": ""} - -TP_ALI2TO = {"sinan": ""} - -TP_AMB_OCO = {"sinan": ""} - -TP_ANALISE = {"sinan": ""} - -TP_CAUSA = {"sinan": ""} - -TP_CAUSOUT = {"sinan": ""} - -TP_COLOUT = {"sinan": ""} - -TP_DESAT = {"cnes": ""} - -TP_DROGA = {"sia": ""} - -TP_IDENTFI = {"sinan": ""} - -TP_INDIRET = {"sinan": ""} - -TP_LEITO = {"cnes": ""} - -TP_LIQUOR = {"sinan": ""} - -TP_LOCAL = {"sinan": ""} - -TP_LOCALLE = {"sinan": ""} - -TP_MOTORA = {"sinan": ""} - -TP_NOT = {"sinan": ""} - -TP_ORIGEM = {"sinan": ""} - -TP_PREST = {"cnes": ""} - -TP_PROFILA = {"sinan": ""} - -TP_PRO_PRE = {"sinan": ""} - -TP_REPETE = {"sinan": ""} - -TP_SENSITI = {"sinan": ""} - -TP_SISTEMA = {"sinan": ""} - -TP_SOROHCV = {"sinan": ""} - -TP_TEMP_FU = {"sinan": ""} - -TP_TOXOUTR = {"sinan": ""} - -TP_UNID = {"cnes": ""} - -TP_VACINA = {"sinan": ""} - -TP_ZN_OCO = {"sinan": ""} - -TRAB_DESC = {"sinan": ""} - -TRAB_DOE = {"sinan": ""} - -TRANSF = {"sinan": ""} - -TRANSFU = {"sinan": ""} - -TRANSFUSAO = {"sinan": ""} - -TRANSPLA = {"sinan": ""} - -TRANSPO_N = {"sinan": ""} - -TRAN_COMP = {"sinan": ""} - -TRAN_MENT = {"sinan": ""} - -TRATADO = {"sinan": ""} - -TRATAM = {"sinan": ""} - -TRATAMENTO = {"sinan": ""} - -TRATANAO = {"sinan": ""} - -TRATPARC = {"sinan": ""} - -TRATSUP_AT = {"sinan": ""} - -TRAT_ATUAL = {"sinan": ""} - -TRAT_SUPER = {"sinan": ""} - -TRA_AMPOLA = {"sinan": ""} - -TRA_ANTIBI = {"sinan": ""} - -TRA_ANTIGO = {"sinan": ""} - -TRA_ANTIVI = {"sinan": ""} - -TRA_CLASSI = {"sinan": ""} - -TRA_CORTIC = {"sinan": ""} - -TRA_CPAP = {"sinan": ""} - -TRA_DATA_A = {"sinan": ""} - -TRA_DATA_S = {"sinan": ""} - -TRA_DIAG_C = {"sinan": ""} - -TRA_DIAG_T = {"sinan": ""} - -TRA_DOSE = {"sinan": ""} - -TRA_DROGA_ = {"sinan": ""} - -TRA_DT = {"sinan": ""} - -TRA_DT_ALT = {"sinan": ""} - -TRA_DT_INT = {"sinan": ""} - -TRA_ESPECI = {"sinan": ""} - -TRA_ESQUEM = {"sinan": ""} - -TRA_ESQU_1 = {"sinan": ""} - -TRA_HOSP = {"sinan": ""} - -TRA_INDI_N = {"sinan": ""} - -TRA_INFILT = {"sinan": ""} - -TRA_INFI_1 = {"sinan": ""} - -TRA_INTERR = {"sinan": ""} - -TRA_MECANI = {"sinan": ""} - -TRA_MOTIVO = {"sinan": ""} - -TRA_MUNICI = {"sinan": ""} - -TRA_NUM_PA = {"sinan": ""} - -TRA_OUTRA_ = {"sinan": ""} - -TRA_OUTR_N = {"sinan": ""} - -TRA_PESO = {"sinan": ""} - -TRA_QTD_SO = {"sinan": ""} - -TRA_SORO = {"sinan": ""} - -TRA_TRATAM = {"sinan": ""} - -TRA_UF = {"sinan": ""} - -TRA_VASOAT = {"sinan": ""} - -TREINA_MIL = {"sinan": ""} - -TRESMAIS = {"sinan": ""} - -TRONCO = {"sinan": ""} - -TUBE = {"sinan": ""} - -TURNO_AT = {"cnes": ""} - -T_FEBRE = {"sinan": ""} - -UF = { - "pni": "", - "sinan": "", -} - -UFATUAL = {"sinan": ""} - -UFCOD = {"ibge": ""} - -UFDIF = {"sia": ""} - -UFINFORM = { - "sim": "", - "sinasc": "", -} - -UFINTERNA = {"sinan": ""} - -UFMUN = {"sia": ""} - -UFMUNRES = {"cnes": ""} - -UFRESAT = {"sinan": ""} - -UFTRANSFU = {"sinan": ""} - -UF_ACID = {"sinan": ""} - -UF_ATENDE = {"sinan": ""} - -UF_EMP = {"sinan": ""} - -UF_H = {"sinan": ""} - -UF_HOSP = {"sinan": ""} - -UF_HOSPITA = {"sinan": ""} - -UF_ING = {"sinan": ""} - -UF_PRE_NAT = {"sinan": ""} - -UF_RES = {"sih": ""} - -UF_TRANSF = {"sinan": ""} - -UF_ZI = {"sih": ""} - -UNI_ATENDE = {"sinan": ""} - -UN_COBAL = {"cnes": ""} - -URGEMERG = {"cnes": ""} - -URINA = {"sinan": ""} - -URO_D = {"sinan": ""} - -URO_D_2 = {"sinan": ""} - -URO_D_3 = {"sinan": ""} - -URO_R1 = {"sinan": ""} - -URO_R2 = {"sinan": ""} - -URO_R3 = {"sinan": ""} - -US_ORTP = {"sih": ""} - -US_RN = {"sih": ""} - -US_SADT = {"sih": ""} - -US_SANGUE = {"sih": ""} - -US_SH = {"sih": ""} - -US_SP = {"sih": ""} - -US_TOT = {"sih": ""} - -UTILIZACAO = {"sinan": ""} - -UTIL_DESC = {"sinan": ""} - -UTI_INT_AL = {"sih": ""} - -UTI_INT_AN = {"sih": ""} - -UTI_INT_IN = {"sih": ""} - -UTI_INT_TO = { - "ciha": "", - "sih": "", -} - -UTI_MES_AL = {"sih": ""} - -UTI_MES_AN = {"sih": ""} - -UTI_MES_IN = {"sih": ""} - -UTI_MES_TO = { - "ciha": "", - "sih": "", -} - -UTI_TOTAL = {"sih": ""} - -UTRANSFU = {"sinan": ""} - -VACINA = {"sinan": ""} - -VACINACAO = {"sinan": ""} - -VACINAD = {"sinan": ""} - -VACINADO = {"sinan": ""} - -VACINADUPL = {"sinan": ""} - -VACINARUBE = {"sinan": ""} - -VAC_HEP_B = {"sinan": ""} - -VAL_ACOMP = {"sih": ""} - -VAL_OBSANG = {"sih": ""} - -VAL_ORTP = {"sih": ""} - -VAL_PED1AC = {"sih": ""} - -VAL_RN = {"sih": ""} - -VAL_SADT = {"sih": ""} - -VAL_SADTSR = {"sih": ""} - -VAL_SANG = {"sih": ""} - -VAL_SANGUE = {"sih": ""} - -VAL_SH = {"sih": ""} - -VAL_SH_FED = {"sih": ""} - -VAL_SH_GES = {"sih": ""} - -VAL_SP = {"sih": ""} - -VAL_SP_FED = {"sih": ""} - -VAL_SP_GES = {"sih": ""} - -VAL_TOT = {"sih": ""} - -VAL_TRANSP = {"sih": ""} - -VAL_UCI = {"sih": ""} - -VAL_UTI = {"sih": ""} - -VARIA_VIR = {"sinan": ""} - -VERSAOSCB = {"sim": ""} - -VERSAOSIST = { - "sim": "", - "sinasc": "", -} - -VIA_1 = {"sinan": ""} - -VIA_2 = {"sinan": ""} - -VIA_3 = {"sinan": ""} - -VINCPREV = {"sih": ""} - -VINCULAC = {"cnes": ""} - -VINCULO = {"sinan": ""} - -VINCUL_A = {"cnes": ""} - -VINCUL_C = {"cnes": ""} - -VINCUL_N = {"cnes": ""} - -VINC_ESP = {"sinan": ""} - -VINC_OUT = {"sinan": ""} - -VINC_SUS = {"cnes": ""} - -VIOL_ESPEC = {"sinan": ""} - -VIOL_FINAN = {"sinan": ""} - -VIOL_FISIC = {"sinan": ""} - -VIOL_INFAN = {"sinan": ""} - -VIOL_LEGAL = {"sinan": ""} - -VIOL_MOTIV = {"sinan": ""} - -VIOL_NEGLI = {"sinan": ""} - -VIOL_OUTR = {"sinan": ""} - -VIOL_PSICO = {"sinan": ""} - -VIOL_SEXU = {"sinan": ""} - -VIOL_TORT = {"sinan": ""} - -VIOL_TRAF = {"sinan": ""} - -VL_APRES = {"sia": ""} - -VL_APROV = {"sia": ""} - -VOMITO = {"sinan": ""} - -VOMITOS = {"sinan": ""} - -VOP_VORH = {"sinan": ""} - -XENODIAG = {"sinan": ""} - -ZONA = {"sinan": ""} - -ZUMBIDO = {"sinan": ""} - -agravaids = {"sinan": ""} - -agravalcoo = {"sinan": ""} - -agravdiabe = {"sinan": ""} - -agravdoenc = {"sinan": ""} - -agravdroga = {"sinan": ""} - -agravoutra = {"sinan": ""} - -agravtabac = {"sinan": ""} - -ant_anemia = {"sinan": ""} - -ant_asteri = {"sinan": ""} - -ant_candid = {"sinan": ""} - -ant_caquex = {"sinan": ""} - -ant_contag = {"sinan": ""} - -ant_dermat = {"sinan": ""} - -ant_diarre = {"sinan": ""} - -ant_disfun = {"sinan": ""} - -ant_droga = {"sinan": ""} - -ant_esof_n = {"sinan": ""} - -ant_febre = {"sinan": ""} - -ant_herpes = {"sinan": ""} - -ant_linfo = {"sinan": ""} - -ant_pneumo = {"sinan": ""} - -ant_pulmon = {"sinan": ""} - -ant_rel_ca = {"sinan": ""} - -ant_tosse = {"sinan": ""} - -ant_toxo = {"sinan": ""} - -ant_trasmi = {"sinan": ""} - -ant_tuberc = {"sinan": ""} - -antrelse_n = {"sinan": ""} - -antsifil_n = {"sinan": ""} - -aval_atu_n = {"sinan": ""} - -avalia_n = {"sinan": ""} - -bacilosc_1 = {"sinan": ""} - -bacilosc_2 = {"sinan": ""} - -bacilosc_3 = {"sinan": ""} - -bacilosc_4 = {"sinan": ""} - -bacilosc_5 = {"sinan": ""} - -bacilosc_6 = {"sinan": ""} - -bacilosco = {"sinan": ""} - -cancro_mole = {"sinan": ""} - -caract_genomica = {"sinan": ""} - -clado = {"sinan": ""} - -clamidea = {"sinan": ""} - -classatual = {"sinan": ""} - -classi_fin = {"sinan": ""} - -classopera = {"sinan": ""} - -co_uf_res = {"sinan": ""} - -comp_sexual = {"sinan": ""} - -contador = { - "sim": "", - "sinasc": "", -} - -contag_cd4 = {"sinan": ""} - -contat_animal = {"sinan": ""} - -contexam = {"sinan": ""} - -contreg = {"sinan": ""} - -criterio = {"sinan": ""} - -cs_escol_n = {"sinan": ""} - -cs_gestant = {"sinan": ""} - -cs_raca = {"sinan": ""} - -cs_sexo = {"sinan": ""} - -cs_zona = {"sinan": ""} - -cultura_es = {"sinan": ""} - -data_vacina = {"sinan": ""} - -def_diagno = {"sinan": ""} - -dip = {"sinan": ""} - -doenca_tra1 = {"sinan": ""} - -donovanose = {"sinan": ""} - -dose_receb = {"sinan": ""} - -dt_coleta = {"sinan": ""} - -dt_diag = {"sinan": ""} - -dt_encerra = {"sinan": ""} - -dt_evolucao = {"sinan": ""} - -dt_inic_tr = {"sinan": ""} - -dt_interna = {"sinan": ""} - -dt_nasc = {"sinan": ""} - -dt_noti_at = {"sinan": ""} - -dt_notific = {"sinan": ""} - -dt_obito = {"sinan": ""} - -dt_sin_pri = {"sinan": ""} - -dtalta_n = {"sinan": ""} - -dtinictrat = {"sinan": ""} - -dtultcomp = {"sinan": ""} - -esq_atu_n = {"sinan": ""} - -esq_ini_n = {"sinan": ""} - -estrangeiro = {"sinan": ""} - -evolucao = {"sinan": ""} - -forma = {"sinan": ""} - -formaclini = {"sinan": ""} - -gonorreia = {"sinan": ""} - -herpes_genital = {"sinan": ""} - -histopatol = {"sinan": ""} - -hiv = {"sinan": ""} - -hospital = {"sinan": ""} - -hpv = {"sinan": ""} - -htlv = {"sinan": ""} - -id_agravo = {"sinan": ""} - -id_mn_resi = {"sinan": ""} - -id_municip = {"sinan": ""} - -id_regiona = {"sinan": ""} - -id_rg_resi = {"sinan": ""} - -id_unidade = {"sinan": ""} - -ident_genero = {"sinan": ""} - -ist_ativa = {"sinan": ""} - -lab_triage = {"sinan": ""} - -labc_igg = {"sinan": ""} - -linfogranuloma = {"sinan": ""} - -local_cont = {"sinan": ""} - -met_lab = {"sinan": ""} - -mododetect = {"sinan": ""} - -modoentr = {"sinan": ""} - -mycoplasma_genital = {"sinan": ""} - -name = {"sinan": ""} - -nervosafet = {"sinan": ""} - -nu_ano = {"sinan": ""} - -nu_idade_n = {"sinan": ""} - -nu_lesoes = {"sinan": ""} - -orienta_sexual = {"sinan": ""} - -outro_des = {"sinan": ""} - -owner_org = {"sinan": ""} - -pac_imunossup = {"sinan": ""} - -pop_liber = {"sinan": ""} - -profile = {"sinan": ""} - -profis_saude = {"sinan": ""} - -raiox_tora = {"sinan": ""} - -resources = {"sinan": ""} - -resultado_exa_lab = {"sinan": ""} - -sg_uf = {"sinan": ""} - -sg_uf_not = {"sinan": ""} - -sifilis = {"sinan": ""} - -sintoma = {"sinan": ""} - -situa_ence = {"sinan": ""} - -test_molec = {"sinan": ""} - -test_sensi = {"sinan": ""} - -title = {"sinan": ""} - -tp_amost = {"sinan": ""} - -tpalta_n = {"sinan": ""} - -tpesquema = {"sinan": ""} - -tra_esquem = {"sinan": ""} - -transm = {"sinan": ""} - -tratamento = {"sinan": ""} - -tratamento_mpox = {"sinan": ""} - -tratparc = {"sinan": ""} - -tratsup_at = {"sinan": ""} - -trichomomas_vaginals = {"sinan": ""} - -uti = {"sinan": ""} - -vacina = {"sinan": ""} - -verruga_genital = {"sinan": ""} - -vinculo_epi = {"sinan": ""} diff --git a/pysus/api/ducklake/catalog/orm/columns.py b/pysus/api/ducklake/catalog/orm/columns.py new file mode 100644 index 00000000..3df18200 --- /dev/null +++ b/pysus/api/ducklake/catalog/orm/columns.py @@ -0,0 +1,22 @@ +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy import Column, Integer, Sequence, String, Boolean, Index + + +class ColumnsBase(DeclarativeBase): + pass + + +class ColumnDefinition(ColumnsBase): + __tablename__ = "dataset_columns" + + id = Column(Integer, Sequence("columns_id_seq", schema="pysus"), primary_key=True) + dataset_id = Column(Integer, nullable=False, index=True) + name = Column(String, nullable=False) + type = Column(String, nullable=False) + description = Column(String, nullable=True) + nullable = Column(Boolean, nullable=False, default=True) + + __table_args__ = ( + Index("ix_columns_dataset_name", "dataset_id", "name"), + {"schema": "pysus"}, + ) diff --git a/pysus/api/ducklake/catalog/orm/dataset.py b/pysus/api/ducklake/catalog/orm/dataset.py index 687799ce..fb096f50 100644 --- a/pysus/api/ducklake/catalog/orm/dataset.py +++ b/pysus/api/ducklake/catalog/orm/dataset.py @@ -1,310 +1,76 @@ -"""Per-dataset catalog ORM models — stored in ``catalog_.db``. - -Defines tables for groups, files, and columns within a single dataset. -""" - from datetime import datetime from typing import Optional - +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship from sqlalchemy import ( - BigInteger, - Boolean, Column, - DateTime, - ForeignKey, - Index, Integer, Sequence, String, + ForeignKey, + BigInteger, + Index, + DateTime, Table, ) -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship -class Base(DeclarativeBase): - """Base class for per-dataset catalog ORM models.""" - +class DatasetBase(DeclarativeBase): pass file_columns = Table( "file_columns", - Base.metadata, - Column( - "file_id", - Integer, - ForeignKey("pysus.files.id"), - primary_key=True, - ), - Column( - "column_id", - Integer, - ForeignKey("pysus.dataset_columns.id"), - primary_key=True, - ), + DatasetBase.metadata, + Column("file_id", Integer, ForeignKey("pysus.files.id"), primary_key=True), + Column("column_id", Integer, primary_key=True), schema="pysus", ) -class Dataset(Base): - """ORM model for the datasets table, representing a dataset collection. - - Parameters - ---------- - id : int, optional - Primary key (auto-generated by sequence). - name : str - Unique short name for the dataset. - long_name : str - Human-readable full name. - description : str, optional - Optional description of the dataset contents. - """ - - __tablename__ = "datasets" - __table_args__: tuple = ({"schema": "pysus"},) - - id = Column( - Integer, - Sequence("datasets_id_seq", schema="pysus"), - primary_key=True, - ) - name = Column(String, nullable=False, unique=True, index=True) - long_name = Column(String, nullable=False) - description = Column(String, nullable=True) - - groups = relationship( - "Group", - back_populates="dataset", - cascade="all, delete-orphan", - ) - files = relationship( - "File", - back_populates="dataset", - cascade="all, delete-orphan", - ) - columns = relationship( - "ColumnDefinition", - back_populates="dataset", - cascade="all, delete-orphan", - ) - - -class ColumnDefinition(Base): - """ORM model for dataset column metadata. - - Parameters - ---------- - id : int, optional - Primary key (auto-generated by sequence). - dataset_id : int - Foreign key referencing the parent dataset. - name : str - Column name. - type : str - Column data type string. - description : str, optional - Optional description of the column. - nullable : bool, optional - Whether the column allows null values. - """ - - __tablename__ = "dataset_columns" - __table_args__: tuple = ({"schema": "pysus"},) - - id = Column( - Integer, - Sequence("columns_id_seq", schema="pysus"), - primary_key=True, - ) - dataset_id = Column( - Integer, - ForeignKey("pysus.datasets.id"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - type = Column(String, nullable=False) - description = Column(String, nullable=True) - nullable = Column(Boolean, nullable=False, default=True) - - dataset = relationship("Dataset", back_populates="columns") - files = relationship( - "File", - secondary=file_columns, - back_populates="columns", - ) - +class Group(DatasetBase): + __tablename__ = "dataset_groups" __table_args__ = ( - Index("ix_columns_dataset_name", "dataset_id", "name"), + Index("ix_groups_dataset_name", "dataset_id", "name"), {"schema": "pysus"}, ) - -class Group(Base): - """ORM model for dataset groups, grouping related files within a dataset. - - Parameters - ---------- - id : int, optional - Primary key (auto-generated by sequence). - name : str - Short name for the group. - dataset_id : int - Foreign key referencing the parent dataset. - long_name : str - Human-readable full name. - description : str, optional - Optional description of the group contents. - """ - - __tablename__ = "dataset_groups" - __table_args__: tuple = ({"schema": "pysus"},) - - id = Column( - Integer, - Sequence("groups_id_seq", schema="pysus"), - primary_key=True, - ) + id = Column(Integer, Sequence("groups_id_seq", schema="pysus"), primary_key=True) name = Column(String, nullable=False) - dataset_id = Column( - Integer, - ForeignKey("pysus.datasets.id"), - nullable=False, - index=True, - ) + dataset_id = Column(Integer, nullable=False, index=True) long_name = Column(String, nullable=False) description = Column(String, nullable=True) - dataset = relationship( - "Dataset", - back_populates="groups", - ) - files = relationship( - "File", - back_populates="group", - cascade="all, delete-orphan", - ) + files = relationship("File", back_populates="group", cascade="all, delete-orphan") + +class File(DatasetBase): + __tablename__ = "files" __table_args__ = ( - Index("ix_groups_dataset_name", "dataset_id", "name"), + Index("ix_files_dataset_group", "dataset_id", "group_id"), + Index("ix_files_temporal", "year", "month"), + Index("ix_files_lookup", "dataset_id", "group_id", "year", "month", "state"), {"schema": "pysus"}, ) - -class File(Base): - """ORM model for the files table, representing individual data files. - - Parameters - ---------- - id : int, optional - Primary key (auto-generated by sequence). - dataset_id : int - Foreign key referencing the parent dataset. - group_id : int, optional - Foreign key referencing the parent group. - path : str - Object storage path to the file. - size : int - File size in bytes. - rows : int - Number of rows in the file. - type : str, optional - File type identifier. - modified : datetime - Timestamp of the last known modification. - origin_modified : datetime, optional - Original modification timestamp from the source. - origin_size : int - Original file size in bytes. - origin_path : str - Original source path of the file. - sha256 : str, optional - SHA-256 hex digest for integrity verification. - year : int, optional - Data year associated with the file. - month : int, optional - Data month associated with the file. - state : str, optional - Two-letter state code associated with the file. - """ - - __tablename__ = "files" - __table_args__: tuple = ({"schema": "pysus"},) - id: Mapped[int] = mapped_column( - Integer, - Sequence("files_id_seq", schema="pysus"), - primary_key=True, - ) - dataset_id: Mapped[int] = mapped_column( - Integer, ForeignKey("pysus.datasets.id"), nullable=False, index=True + Integer, Sequence("files_id_seq", schema="pysus"), primary_key=True ) + dataset_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) group_id: Mapped[int | None] = mapped_column( - Integer, - ForeignKey("pysus.dataset_groups.id"), - nullable=True, - index=True, + Integer, ForeignKey("pysus.dataset_groups.id"), nullable=True, index=True ) - path: Mapped[str] = mapped_column(String, nullable=False, unique=True) size: Mapped[int] = mapped_column(BigInteger, nullable=False) rows: Mapped[int] = mapped_column(Integer, nullable=False) type: Mapped[str] = mapped_column(String, nullable=True) modified: Mapped[datetime] = mapped_column(DateTime, nullable=False) - origin_modified: Mapped[datetime | None] = mapped_column( - DateTime, - nullable=True, - ) + origin_modified: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) origin_size: Mapped[int] = mapped_column(BigInteger, nullable=False) origin_path: Mapped[str] = mapped_column(String, nullable=False) - sha256: Mapped[str | None] = mapped_column( - String(64), - nullable=True, - index=True, - ) - - year: Mapped[int | None] = mapped_column( - Integer, - nullable=True, - index=True, - ) - month: Mapped[int | None] = mapped_column( - Integer, - nullable=True, - index=True, - ) - state: Mapped[str | None] = mapped_column( - String(2), - nullable=True, - index=True, - ) + sha256: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) + year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) + month: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) + state: Mapped[str | None] = mapped_column(String(2), nullable=True, index=True) - dataset: Mapped["Dataset"] = relationship( - "Dataset", - back_populates="files", - ) - group: Mapped[Optional["Group"]] = relationship( - "Group", - back_populates="files", - ) - columns: Mapped[list["ColumnDefinition"]] = relationship( - "ColumnDefinition", - secondary=file_columns, - back_populates="files", - cascade="all, delete", - ) + group: Mapped[Optional["Group"]] = relationship("Group", back_populates="files") - __table_args__ = ( - Index("ix_files_dataset_group", "dataset_id", "group_id"), - Index("ix_files_temporal", "year", "month"), - Index( - "ix_files_lookup", - "dataset_id", - "group_id", - "year", - "month", - "state", - ), - {"schema": "pysus"}, - ) diff --git a/pysus/api/ducklake/catalog/orm/default.py b/pysus/api/ducklake/catalog/orm/default.py index bd412080..4de0a24f 100644 --- a/pysus/api/ducklake/catalog/orm/default.py +++ b/pysus/api/ducklake/catalog/orm/default.py @@ -41,3 +41,6 @@ class Dataset(Base): name = Column(String, nullable=False, unique=True, index=True) long_name = Column(String, nullable=False) description = Column(String, nullable=True) + + def __repr__(self): + return self.name diff --git a/pysus/api/ducklake/catalog/parsers.py b/pysus/api/ducklake/catalog/parsers.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pysus/api/ducklake/client.py b/pysus/api/ducklake/client.py index f7339569..ee16231c 100644 --- a/pysus/api/ducklake/client.py +++ b/pysus/api/ducklake/client.py @@ -6,417 +6,112 @@ from collections.abc import Callable from pathlib import Path -from typing import Any -import boto3 -import httpx -from anyio import sleep, to_thread -from botocore.config import Config -from pydantic import BaseModel, PrivateAttr, SecretStr -from pysus import CACHEPATH -from pysus.api.models import BaseRemoteClient, BaseRemoteFile +from anyio import to_thread +from pydantic import SecretStr, PrivateAttr +from pysus.api.models import BaseRemoteClient from pysus.api.types import DUCKLAKE -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy.pool import StaticPool from .catalog.orm.default import Dataset +from .catalog.adapters import DatasetAdapter, CatalogAdapter from .models import DuckDataset, File - - -class DuckLakeCredentials(BaseModel): - """Credentials for authenticating with the S3-compatible object storage. - - Parameters - ---------- - access_key : SecretStr - The S3 access key ID. - secret_key : SecretStr - The S3 secret access key. - """ - - access_key: SecretStr - secret_key: SecretStr +from .catalog.adapters import DuckLakeCredentials +from .functional import download_s3 class DuckLake(BaseRemoteClient): - """Client for the DuckLake S3-based public health dataset catalog. - - Parameters - ---------- - endpoint : str, optional - S3-compatible object storage endpoint. - region : str, optional - Storage region name. - bucket : str, optional - Bucket name containing the catalog. - credentials : DuckLakeCredentials, optional - Credentials for authenticated S3 operations. - """ - - endpoint: str = "nbg1.your-objectstorage.com" - region: str = "nbg1" - bucket: str = "pysus" credentials: DuckLakeCredentials | None = None - - _s3_client: Any = PrivateAttr(default=None) - _Session: Any = PrivateAttr(default=None) - _datasets: list = PrivateAttr(default_factory=list) + _datasets: list[DuckDataset] = PrivateAttr(default_factory=list) def __init__(self, engine=None, **data) -> None: - """Initialize the DuckLake client. - - Parameters - ---------- - engine : object, optional - Pre-configured SQLAlchemy engine for the discovery catalog. - ``**data`` - Fields passed to the Pydantic base model. - """ super().__init__(**data) - self._engine = engine - self._cache_dir: Path = Path(CACHEPATH) / "ducklake" - self._cache_dir.mkdir(parents=True, exist_ok=True) - self._catalog_local: Path = self._cache_dir / "catalog.duckdb" - self._catalog_remote: str = "public/catalog.duckdb" + self.catalog_adap = CatalogAdapter( + engine=engine, + credentials=self.credentials, + ) @property def name(self) -> str: - """Return the short name of this client. - - Returns - ------- - str - The client short name. - """ return DUCKLAKE @property def long_name(self) -> str: - """Return the human-readable name of this client. - - Returns - ------- - str - The client display name. - """ return "PySUS s3 Client" @property def description(self) -> str: - """Return a description of this client. - - Returns - ------- - str - A description string (currently empty). - """ - return "" # TODO: - - @property - def catalog_path(self) -> Path: - """Return the local path to the discovery catalog database. - - Returns - ------- - Path - Filesystem path to the local discovery catalog file. - """ - return self._catalog_local - - @property - def _catalog_url(self) -> str: - """Return the remote URL of the discovery catalog.""" - return f"https://{self.endpoint}/{self.bucket}/{self._catalog_remote}" - - @property - def _is_authenticated(self) -> bool: - """Return whether the client has credentials configured.""" - return self.credentials is not None + return "" async def datasets(self, **kwargs) -> list[DuckDataset]: - """Return all datasets from the catalog as DuckDataset instances. - - Parameters - ---------- - ``**kwargs`` - Additional filter arguments (currently unused). - - Returns - ------- - list[DuckDataset] - List of all datasets in the catalog. - """ - if not self._Session: - await self.connect() + await self.catalog_adap.connect() def _fetch(): - with self._Session() as session: + with self.catalog_adap.get_session() as session: results = session.query(Dataset).all() session.expunge_all() return results records = await to_thread.run_sync(_fetch) - return [DuckDataset(record=rec, client=self) for rec in records] - async def login( - self, - access_key: str | None = None, - secret_key: str | None = None, - **kwargs, - ) -> None: - """Authenticate with S3 credentials and reconnect to the catalog. - - Parameters - ---------- - access_key : str, optional - S3 access key ID. If omitted, credentials are cleared. - secret_key : str, optional - S3 secret access key. If omitted, credentials are cleared. - ``**kwargs`` - Additional arguments (currently unused). - """ - if access_key and secret_key: - self.credentials = DuckLakeCredentials( - access_key=SecretStr(access_key), - secret_key=SecretStr(secret_key), + duck_datasets: list[DuckDataset] = [] + for rec in records: + dataset_adapter = DatasetAdapter( + name=str(rec.name), credentials=self.credentials ) - else: - self.credentials = None - - await self.connect(force=True) - - if self._is_authenticated: - self._s3_client = await to_thread.run_sync( - self._get_s3_client, + duck_datasets.append( + DuckDataset(record=rec, client=self, adapter=dataset_adapter) ) - def _setup_engine(self, local_path: Path | None = None): - """Create and configure a DuckDB engine with S3 settings. + self._datasets = duck_datasets + return duck_datasets - Parameters - ---------- - local_path : Path, optional - Path to the catalog database file. - Defaults to the discovery catalog. - """ - if local_path is None: - local_path = self._catalog_local - engine = create_engine( - f"duckdb:///{local_path}", - poolclass=StaticPool, + async def login( + self, + access_key: str, + secret_key: str, + **kwargs, + ) -> None: + self.credentials = DuckLakeCredentials( + access_key=SecretStr(access_key), + secret_key=SecretStr(secret_key), ) - - with engine.connect() as conn: - conn.exec_driver_sql("INSTALL ducklake; LOAD ducklake;") - - has_pysus = conn.exec_driver_sql( - "SELECT 1 FROM information_schema.schemata" - " WHERE schema_name = 'pysus'" - ).fetchone() - - if has_pysus: - conn.exec_driver_sql("SET search_path='pysus,main';") - else: - conn.exec_driver_sql("SET search_path='main';") - - s3_cfg = { - "s3_endpoint": self.endpoint, - "s3_region": self.region, - "s3_url_style": "path", - "s3_use_ssl": "true", - } - - if self.credentials and self._is_authenticated: - s3_cfg["s3_access_key_id"] = ( - self.credentials.access_key.get_secret_value() - ) - s3_cfg["s3_secret_access_key"] = ( - self.credentials.secret_key.get_secret_value() - ) - - for key, value in s3_cfg.items(): - conn.exec_driver_sql(f"SET {key}='{value}'") - - conn.commit() - - return engine + self.catalog_adap.credentials = self.credentials + await self.catalog_adap.connect(force=True) async def connect(self, force: bool = False) -> None: - """Connect to the discovery catalog, downloading first if needed. - - Parameters - ---------- - force : bool, optional - Whether to re-download and re-connect even if already connected. - """ - if self._engine and not force: - if not self._Session: - self._Session = sessionmaker(bind=self._engine) - return - - await self._download_catalog( - self._catalog_local, - self._catalog_remote, - ) - self._engine = await to_thread.run_sync(self._setup_engine) - self._Session = sessionmaker(bind=self._engine) + await self.catalog_adap.connect(force=force) async def close(self, update_catalog: bool = False) -> None: - """Close all datasets and dispose the discovery engine. - - Parameters - ---------- - update_catalog : bool, optional - Whether to upload all per-dataset catalogs before closing. - Requires authenticated credentials. - """ - if update_catalog: - await self._upload_catalog() - - datasets: list["DuckDataset"] = list(self._datasets) - for ds in datasets: + for ds in self._datasets: await ds.close(update_catalog=update_catalog) - self._datasets.clear() - - if self._engine: - await to_thread.run_sync(self._engine.dispose) - self._engine = None - self._Session = None - self._s3_client = None - async def _download( - self, - remote_path: str, - local_path: Path, - *, - callback: Callable[[int, int], None] | None = None, - ) -> None: - """Download *remote_path* to *local_path* with streaming and retries. - - Parameters - ---------- - remote_path : str - Object key within the bucket. - local_path : Path - Local destination path. - callback : Callable[[int, int], None], optional - Progress callback receiving ``(downloaded, total)`` bytes. - """ - url = f"https://{self.endpoint}/{self.bucket}/{remote_path}" - max_retries = 5 - - for attempt in range(max_retries): - try: - async with httpx.AsyncClient(follow_redirects=True) as client: - async with client.stream("GET", url) as r: - r.raise_for_status() - total = int(r.headers.get("Content-Length", 0)) - downloaded = 0 - with open(local_path, "wb") as f: - async for chunk in r.aiter_bytes( - chunk_size=1024 * 1024, - ): - await to_thread.run_sync(f.write, chunk) - downloaded += len(chunk) - if callback: - callback(downloaded, total) - return - except OSError as e: - if attempt < max_retries - 1: - await sleep(1) - else: - raise e - - async def _download_catalog( - self, local_path: Path, remote_path: str - ) -> None: - """Download a catalog database from remote storage with retries. - - Parameters - ---------- - local_path : Path - Local destination path for the catalog file. - remote_path : str - Remote object key within the bucket. - """ - url = f"https://{self.endpoint}/{self.bucket}/{remote_path}" - - if local_path.exists(): - try: - local_size = local_path.stat().st_size - except OSError: - local_size = -1 - else: - local_size = -1 - - async with httpx.AsyncClient(follow_redirects=True) as client: - try: - head = await client.head(url) - head.raise_for_status() - remote_size = int(head.headers.get("content-length", 0)) - except Exception: # noqa: B902 - remote_size = 0 - - if remote_size == local_size: - return - - await self._download(remote_path, local_path) + await self.catalog_adap.close(update=update_catalog) + self._datasets.clear() - async def _download_file( + async def download( self, - file: BaseRemoteFile, + file: File, output: Path, callback: Callable[[int, int], None] | None = None, ) -> Path: - """Download a single file from object storage to the local path.""" if not isinstance(file, File): raise ValueError("FTP File was not properly instantiated") - await self._download(file.record.path, output, callback=callback) - return output - - def _get_s3_client(self): - """Create and return a boto3 S3 client for the configured endpoint.""" - if not self.credentials: - raise ConnectionError("S3 Credentials not found") - return boto3.client( - "s3", - endpoint_url=f"https://{self.endpoint}", - aws_access_key_id=self.credentials.access_key.get_secret_value(), - aws_secret_access_key=( - self.credentials.secret_key.get_secret_value() - ), - region_name=self.region, - config=Config(signature_version="s3v4"), + access_key = ( + self.credentials.access_key.get_secret_value() if self.credentials else None + ) + secret_key = ( + self.credentials.secret_key.get_secret_value() if self.credentials else None ) - async def _upload_catalog(self) -> None: - """Upload all per-dataset catalogs to remote storage. - - Requires authenticated credentials. - """ - if not self.credentials: - raise PermissionError( - "Admin credentials required to upload catalog.", - ) - - datasets = await self.datasets() - for ds in datasets: - if not ds._catalog_local.exists(): - continue - - _local = str(ds._catalog_local) - _name = ds._catalog_name - - def _upload(local=_local, name=_name): - self._s3_client.upload_file( - local, - self.bucket, - name, - ) - - await to_thread.run_sync(_upload) + await download_s3( + remote_path=file.record.path, + local_path=output, + access_key=access_key, + secret_key=secret_key, + callback=callback, + ) + return output DuckDataset.model_rebuild(_types_namespace={"DuckLake": DuckLake}) diff --git a/pysus/api/ducklake/functional.py b/pysus/api/ducklake/functional.py new file mode 100644 index 00000000..d13d8001 --- /dev/null +++ b/pysus/api/ducklake/functional.py @@ -0,0 +1,179 @@ +from pathlib import Path +from typing import Callable + +from anyio import sleep, to_thread +import httpx +import boto3 +from botocore.config import Config +from botocore import UNSIGNED + +from pysus.api import types + + +async def download_http( + remote_path: str, + local_path: Path, + callback: Callable[[int, int], None] | None = None, +) -> None: + """Download *remote_path* to *local_path* with HTTP streaming and retries. + + Parameters + ---------- + remote_path : str + Object key within the bucket. + local_path : Path + Local destination path. + callback : Callable[[int, int], None], optional + Progress callback receiving ``(downloaded, total)`` bytes. + """ + url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote_path}" + max_retries = 5 + + for attempt in range(max_retries): + try: + async with httpx.AsyncClient(follow_redirects=True) as client: + async with client.stream("GET", url) as r: + r.raise_for_status() + total = int(r.headers.get("Content-Length", 0)) + downloaded = 0 + with open(local_path, "wb") as f: + async for chunk in r.aiter_bytes(chunk_size=1024 * 1024): + await to_thread.run_sync(f.write, chunk) + downloaded += len(chunk) + if callback: + callback(downloaded, total) + return + except OSError as e: + if attempt < max_retries - 1: + await sleep(1) + else: + raise e + + +async def download_s3( + remote_path: str, + local_path: Path, + access_key: str | None = None, + secret_key: str | None = None, + callback: Callable[[int, int], None] | None = None, +) -> None: + """Download *remote_path* to *local_path* using boto3 with optional credentials. + + Parameters + ---------- + remote_path : str + Object key within the bucket. + local_path : Path + Local destination path. + access_key : str, optional + S3 access key ID. + secret_key : str, optional + S3 secret access key. + callback : Callable[[int, int], None], optional + Progress callback receiving ``(downloaded, total)`` bytes. + """ + max_retries = 5 + + def _get_client_args(): + args: dict = { + "service_name": "s3", + "endpoint_url": f"https://{types.S3_ENDPOINT}", + "region_name": types.S3_REGION, + } + if access_key and secret_key: + args["aws_access_key_id"] = access_key + args["aws_secret_access_key"] = secret_key + args["config"] = Config(signature_version="s3v4") + else: + args["config"] = Config(signature_version=UNSIGNED) + return args + + def _get_total_size(client_args) -> int: + try: + client = boto3.client(**client_args) + meta = client.head_object(Bucket=types.S3_BUCKET, Key=remote_path) + return int(meta.get("ContentLength", 0)) + except Exception: + return 0 + + def _download(client_args, total_size: int): + client = boto3.client(**client_args) + downloaded = 0 + + def boto_callback(bytes_amount): + nonlocal downloaded + downloaded += bytes_amount + if callback: + callback(downloaded, total_size) + + client.download_file( + Bucket=types.S3_BUCKET, + Key=remote_path, + Filename=str(local_path), + Callback=boto_callback if callback else None, + ) + + for attempt in range(max_retries): + try: + client_args = _get_client_args() + total_size = await to_thread.run_sync(_get_total_size, client_args) + await to_thread.run_sync(_download, client_args, total_size) + return + except Exception as e: + if attempt < max_retries - 1: + await sleep(1) + else: + raise e + + +async def upload_s3( + local_path: Path, + remote_path: str, + access_key: str, + secret_key: str, + callback: Callable[[int, int], None] | None = None, +) -> None: + max_retries = 5 + + def _get_client_args(): + args: dict = { + "service_name": "s3", + "endpoint_url": f"https://{types.S3_ENDPOINT}", + "region_name": types.S3_REGION, + } + if access_key and secret_key: + args["aws_access_key_id"] = access_key + args["aws_secret_access_key"] = secret_key + args["config"] = Config(signature_version="s3v4") + else: + args["config"] = Config(signature_version=UNSIGNED) + return args + + def _upload(client_args, total_size: int): + client = boto3.client(**client_args) + uploaded = 0 + + def boto_callback(bytes_amount): + nonlocal uploaded + uploaded += bytes_amount + if callback: + callback(uploaded, total_size) + + client.upload_file( + Filename=str(local_path), + Bucket=types.S3_BUCKET, + Key=remote_path, + Callback=boto_callback if callback else None, + ) + + for attempt in range(max_retries): + try: + client_args = _get_client_args() + total_size = local_path.stat().st_size + await to_thread.run_sync(_upload, client_args, total_size) + return + except Exception as e: + if attempt < max_retries - 1: + await sleep(1) + else: + raise e diff --git a/pysus/api/ducklake/models.py b/pysus/api/ducklake/models.py index c9c4d3e5..449cf635 100644 --- a/pysus/api/ducklake/models.py +++ b/pysus/api/ducklake/models.py @@ -13,11 +13,14 @@ from anyio import to_thread from pydantic import Field, PrivateAttr from pysus import CACHEPATH -from pysus.api.ducklake.catalog.orm.dataset import Dataset -from pysus.api.ducklake.catalog.orm.dataset import File as CatalogFile -from pysus.api.ducklake.catalog.orm.dataset import Group +from .catalog.adapters import DatasetAdapter +from .catalog.orm.default import Dataset +from .catalog.orm.dataset import ( + File as CatalogFile, + Group, +) from pysus.api.models import BaseRemoteDataset, BaseRemoteFile, BaseRemoteGroup -from sqlalchemy.orm import contains_eager, joinedload, sessionmaker +from sqlalchemy import select, orm if TYPE_CHECKING: # pragma: no cover from .client import DuckLake @@ -127,7 +130,7 @@ async def _download( if not output: output = CACHEPATH / self.name - return await self.client._download_file( + return await self.client.download( self, output, callback=callback, @@ -161,148 +164,39 @@ def _calculate(): class DuckDataset(BaseRemoteDataset): - """A dataset from the DuckLake catalog, containing groups and files. - - Each dataset manages its own DuckDB engine connected to a - per-dataset catalog file (``catalog_.db``). - - Parameters - ---------- - record : Dataset - The underlying ORM record. - client : BaseRemoteClient - The parent client instance. - """ - record: Dataset = Field(exclude=True) client: "DuckLake" = Field(exclude=True) - - _engine: Any = PrivateAttr(default=None) - _Session: Any = PrivateAttr(default=None) + adapter: DatasetAdapter = Field(exclude=True) def __init__(self, **data) -> None: super().__init__(**data) - self._cache_dir: Path = Path(CACHEPATH) / "ducklake" - self._cache_dir.mkdir(parents=True, exist_ok=True) - self._catalog_name: str = f"catalog_{self.record.name.lower()}.duckdb" - self._catalog_local: Path = self._cache_dir / self._catalog_name def __repr__(self) -> str: - """Return a string representation of the dataset. - - Returns - ------- - str - The uppercased dataset name. - """ return self.name.upper() @property def name(self) -> str: - """Return the short name of the dataset. - - Returns - ------- - str - The dataset short name. - """ - return self.record.name # type: ignore + return str(self.record.name) @property def long_name(self) -> str: - """Return the human-readable name of the dataset. - - Returns - ------- - str - The dataset display name, falling back to the short name. - """ - return "" # TODO: + return "" @property def description(self) -> str: - """Return the description of the dataset. - - Returns - ------- - str - The dataset description, or an empty string if unavailable. - """ - return "" # TODO: - - @property - def catalog_path(self) -> Path: - """Return the local path to the downloaded catalog database. - - Returns - ------- - Path - Filesystem path to the local catalog database file. - """ - return self._catalog_local + return "" async def connect( self, force: bool = False, - callback: Callable[[int, int], None] | None = None, ) -> None: - """Connect to the catalog, downloading it first if necessary. - - Parameters - ---------- - force : bool, optional - Whether to re-download and re-connect even if already connected. - """ - if self._engine and not force: - if not self._Session: - self._Session = sessionmaker(bind=self._engine) - return - if self not in self.client._datasets: self.client._datasets.append(self) - await self.client._download( - f"public/{self._catalog_name}", - self._catalog_local, - callback=callback, - ) - self._engine = await to_thread.run_sync( - lambda: self.client._setup_engine(self._catalog_local) - ) - self._Session = sessionmaker(bind=self._engine) + await self.adapter.connect(force=force) async def close(self, update_catalog: bool = False): - """Dispose the engine, optionally uploading the per-dataset catalog. - - Parameters - ---------- - update_catalog : bool, optional - Whether to upload the per-dataset catalog to remote storage. - Requires the parent client to be authenticated. - """ - if self._engine: - await to_thread.run_sync(self._engine.dispose) - self._engine = None - self._Session = None - - if update_catalog and self.client._is_authenticated: - await self._upload_catalog() - - async def _upload_catalog(self): - """Upload the per-dataset catalog to remote storage.""" - if not self.client.credentials: - raise PermissionError( - "Admin credentials required to upload catalog.", - ) - - def _upload(): - self.client._s3_client.upload_file( - str(self._catalog_local), - self.client.bucket, - f"catalog_{self.record.name.lower()}.duckdb", - ) - - await to_thread.run_sync(_upload) + await self.adapter.close(update=update_catalog) async def query( self, @@ -311,72 +205,58 @@ async def query( year: int | None = None, month: int | None = None, ) -> list[File]: - """Filter files in this dataset's catalog by group, state, year, month. - - Parameters - ---------- - group : str, optional - Group name pattern to filter by (case-insensitive ILIKE). - state : str, optional - Two-letter state code to filter by. - year : int, optional - Year to filter by. - month : int, optional - Month to filter by. - - Returns - ------- - list[File] - List of matching file objects. - """ - if not self._Session: - await self.connect() + await self.adapter.connect() def _query() -> list[CatalogFile]: - with self._Session() as session: - q = session.query(CatalogFile).options( - joinedload(CatalogFile.group), - joinedload(CatalogFile.dataset), + with self.adapter.get_session() as session: + stmt = ( + select(CatalogFile) + .filter(CatalogFile.dataset_id == self.record.id) + .options( + orm.joinedload(CatalogFile.group), + ) ) if group: - q = ( - q.join(CatalogFile.group) - .options(contains_eager(CatalogFile.group)) + stmt = ( + stmt.join(CatalogFile.group) + .options(orm.contains_eager(CatalogFile.group)) .filter(Group.name.ilike(group)) ) if state: - q = q.filter(CatalogFile.state == state.upper()) + stmt = stmt.filter(CatalogFile.state == state.upper()) if year: - q = q.filter(CatalogFile.year == year) + stmt = stmt.filter(CatalogFile.year == year) if month: - q = q.filter(CatalogFile.month == month) - results = q.all() + stmt = stmt.filter(CatalogFile.month == month) + + results = session.scalars(stmt).all() session.expunge_all() - return results + return list(results) records: list[CatalogFile] = await to_thread.run_sync(_query) return [File(record=r, dataset=self) for r in records] async def _fetch_content(self) -> list[Union["DuckGroup", File]]: - """Fetch groups and files belonging to this dataset.""" - if not self._Session: - await self.connect() + await self.adapter.connect() def _fetch(): - with self._Session() as session: - dataset = ( - session.query(Dataset) - .options( - joinedload(Dataset.groups).joinedload(Group.files), - joinedload(Dataset.files), - ) - .filter(Dataset.name == self.record.name) - .first() + with self.adapter.get_session() as session: + stmt = ( + select(Group) + .options(orm.joinedload(Group.files)) + .filter(Group.dataset_id == self.record.id) ) - if not dataset: - return [], [] + groups = session.scalars(stmt).all() + + ungrouped = session.scalars( + select(CatalogFile).filter( + CatalogFile.dataset_id == self.record.id, + CatalogFile.group_id.is_(None), + ) + ).all() + session.expunge_all() - return dataset.groups, dataset.files + return list(groups), list(ungrouped) groups, files = await to_thread.run_sync(_fetch) @@ -386,15 +266,7 @@ def _fetch(): items.extend([DuckGroup(record=g, dataset=self) for g in groups]) if files: - items.extend( - [ - File( - record=f, - dataset=self, - ) - for f in files - ] - ) + items.extend([File(record=f, dataset=self) for f in files]) return items diff --git a/pysus/api/ftp/models.py b/pysus/api/ftp/models.py index 88edb1a3..8731bc7a 100644 --- a/pysus/api/ftp/models.py +++ b/pysus/api/ftp/models.py @@ -148,7 +148,7 @@ async def _download( cache_dir.mkdir(parents=True, exist_ok=True) output = cache_dir / self.basename - return await self.client._download_file(self, output, callback) + return await self.client.download(self, output, callback) class Directory: diff --git a/pysus/api/models.py b/pysus/api/models.py index 9f0c0967..d93ed59f 100644 --- a/pysus/api/models.py +++ b/pysus/api/models.py @@ -520,7 +520,7 @@ async def datasets(self, **kwargs) -> list: """Return a list of available datasets matching *kwargs*.""" @abstractmethod - async def _download_file( + async def download( self, file: BaseRemoteFile, output: Path, diff --git a/pysus/api/types.py b/pysus/api/types.py index 2e3708a8..14d37015 100644 --- a/pysus/api/types.py +++ b/pysus/api/types.py @@ -3,6 +3,21 @@ from pydantic import AfterValidator +def _validate_s3_endpoint(v: str) -> str: + assert v == "nbg1.your-objectstorage.com" + return v + + +def _validate_s3_region(v: str) -> str: + assert v == "nbg1" + return v + + +def _validate_s3_bucket(v: str) -> str: + assert v == "pysus" + return v + + def _validate_origin(v: str) -> str: valid = (FTP, DADOSGOV, DUCKLAKE) assert v in valid, f"Invalid origin: {v!r}" @@ -93,6 +108,12 @@ def _validate_state(v: str) -> str: DADOSGOV: Annotated[str, AfterValidator(_validate_origin)] = "DadosGov" DUCKLAKE: Annotated[str, AfterValidator(_validate_origin)] = "DuckLake" +S3_ENDPOINT: Annotated[str, AfterValidator(_validate_s3_endpoint)] = ( + "nbg1.your-objectstorage.com" +) +S3_REGION: Annotated[str, AfterValidator(_validate_s3_region)] = "nbg1" +S3_BUCKET: Annotated[str, AfterValidator(_validate_s3_bucket)] = "pysus" + VARCHAR: Annotated[str, AfterValidator(_validate_column_type)] = "VARCHAR" INTEGER: Annotated[str, AfterValidator(_validate_column_type)] = "INTEGER" BIGINT: Annotated[str, AfterValidator(_validate_column_type)] = "BIGINT" diff --git a/pysus/tests/api/ducklake/test_client.py b/pysus/tests/api/ducklake/test_client.py index e50a6b05..cf045025 100644 --- a/pysus/tests/api/ducklake/test_client.py +++ b/pysus/tests/api/ducklake/test_client.py @@ -40,16 +40,12 @@ async def test_description(self): async def test_ducklake_catalog_path(self, tmp_path): with patch("pysus.api.ducklake.client.CACHEPATH", tmp_path): client = DuckLake() - assert ( - client.catalog_path == tmp_path / "ducklake" / "catalog.duckdb" - ) + assert client.catalog_path == tmp_path / "ducklake" / "catalog.duckdb" @pytest.mark.asyncio async def test_ducklake_catalog_url(self): client = DuckLake() - expected = ( - "https://nbg1.your-objectstorage.com/pysus/public/catalog.duckdb" - ) + expected = "https://nbg1.your-objectstorage.com/pysus/public/catalog.duckdb" assert client._catalog_url == expected @pytest.mark.asyncio @@ -136,9 +132,7 @@ async def test_upload_catalog_requires_auth(self): class TestDuckLakeDatasets: @pytest.mark.asyncio - async def test_datasets_creates_session_and_returns_duckdatasets( - self, tmp_path - ): + async def test_datasets_creates_session_and_returns_duckdatasets(self, tmp_path): with patch("pysus.api.ducklake.client.CACHEPATH", tmp_path): client = DuckLake() @@ -177,9 +171,7 @@ async def test_datasets_connects_if_no_session(self, tmp_path): async def _connect(*args, **kwargs): client._Session = MagicMock(return_value=mock_session) - with patch.object( - DuckLake, "connect", new=AsyncMock(side_effect=_connect) - ): + with patch.object(DuckLake, "connect", new=AsyncMock(side_effect=_connect)): def run_sync(fn, *args, **kwargs): return fn() @@ -205,9 +197,7 @@ def test_setup_engine_has_pysus_schema(self): result = client._setup_engine() calls = [str(c) for c in mock_conn.exec_driver_sql.call_args_list] - assert any( - "SET search_path" in c and "pysus,main" in c for c in calls - ) + assert any("SET search_path" in c and "pysus,main" in c for c in calls) assert result is mock_engine def test_setup_engine_no_pysus_schema(self): @@ -236,19 +226,13 @@ def test_setup_engine_with_credentials(self): mock_conn.exec_driver_sql().fetchone.return_value = None client = DuckLake( - credentials=DuckLakeCredentials( - access_key="ak", secret_key="sk" - ) + credentials=DuckLakeCredentials(access_key="ak", secret_key="sk") ) client._setup_engine() calls = [str(c) for c in mock_conn.exec_driver_sql.call_args_list] - s3_access = any( - "s3_access_key_id" in c and "ak" in c for c in calls - ) - s3_secret = any( - "s3_secret_access_key" in c and "sk" in c for c in calls - ) + s3_access = any("s3_access_key_id" in c and "ak" in c for c in calls) + s3_secret = any("s3_secret_access_key" in c and "sk" in c for c in calls) assert s3_access assert s3_secret @@ -288,9 +272,7 @@ def run_sync(fn, *args, **kwargs): "pysus.api.ducklake.client.to_thread.run_sync", side_effect=run_sync, ): - with patch.object( - client, "_setup_engine", return_value=MagicMock() - ): + with patch.object(client, "_setup_engine", return_value=MagicMock()): await client.connect() mock_dl.assert_awaited_once_with( client._catalog_local, @@ -320,9 +302,7 @@ async def __anext__(self): "pysus.api.ducklake.client.httpx.AsyncClient", return_value=mock_client, ) - sleep_patcher = patch( - "pysus.api.ducklake.client.sleep", new_callable=AsyncMock - ) + sleep_patcher = patch("pysus.api.ducklake.client.sleep", new_callable=AsyncMock) first_stream_cm = MagicMock() first_resp = MagicMock() @@ -345,7 +325,7 @@ async def success_iter(): mock_client.stream.side_effect = [first_stream_cm, second_stream_cm] with httpx_patcher, sleep_patcher as mock_sleep: - await client._download(remote_path, local_path) + await client.download(remote_path, local_path) assert local_path.exists() assert local_path.read_bytes() == b"data" @@ -371,9 +351,7 @@ async def __anext__(self): "pysus.api.ducklake.client.httpx.AsyncClient", return_value=mock_client, ) - sleep_patcher = patch( - "pysus.api.ducklake.client.sleep", new_callable=AsyncMock - ) + sleep_patcher = patch("pysus.api.ducklake.client.sleep", new_callable=AsyncMock) stream_cm = MagicMock() resp = MagicMock() @@ -386,7 +364,7 @@ async def __anext__(self): with httpx_patcher, sleep_patcher as mock_sleep: with pytest.raises(OSError, match="Connection dropped"): - await client._download(remote_path, local_path) + await client.download(remote_path, local_path) assert mock_client.stream.call_count == 5 assert mock_sleep.await_count == 4 @@ -420,7 +398,7 @@ async def success_iter(): "pysus.api.ducklake.client.httpx.AsyncClient", return_value=mock_client, ): - await client._download(remote_path, local_path, callback=callback) + await client.download(remote_path, local_path, callback=callback) callback.assert_any_call(5, 10) callback.assert_any_call(10, 10) @@ -578,12 +556,8 @@ class TestDuckLakeDownloadFile: @pytest.mark.asyncio async def test_download_file_invalid_type_raises(self): client = DuckLake() - with pytest.raises( - ValueError, match="FTP File was not properly instantiated" - ): - await client._download_file( - "not-a-file", Path("/tmp/test") - ) # type: ignore + with pytest.raises(ValueError, match="FTP File was not properly instantiated"): + await client.download("not-a-file", Path("/tmp/test")) # type: ignore @pytest.mark.asyncio async def test_download_file_valid(self, tmp_path): @@ -604,7 +578,7 @@ async def test_download_file_valid(self, tmp_path): output = tmp_path / "output.csv" with patch.object(client, "_download") as mock_dl: - result = await client._download_file(f, output) + result = await client.download(f, output) mock_dl.assert_awaited_once_with(record.path, output, callback=None) assert result == output @@ -623,9 +597,7 @@ async def test_upload_catalog_with_datasets(self, tmp_path): ds._catalog_local = local_db ds._catalog_name = "catalog_test.duckdb" - with patch.object( - DuckLake, "datasets", new=AsyncMock(return_value=[ds]) - ): + with patch.object(DuckLake, "datasets", new=AsyncMock(return_value=[ds])): await client._upload_catalog() client._s3_client.upload_file.assert_called_once_with( str(local_db), client.bucket, ds._catalog_name @@ -643,8 +615,6 @@ async def test_upload_catalog_skips_missing_local(self, tmp_path): ds._catalog_local = nonexistent ds._catalog_name = "catalog_test.duckdb" - with patch.object( - DuckLake, "datasets", new=AsyncMock(return_value=[ds]) - ): + with patch.object(DuckLake, "datasets", new=AsyncMock(return_value=[ds])): await client._upload_catalog() client._s3_client.upload_file.assert_not_called() From 042f20e0bc57c74867d399af172395e61e58055b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Sat, 13 Jun 2026 08:15:59 -0300 Subject: [PATCH 2/4] chore: implement context managers to every duckdb interaction --- pysus/api/ducklake/README.md | 37 ++++++++++++ pysus/api/ducklake/__init__.py | 7 --- pysus/api/ducklake/catalog/adapters.py | 40 ++++++++++++- pysus/api/ducklake/client.py | 83 +++++++++++++++++++------- pysus/api/ducklake/models.py | 67 ++++++++++++--------- 5 files changed, 174 insertions(+), 60 deletions(-) diff --git a/pysus/api/ducklake/README.md b/pysus/api/ducklake/README.md index e69de29b..2bd01f55 100644 --- a/pysus/api/ducklake/README.md +++ b/pysus/api/ducklake/README.md @@ -0,0 +1,37 @@ +# DuckLake Catalog Component + +This module provides the application-level models and data adapters required to interface with remote DuckLake resources over S3. It wraps low-level database operations into unified interfaces (`File`, `DuckDataset`, and `DuckGroup`) and implements resilient database connection management via async context managers. + +## Features + +* **Deterministic Resource Management**: Implements asynchronous context managers (`async with`) across clients and database adapters to prevent DuckDB file locks and connection leaks. +* **Fail-Safe Cleanups**: Features fallback `__del__` destructors to safely terminate remaining active engines during garbage collection or interpreter shutdown. +* **Intelligent Syncing**: Employs an `update_on_close` mechanism to optionally push state changes back to S3 automatically upon exiting a context. +* **SQLAlchemy Eager Loading Fixes**: Optimizes attribute mappings using isolated path strategies (`joinedload` vs. `contains_eager`) based on query parameters. +* **Resilient S3 Verifications**: Gracefully intercepts 404 responses during S3 download handshakes to stop failing connection retry loops early. + +--- + +## Architecture Overview + +The system separates the raw database management layer (Adapters) from the client wrapper layer (Client Models). + +1. **Adapters (`BaseAdapter`)**: Track local and remote `.duckdb` target states, manage connections, handle S3 transfers, and expose scoped SQLAlchemy transaction sessions. +2. **Client Components (`DuckLake`)**: Coordinate high-level actions, parse credential models, route queries, and handle collection loops. + +--- + +## Lifecycle & Connection Handling + +### Using Context Managers (Recommended) + +Using `async with` blocks guarantees deterministic resource teardown. The moment execution exits the context layout block—even due to a runtime crash—all engines are disposed of cleanly. + +```python +from pysus.api.ducklake.client import DuckLake + +async with DuckLake() as dl: + datasets = await dl.datasets() + +sia = datasets[4] # e.g., SIA +files = await sia.query(state="SP", year=2026) diff --git a/pysus/api/ducklake/__init__.py b/pysus/api/ducklake/__init__.py index 4ba051d0..e69de29b 100644 --- a/pysus/api/ducklake/__init__.py +++ b/pysus/api/ducklake/__init__.py @@ -1,7 +0,0 @@ -"""DuckLake subpackage for interacting with the PySUS S3 catalog. - -Provides a DuckDB-based client for querying and downloading -public health datasets stored in object storage. -""" - -from .client import DuckLake as DuckLakeClient # noqa diff --git a/pysus/api/ducklake/catalog/adapters.py b/pysus/api/ducklake/catalog/adapters.py index eb16883b..4e6694cd 100644 --- a/pysus/api/ducklake/catalog/adapters.py +++ b/pysus/api/ducklake/catalog/adapters.py @@ -1,5 +1,6 @@ from abc import ABC from pathlib import Path +import asyncio import httpx from anyio import to_thread @@ -12,6 +13,7 @@ from pysus import CACHEPATH from pysus.api import types from pysus.api.ducklake.functional import download_s3, upload_s3 +from pysus.api.ducklake.catalog.orm.dataset import DatasetBase class DuckLakeCredentials(BaseModel): @@ -25,12 +27,17 @@ class BaseAdapter(ABC): db_remote: Path def __init__( - self, engine=None, credentials: DuckLakeCredentials | None = None, **data + self, + engine=None, + credentials: DuckLakeCredentials | None = None, + update_on_close: bool = False, + **data, ) -> None: self._engine = engine self._session_factory = None self.cache_dir.mkdir(parents=True, exist_ok=True) self.credentials = credentials + self.update_on_close = update_on_close @property def remote_url(self) -> str: @@ -64,6 +71,7 @@ def setup_engine( with engine.connect() as conn: conn.exec_driver_sql("INSTALL ducklake; LOAD ducklake;") + conn.exec_driver_sql("CREATE SCHEMA IF NOT EXISTS pysus;") has_pysus = conn.exec_driver_sql( "SELECT 1 FROM information_schema.schemata WHERE schema_name = 'pysus'" @@ -90,6 +98,7 @@ def setup_engine( conn.commit() + DatasetBase.metadata.create_all(bind=engine) return engine async def _download_catalog(self, local_path: Path, remote_path: str) -> None: @@ -106,8 +115,14 @@ async def _download_catalog(self, local_path: Path, remote_path: str) -> None: async with httpx.AsyncClient(follow_redirects=True) as client: try: head = await client.head(url) + + if head.status_code == 404: + return + head.raise_for_status() remote_size = int(head.headers.get("content-length", 0)) + except httpx.HTTPStatusError: + return except Exception: remote_size = 0 @@ -153,6 +168,28 @@ async def close(self, update: bool = False) -> None: self._engine = None self._session_factory = None + def __del__(self) -> None: + if not hasattr(self, "_engine") or not self._engine: + return + try: + loop = asyncio.get_running_loop() + if loop.is_running(): + loop.create_task(self.close(update=False)) + except RuntimeError: + try: + asyncio.run(self.close(update=False)) + except Exception: # noqa + pass + except Exception: # noqa + pass + + async def __aenter__(self): + await self.connect() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + await self.close(update=self.update_on_close) + class CatalogAdapter(BaseAdapter): def __init__(self, engine=None, **data) -> None: @@ -174,3 +211,4 @@ def __init__(self, engine=None, **data) -> None: super().__init__(engine=engine, **data) self.db_local: Path = self.cache_dir / "columns.duckdb" self.db_remote: str = "public/columns.duckdb" + diff --git a/pysus/api/ducklake/client.py b/pysus/api/ducklake/client.py index ee16231c..f14fe96c 100644 --- a/pysus/api/ducklake/client.py +++ b/pysus/api/ducklake/client.py @@ -4,11 +4,12 @@ capabilities backed by per-dataset DuckDB engines. """ +import asyncio from collections.abc import Callable from pathlib import Path from anyio import to_thread -from pydantic import SecretStr, PrivateAttr +from pydantic import SecretStr, PrivateAttr, Field from pysus.api.models import BaseRemoteClient from pysus.api.types import DUCKLAKE @@ -21,13 +22,17 @@ class DuckLake(BaseRemoteClient): credentials: DuckLakeCredentials | None = None + update_on_close: bool = Field(default=False, exclude=True) _datasets: list[DuckDataset] = PrivateAttr(default_factory=list) + _catalog_adap: CatalogAdapter = PrivateAttr() - def __init__(self, engine=None, **data) -> None: + def __init__(self, engine=None, update_on_close: bool = False, **data) -> None: super().__init__(**data) - self.catalog_adap = CatalogAdapter( + self.update_on_close = update_on_close + self._catalog_adap = CatalogAdapter( engine=engine, credentials=self.credentials, + update_on_close=self.update_on_close, ) @property @@ -43,24 +48,31 @@ def description(self) -> str: return "" async def datasets(self, **kwargs) -> list[DuckDataset]: - await self.catalog_adap.connect() - def _fetch(): - with self.catalog_adap.get_session() as session: + with self._catalog_adap.get_session() as session: results = session.query(Dataset).all() session.expunge_all() return results - records = await to_thread.run_sync(_fetch) - duck_datasets: list[DuckDataset] = [] - for rec in records: - dataset_adapter = DatasetAdapter( - name=str(rec.name), credentials=self.credentials - ) - duck_datasets.append( - DuckDataset(record=rec, client=self, adapter=dataset_adapter) - ) + + async with self._catalog_adap: + records = await to_thread.run_sync(_fetch) + + for rec in records: + dataset_adapter = DatasetAdapter( + name=str(rec.name), + credentials=self.credentials, + update_on_close=self.update_on_close, + ) + duck_datasets.append( + DuckDataset( + record=rec, + client=self, + adapter=dataset_adapter, + update_on_close=self.update_on_close, + ) + ) self._datasets = duck_datasets return duck_datasets @@ -75,18 +87,21 @@ async def login( access_key=SecretStr(access_key), secret_key=SecretStr(secret_key), ) - self.catalog_adap.credentials = self.credentials - await self.catalog_adap.connect(force=True) + self._catalog_adap.credentials = self.credentials + await self._catalog_adap.connect(force=True) async def connect(self, force: bool = False) -> None: - await self.catalog_adap.connect(force=force) + await self._catalog_adap.connect(force=force) + + async def close(self, update_catalog: bool | None = None) -> None: + should_update = ( + self.update_on_close if update_catalog is None else update_catalog + ) - async def close(self, update_catalog: bool = False) -> None: for ds in self._datasets: - await ds.close(update_catalog=update_catalog) + await ds.close(update_catalog=should_update) - await self.catalog_adap.close(update=update_catalog) - self._datasets.clear() + await self._catalog_adap.close(update=should_update) async def download( self, @@ -95,7 +110,7 @@ async def download( callback: Callable[[int, int], None] | None = None, ) -> Path: if not isinstance(file, File): - raise ValueError("FTP File was not properly instantiated") + raise ValueError("DuckLake File was not properly instantiated") access_key = ( self.credentials.access_key.get_secret_value() if self.credentials else None @@ -113,5 +128,27 @@ async def download( ) return output + async def __aenter__(self): + await self.connect() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + await self.close(update_catalog=None) + + def __del__(self) -> None: + if not hasattr(self, "_catalog_adap"): + return + try: + loop = asyncio.get_running_loop() + if loop.is_running(): + loop.create_task(self.close(update_catalog=False)) + except RuntimeError: + try: + asyncio.run(self.close(update_catalog=False)) + except Exception: + pass + except Exception: + pass + DuckDataset.model_rebuild(_types_namespace={"DuckLake": DuckLake}) diff --git a/pysus/api/ducklake/models.py b/pysus/api/ducklake/models.py index 449cf635..7de750a7 100644 --- a/pysus/api/ducklake/models.py +++ b/pysus/api/ducklake/models.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union from anyio import to_thread -from pydantic import Field, PrivateAttr +from pydantic import Field from pysus import CACHEPATH from .catalog.adapters import DatasetAdapter from .catalog.orm.default import Dataset @@ -164,9 +164,10 @@ def _calculate(): class DuckDataset(BaseRemoteDataset): - record: Dataset = Field(exclude=True) + record: "Dataset" = Field(exclude=True) client: "DuckLake" = Field(exclude=True) - adapter: DatasetAdapter = Field(exclude=True) + adapter: "DatasetAdapter" = Field(exclude=True) + update_on_close: bool = Field(default=False, exclude=True) def __init__(self, **data) -> None: super().__init__(**data) @@ -180,11 +181,11 @@ def name(self) -> str: @property def long_name(self) -> str: - return "" + return str(self.record.long_name) @property def description(self) -> str: - return "" + return str(self.record.description) async def connect( self, @@ -195,8 +196,11 @@ async def connect( await self.adapter.connect(force=force) - async def close(self, update_catalog: bool = False): - await self.adapter.close(update=update_catalog) + async def close(self, update_catalog: bool | None = None): + should_update = ( + self.update_on_close if update_catalog is None else update_catalog + ) + await self.adapter.close(update=should_update) async def query( self, @@ -205,23 +209,21 @@ async def query( year: int | None = None, month: int | None = None, ) -> list[File]: - await self.adapter.connect() - def _query() -> list[CatalogFile]: with self.adapter.get_session() as session: - stmt = ( - select(CatalogFile) - .filter(CatalogFile.dataset_id == self.record.id) - .options( - orm.joinedload(CatalogFile.group), - ) + stmt = select(CatalogFile).filter( + CatalogFile.dataset_id == self.record.id, ) + if group: stmt = ( stmt.join(CatalogFile.group) .options(orm.contains_eager(CatalogFile.group)) .filter(Group.name.ilike(group)) ) + else: + stmt = stmt.options(orm.joinedload(CatalogFile.group)) + if state: stmt = stmt.filter(CatalogFile.state == state.upper()) if year: @@ -233,12 +235,11 @@ def _query() -> list[CatalogFile]: session.expunge_all() return list(results) - records: list[CatalogFile] = await to_thread.run_sync(_query) - return [File(record=r, dataset=self) for r in records] + async with self.adapter: + records: list[CatalogFile] = await to_thread.run_sync(_query) + return [File(record=r, dataset=self) for r in records] async def _fetch_content(self) -> list[Union["DuckGroup", File]]: - await self.adapter.connect() - def _fetch(): with self.adapter.get_session() as session: stmt = ( @@ -258,17 +259,25 @@ def _fetch(): session.expunge_all() return list(groups), list(ungrouped) - groups, files = await to_thread.run_sync(_fetch) + async with self.adapter: + groups, files = await to_thread.run_sync(_fetch) + + items: list[Union[DuckGroup, File]] = [] - items: list[Union["DuckGroup", File]] = [] + if groups: + items.extend([DuckGroup(record=g, dataset=self) for g in groups]) - if groups: - items.extend([DuckGroup(record=g, dataset=self) for g in groups]) + if files: + items.extend([File(record=f, dataset=self) for f in files]) - if files: - items.extend([File(record=f, dataset=self) for f in files]) + return items - return items + async def __aenter__(self): + await self.connect() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + await self.close(update_catalog=None) class DuckGroup(BaseRemoteGroup): @@ -294,7 +303,7 @@ def name(self) -> str: str The group short name. """ - return self.record.name # type: ignore + return str(self.record.name) @property def long_name(self) -> str: @@ -305,7 +314,7 @@ def long_name(self) -> str: str The group display name, falling back to the short name. """ - return self.record.long_name or self.name # type: ignore + return str(self.record.long_name) @property def description(self) -> str: @@ -316,7 +325,7 @@ def description(self) -> str: str The group description, or an empty string if unavailable. """ - return self.record.description # type: ignore + return str(self.record.description) async def _fetch_files(self) -> list[BaseRemoteFile]: """Fetch the list of files belonging to this group.""" From 7c9bfbbe6d2a64d1ace6ea36adc1ebc01c76c75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Sat, 13 Jun 2026 11:32:26 -0300 Subject: [PATCH 3/4] chore: make download only the default on _impl --- README.md | 27 ++- pysus/api/_impl/databases.py | 173 ++++++++--------- pysus/api/client.py | 100 +++++----- pysus/api/dadosgov/client.py | 2 +- pysus/api/dadosgov/models.py | 16 +- pysus/api/ducklake/README.md | 6 +- pysus/api/ducklake/catalog/adapters.py | 118 +++++++----- pysus/api/ducklake/catalog/orm/columns.py | 6 +- pysus/api/ducklake/catalog/orm/dataset.py | 62 +++++-- pysus/api/ducklake/catalog/orm/default.py | 14 +- pysus/api/ducklake/client.py | 77 +++++--- pysus/api/ducklake/functional.py | 41 ++--- pysus/api/ducklake/models.py | 215 +++++++--------------- pysus/api/ftp/client.py | 4 +- pysus/management/client.py | 188 ++++++++++--------- pysus/tests/api/ducklake/test_catalog.py | 10 +- pysus/tests/api/ducklake/test_client.py | 63 +++++-- pysus/tests/api/ducklake/test_models.py | 2 +- 18 files changed, 588 insertions(+), 536 deletions(-) diff --git a/README.md b/README.md index f12070d6..f0a029f4 100644 --- a/README.md +++ b/README.md @@ -52,28 +52,39 @@ docker compose -f docker/docker-compose.yaml down ### Simplified Database Functions (New in 2.0) -The easiest way to get data as a pandas DataFrame: +By default, the high-level convenience functions query and download data locally, returning a list of paths to the downloaded Parquet files. This allows you to inspect the file structure or load them with your preferred tool (e.g., pandas, Polars, DuckDB). ```python from pysus import sinan, sinasc, sim, sih, sia, pni, ibge, cnes, ciha -# Download SINAN Dengue data for 2000 -df = sinan(disease="deng", year=2000) +# Download SINAN Dengue data for 2000 and return a list of Parquet paths +parquet_files = sinan(disease="deng", year=2000) # Multiple years -df = sinan(disease="deng", year=[2023, 2024]) +parquet_files = sinan(disease="deng", year=[2023, 2024]) # SINASC births for SĂ£o Paulo, 2020-2023 -df = sinasc(state="SP", year=[2020, 2021, 2022, 2023]) +parquet_files = sinasc(state="SP", year=[2020, 2021, 2022, 2023]) # SIM mortality data -df = sim(state="SP", year=2024) +parquet_files = sim(state="SP", year=2024) # SIH hospitalizations with month -df = sih(state="SP", year=2024, month=[1, 2, 3]) +parquet_files = sih(state="SP", year=2024, month=[1, 2, 3]) # CNES health facilities -df = cnes(state="SP", year=2024, month=1) +parquet_files = cnes(state="SP", year=2024, month=1) +``` + +### Loading as a DataFrame Directly +If you prefer to load and combine the data automatically into a single pandas DataFrame, pass the as_dataframe=True parameter to any of the functions: + +```python +import pandas as pd +from pysus import sinan + +# Download and return a concatenated pandas DataFrame +df = sinan(disease="deng", year=2024, as_dataframe=True) ``` ### Listing the files diff --git a/pysus/api/_impl/databases.py b/pysus/api/_impl/databases.py index fa5a7a8c..c5321c9f 100644 --- a/pysus/api/_impl/databases.py +++ b/pysus/api/_impl/databases.py @@ -7,6 +7,14 @@ and hospitalisation records (CIHA). """ +import asyncio +from typing import Literal + +import pandas as pd +from pysus.api import types +from pysus.api.client import PySUS +from tqdm import tqdm + __all__ = [ "sinan", "sinasc", @@ -20,14 +28,6 @@ "list_files", ] -import asyncio -from typing import Literal - -import pandas as pd -from pysus.api import types -from pysus.api.client import PySUS -from tqdm import tqdm - def _fetch_data( dataset: str, @@ -36,46 +36,42 @@ def _fetch_data( year: int | list[int] | None = None, month: int | list[int] | None = None, show_progress: bool = True, + as_dataframe: bool = False, **kwargs, -) -> pd.DataFrame: - """Query, download, and concatenate Parquet files for a given dataset. +) -> list[str] | pd.DataFrame: + """Query, download, and process Parquet files for a given dataset. Internally creates an async event loop, queries the PySUS API for matching - files, downloads them, and reads them into a single DataFrame. + files, and downloads them. By default, returns a list of local file paths. Parameters ---------- dataset : str - Name of the dataset (e.g. ``"sinan"``, ``"sinasc"``). + Name of the dataset (e.g. "sinan", "sinasc"). group : str, optional Group or disease code to filter by. state : str, optional - Two-letter state abbreviation (e.g. ``"RJ"``). + Two-letter state abbreviation (e.g. "RJ"). year : int | list[int], optional Year or list of years to fetch. month : int | list[int], optional Month or list of months to fetch. show_progress : bool, optional - Whether to display a tqdm progress bar during download. Default is - ``True``. + Whether to display a tqdm progress bar during download. Default is True. + as_dataframe : bool, optional + Whether to concatenate and return the data as a pandas DataFrame. + Default is False. **kwargs Additional arguments forwarded to :meth:`PySUS.read_parquet`. Returns ------- - pd.DataFrame - Concatenated data from all matching Parquet files. Returns an empty - DataFrame when no files are found. - - Raises - ------ - RuntimeError - If an event loop is already running but ``nest_asyncio`` is not - installed. + list[str] | pd.DataFrame + A list of paths to the downloaded Parquet files by default. If + as_dataframe is True, returns a concatenated DataFrame. """ async def _fetch(): - """Coroutine that performs the actual API query, download, and read.""" async with PySUS() as pysus: years = [year] if isinstance(year, int) else (year or [None]) @@ -102,20 +98,23 @@ async def _fetch(): unit="file", ): f = await pysus.download(file) - paths.append(f.path) + paths.append(str(f.path)) else: for file in files: f = await pysus.download(file) - paths.append(f.path) - - return ( - pysus.read_parquet( - paths, - **kwargs, - ).df() - if paths - else pd.DataFrame() - ) + paths.append(str(f.path)) + + if as_dataframe: + return ( + pysus.read_parquet( + paths, + **kwargs, + ).df() + if paths + else pd.DataFrame() + ) + + return paths try: loop = asyncio.get_running_loop() @@ -124,7 +123,7 @@ async def _fetch(): if loop and loop.is_running(): try: - import nest_asyncio # noqa: PLC0415 + import nest_asyncio nest_asyncio.apply() except ImportError: @@ -192,7 +191,7 @@ def sinan( ], year: int | list[int], **kwargs, -) -> pd.DataFrame: +) -> list[str] | pd.DataFrame: """Fetch SINAN records for a given disease and year(s). SINAN (Sistema de InformaĂ§Ă£o de Agravos de NotificaĂ§Ă£o) is the Brazilian @@ -201,7 +200,7 @@ def sinan( Parameters ---------- disease : Literal - Disease code (e.g. ``"DENG"`` for dengue, ``"ZIKA"`` for zika). + Disease code (e.g. "DENG" for dengue, "ZIKA" for zika). year : int | list[int] Year or list of years to fetch. **kwargs @@ -209,13 +208,14 @@ def sinan( Returns ------- - pd.DataFrame - SINAN records for the specified disease and year(s). + list[str] | pd.DataFrame + List of downloaded Parquet paths, or a DataFrame if specified. """ return _fetch_data( dataset="sinan", group=disease.upper(), year=year, + **kwargs, ) @@ -224,16 +224,16 @@ def sinasc( year: int | list[int], group: str | None = None, **kwargs, -) -> pd.DataFrame: +) -> list[str] | pd.DataFrame: """Fetch SINASC birth certificates for a given state, year(s), and group. - SINASC (Sistema de InformaĂ§Ă£o sobre Nascidos Vivos) is the Brazilian live + SINASC (Sistema de InformaĂ§Ă£o sobre Nascidos Vivo) is the Brazilian live birth information system. Parameters ---------- state : types.State - Two-letter state abbreviation (e.g. ``"RJ"``). + Two-letter state abbreviation (e.g. "RJ"). year : int | list[int] Year or list of years to fetch. group : str, optional @@ -243,14 +243,15 @@ def sinasc( Returns ------- - pd.DataFrame - SINASC birth records for the specified state, year(s), and group. + list[str] | pd.DataFrame + List of downloaded Parquet paths, or a DataFrame if specified. """ return _fetch_data( dataset="sinasc", state=state.upper(), group=group, year=year, + **kwargs, ) @@ -259,7 +260,7 @@ def sim( year: int | list[int], group: str | None = None, **kwargs, -) -> pd.DataFrame: +) -> list[str] | pd.DataFrame: """Fetch SIM mortality records for a given state, year(s), and group. SIM (Sistema de InformaĂ§Ă£o sobre Mortalidade) is the Brazilian mortality @@ -268,7 +269,7 @@ def sim( Parameters ---------- state : State - Two-letter state abbreviation (e.g. ``"RJ"``). + Two-letter state abbreviation (e.g. "RJ"). year : int | list[int] Year or list of years to fetch. group : str, optional @@ -278,14 +279,15 @@ def sim( Returns ------- - pd.DataFrame - SIM mortality records for the specified state, year(s), and group. + list[str] | pd.DataFrame + List of downloaded Parquet paths, or a DataFrame if specified. """ return _fetch_data( dataset="sim", state=state.upper(), group=group, year=year, + **kwargs, ) @@ -295,7 +297,7 @@ def sih( month: int | list[int], group: str | None = None, **kwargs, -) -> pd.DataFrame: +) -> list[str] | pd.DataFrame: """Fetch SIH hospital admissions for a state, year, month, and group. SIH (Sistema de InformaĂ§Ă£o Hospitalar) is the Brazilian hospital @@ -304,7 +306,7 @@ def sih( Parameters ---------- state : types.State - Two-letter state abbreviation (e.g. ``"RJ"``). + Two-letter state abbreviation (e.g. "RJ"). year : int | list[int] Year or list of years to fetch. month : int | list[int] @@ -316,8 +318,8 @@ def sih( Returns ------- - pd.DataFrame - SIH hospital admission records. + list[str] | pd.DataFrame + List of downloaded Parquet paths, or a DataFrame if specified. """ return _fetch_data( dataset="sih", @@ -325,6 +327,7 @@ def sih( group=group, year=year, month=month, + **kwargs, ) @@ -334,7 +337,7 @@ def sia( month: int | list[int], group: str | None = None, **kwargs, -) -> pd.DataFrame: +) -> list[str] | pd.DataFrame: """Fetch SIA ambulatory care for a state, year, month, and group. SIA (Sistema de InformaĂ§Ă£o Ambulatorial) is the Brazilian ambulatory care @@ -343,7 +346,7 @@ def sia( Parameters ---------- state : types.State - Two-letter state abbreviation (e.g. ``"RJ"``). + Two-letter state abbreviation (e.g. "RJ"). year : int | list[int] Year or list of years to fetch. month : int | list[int] @@ -355,8 +358,8 @@ def sia( Returns ------- - pd.DataFrame - SIA ambulatory care records. + list[str] | pd.DataFrame + List of downloaded Parquet paths, or a DataFrame if specified. """ return _fetch_data( dataset="sia", @@ -364,6 +367,7 @@ def sia( group=group, year=year, month=month, + **kwargs, ) @@ -372,7 +376,7 @@ def pni( year: int | list[int], group: str | None = None, **kwargs, -) -> pd.DataFrame: +) -> list[str] | pd.DataFrame: """Fetch PNI immunisation records for a given state, year(s), and group. PNI (Programa Nacional de Imunizações) is the Brazilian national @@ -381,7 +385,7 @@ def pni( Parameters ---------- state : State - Two-letter state abbreviation (e.g. ``"RJ"``). + Two-letter state abbreviation (e.g. "RJ"). year : int | list[int] Year or list of years to fetch. group : str, optional @@ -391,14 +395,15 @@ def pni( Returns ------- - pd.DataFrame - PNI immunisation records. + list[str] | pd.DataFrame + List of downloaded Parquet paths, or a DataFrame if specified. """ return _fetch_data( dataset="pni", state=state.upper(), group=group, year=year, + **kwargs, ) @@ -406,7 +411,7 @@ def ibge( year: int | list[int], group: str | None = None, **kwargs, -) -> pd.DataFrame: +) -> list[str] | pd.DataFrame: """Fetch IBGE census data for given year(s) and optional group. IBGE (Instituto Brasileiro de Geografia e EstatĂ­stica) provides census @@ -423,10 +428,10 @@ def ibge( Returns ------- - pd.DataFrame - IBGE census data for the specified year(s) and group. + list[str] | pd.DataFrame + List of downloaded Parquet paths, or a DataFrame if specified. """ - return _fetch_data(dataset="ibge", group=group, year=year) + return _fetch_data(dataset="ibge", group=group, year=year, **kwargs) def cnes( @@ -435,7 +440,7 @@ def cnes( month: int | list[int], group: str | None = None, **kwargs, -) -> pd.DataFrame: +) -> list[str] | pd.DataFrame: """Fetch CNES health facilities for a state, year, month, and group. CNES (Cadastro Nacional de Estabelecimentos de SaĂºde) is the Brazilian @@ -444,7 +449,7 @@ def cnes( Parameters ---------- state : State - Two-letter state abbreviation (e.g. ``"RJ"``). + Two-letter state abbreviation (e.g. "RJ"). year : int | list[int] Year or list of years to fetch. month : int | list[int] @@ -456,8 +461,8 @@ def cnes( Returns ------- - pd.DataFrame - CNES health-facility records. + list[str] | pd.DataFrame + List of downloaded Parquet paths, or a DataFrame if specified. """ return _fetch_data( dataset="cnes", @@ -465,6 +470,7 @@ def cnes( group=group, year=year, month=month, + **kwargs, ) @@ -474,7 +480,7 @@ def ciha( month: int | list[int], group: str | None = "CIHA", **kwargs, -) -> pd.DataFrame: +) -> list[str] | pd.DataFrame: """Fetch CIHA hospitalisation records for state, year, month, and group. CIHA (ComunicaĂ§Ă£o de InternaĂ§Ă£o Hospitalar) provides hospitalisation @@ -483,20 +489,20 @@ def ciha( Parameters ---------- state : State - Two-letter state abbreviation (e.g. ``"RJ"``). + Two-letter state abbreviation (e.g. "RJ"). year : int | list[int] Year or list of years to fetch. month : int | list[int] Month or list of months to fetch. group : str, optional - Additional grouping code. Default is ``"CIHA"``. - ``**kwargs`` + Additional grouping code. Default is "CIHA". + **kwargs Additional arguments forwarded to :func:`_fetch_data`. Returns ------- - pd.DataFrame - CIHA hospitalisation records. + list[str] | pd.DataFrame + List of downloaded Parquet paths, or a DataFrame if specified. """ return _fetch_data( dataset="ciha", @@ -504,6 +510,7 @@ def ciha( group=group, year=year, month=month, + **kwargs, ) @@ -518,20 +525,19 @@ def list_files( ) -> pd.DataFrame: """List catalog files filtered by client, group, state, year, and month. - Queries the PySUS API metadata and returns a DataFrame with file name, - path, dataset, group, year, month, state, and last-modified timestamp for - every matching file without downloading the actual data. + Queries the PySUS API metadata and returns a DataFrame with file data + without downloading the actual files. Parameters ---------- dataset : Literal - Dataset name (e.g. ``"SINAN"``, ``"SINASC"``, etc.). + Dataset name (e.g. "SINAN", "SINASC", etc.). client : Origin, optional Data source client to query. group : str, optional Group or disease code to filter by. state : str, optional - Two-letter state abbreviation (e.g. ``"RJ"``). + Two-letter state abbreviation (e.g. "RJ"). year : int | list[int], optional Year or list of years to filter by. month : int | list[int], optional @@ -542,12 +548,11 @@ def list_files( Returns ------- pd.DataFrame - DataFrame with columns ``name``, ``path``, ``dataset``, ``group``, - ``year``, ``month``, ``state``, and ``modify``. + DataFrame with columns name, path, dataset, group, year, month, state, + and modify. """ async def _list(): - """Coroutine that queries the PySUS API and builds the file list.""" async with PySUS() as pysus: years = [year] if isinstance(year, int) else (year or [None]) diff --git a/pysus/api/client.py b/pysus/api/client.py index 95cf8f06..2854f075 100644 --- a/pysus/api/client.py +++ b/pysus/api/client.py @@ -100,14 +100,8 @@ def __init__(self, db_path: Path = CACHEPATH / "config.db"): self._dadosgov: DadosGovClient | None = None async def __aenter__(self): - """Set up DuckLake catalog and return self as async context manager.""" - self._ducklake = DuckLake() await self._ducklake.connect() - self._attach_client_catalog( - "ducklake", - str(self._ducklake.catalog_path), - ) return self async def __aexit__(self, exc_type, exc_val, exc_tb): @@ -123,14 +117,9 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): async def get_ducklake(self) -> DuckLake: """Return the DuckLake client, initializing it lazily if needed.""" - if self._ducklake is None: self._ducklake = DuckLake() await self._ducklake.connect() - self._attach_client_catalog( - "ducklake", - str(self._ducklake.catalog_path), - ) return self._ducklake async def get_dadosgov(self, access_token: str | None) -> DadosGovClient: @@ -181,19 +170,6 @@ async def get_local_file( return await ExtensionFactory.instantiate(str(record.path)) - def _attach_client_catalog(self, name: str, path: str): - """Attach an external DuckDB catalog to the engine if not attached.""" - - abs_path = str(Path(path).absolute()) - with self.engine.connect() as conn: - q = "SELECT database_name FROM duckdb_databases() WHERE path = ?" - existing = conn.exec_driver_sql(q, (abs_path,)).fetchone() - - if not existing: - conn.exec_driver_sql( - f"ATTACH '{abs_path}' AS {name} (READ_ONLY)", - ) - def _get_dest_path(self, file: BaseRemoteFile) -> Path: """Build the local filesystem path for a given remote file.""" @@ -403,7 +379,9 @@ async def download_to_parquet( if hasattr(local_file, "to_parquet"): original_path = local_file.path - parquet_file = await local_file.to_parquet(callback=callback) + parquet_file = await local_file.to_parquet( + callback=callback, + ) parquet_file.add_dv = add_dv await self._update_state( @@ -480,28 +458,28 @@ def get_completed_remote_paths(self) -> set[str]: async def query( self, client: Origin | None = None, - dataset: str | None = None, - group: str | None = None, - state: str | None = None, - year: int | None = None, - month: int | None = None, - ): + dataset: str | list[str] | None = None, + group: str | list[str] | None = None, + state: str | list[str] | None = None, + year: int | list[int] | None = None, + month: int | list[int] | None = None, + ) -> list[BaseRemoteFile]: """Query available datasets through the DuckLake catalog. Parameters ---------- client : Origin, optional Source client to filter by. - dataset : str, optional - Dataset name to filter by. - group : str, optional - Group name pattern to filter by (case-insensitive ILIKE). - state : str, optional - Two-letter state code to filter by. - year : int, optional - Year to filter by. - month : int, optional - Month to filter by. + dataset : str or list of str, optional + Dataset name(s) to filter by. + group : str or list of str, optional + Group name pattern(s) to filter by (case-insensitive ILIKE). + state : str or list of str, optional + Two-letter state code(s) to filter by. + year : int or list of int, optional + Year(s) to filter by. + month : int or list of int, optional + Month(s) to filter by. Returns ------- @@ -517,32 +495,32 @@ async def query( all_datasets = await self._ducklake.datasets() if dataset: - matching = [d for d in all_datasets if d.name.lower() == dataset.lower()] - if not matching: - return [] - target = matching[0] - files = await target.query( + target_names = ( + [dataset.lower()] + if isinstance(dataset, str) + else [d.lower() for d in dataset] + ) + target_datasets = [ + d for d in all_datasets if d.name.lower() in target_names + ] + else: + target_datasets = all_datasets + + files: list[BaseRemoteFile] = [] + for ds in target_datasets: + ds_files = await ds.query( group=group, state=state, year=year, month=month, ) - else: - files = [] - for ds in all_datasets: - ds_files = await ds.query( - group=group, - state=state, - year=year, - month=month, - ) - files.extend(ds_files) + files.extend(ds_files) if not client: return files prefix = f"public/data/{client.lower()}/" - return [f for f in files if f.record.path.startswith(prefix)] + return [f for f in files if str(f.path).startswith(prefix)] def read_parquet( self, @@ -616,7 +594,9 @@ def get_columns(path: Path) -> set[tuple[str, str]]: else: paths_str = ", ".join(f"'{p}'" for p in paths) - query = f"SELECT * FROM read_parquet([{paths_str}], union_by_name=True)" + query = ( + f"SELECT * FROM read_parquet([{paths_str}], union_by_name=True)" + ) if sql: if sql.upper().startswith("SELECT"): @@ -629,7 +609,9 @@ def get_columns(path: Path) -> set[tuple[str, str]]: if not add_dv: return base - geocode_cols = [col[0] for col in base.description if is_geocode_column(col[0])] + geocode_cols = [ + col[0] for col in base.description if is_geocode_column(col[0]) + ] if not geocode_cols: return base diff --git a/pysus/api/dadosgov/client.py b/pysus/api/dadosgov/client.py index 6b8e7113..c1d85942 100644 --- a/pysus/api/dadosgov/client.py +++ b/pysus/api/dadosgov/client.py @@ -265,7 +265,7 @@ async def get_dataset(self, id: str) -> ConjuntoDados: client=self, ) - async def _download_file( + async def download( self, file: BaseRemoteFile, output: pathlib.Path, diff --git a/pysus/api/dadosgov/models.py b/pysus/api/dadosgov/models.py index dab3d286..ddb924c3 100644 --- a/pysus/api/dadosgov/models.py +++ b/pysus/api/dadosgov/models.py @@ -30,9 +30,13 @@ def _dedup_entries( if m: stem = filename[: m.start()] fmt = m.group(1).lower() - grouped.setdefault(stem, []).append((fmt, filename, recurso, metadata)) + grouped.setdefault(stem, []).append( + (fmt, filename, recurso, metadata) + ) else: - grouped.setdefault(filename, []).append(("", filename, recurso, metadata)) + grouped.setdefault(filename, []).append( + ("", filename, recurso, metadata) + ) result: list[tuple[str, Any, dict]] = [] for _, items in grouped.items(): @@ -245,7 +249,9 @@ class Group(BaseRemoteGroup): """A group of files within a dataset.""" record: ConjuntoDados - _formatter: Callable[[str], dict[str, Any]] | None = PrivateAttr(default=None) + _formatter: Callable[[str], dict[str, Any]] | None = PrivateAttr( + default=None + ) def __init__( self, @@ -313,7 +319,9 @@ async def _fetch_files(self) -> list[BaseRemoteFile]: """Build File objects from the underlying resources.""" entries: list[tuple[str, Any, dict]] = [] for recurso in self.record.resources: - filename = recurso.file_name or recurso.url.split("/")[-1].split("?")[0] + filename = ( + recurso.file_name or recurso.url.split("/")[-1].split("?")[0] + ) if filename.lower().endswith(".pdf") or filename.startswith("get_"): continue metadata = {} diff --git a/pysus/api/ducklake/README.md b/pysus/api/ducklake/README.md index 2bd01f55..2934343c 100644 --- a/pysus/api/ducklake/README.md +++ b/pysus/api/ducklake/README.md @@ -14,7 +14,7 @@ This module provides the application-level models and data adapters required to ## Architecture Overview -The system separates the raw database management layer (Adapters) from the client wrapper layer (Client Models). +The system separates the raw database management layer (Adapters) from the client wrapper layer (Client Models). 1. **Adapters (`BaseAdapter`)**: Track local and remote `.duckdb` target states, manage connections, handle S3 transfers, and expose scoped SQLAlchemy transaction sessions. 2. **Client Components (`DuckLake`)**: Coordinate high-level actions, parse credential models, route queries, and handle collection loops. @@ -32,6 +32,4 @@ from pysus.api.ducklake.client import DuckLake async with DuckLake() as dl: datasets = await dl.datasets() - -sia = datasets[4] # e.g., SIA -files = await sia.query(state="SP", year=2026) +``` diff --git a/pysus/api/ducklake/catalog/adapters.py b/pysus/api/ducklake/catalog/adapters.py index 4e6694cd..a2a14808 100644 --- a/pysus/api/ducklake/catalog/adapters.py +++ b/pysus/api/ducklake/catalog/adapters.py @@ -1,19 +1,19 @@ +import asyncio +import os from abc import ABC from pathlib import Path -import asyncio import httpx from anyio import to_thread from pydantic import BaseModel, SecretStr -from sqlalchemy.engine import Engine -from sqlalchemy import create_engine -from sqlalchemy.pool import StaticPool -from sqlalchemy.orm import sessionmaker, Session - from pysus import CACHEPATH from pysus.api import types -from pysus.api.ducklake.functional import download_s3, upload_s3 from pysus.api.ducklake.catalog.orm.dataset import DatasetBase +from pysus.api.ducklake.functional import download_http, upload_s3 +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine, Result +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import StaticPool class DuckLakeCredentials(BaseModel): @@ -45,21 +45,59 @@ def remote_url(self) -> str: def get_session(self) -> Session: if not self._session_factory: - raise RuntimeError("Database engine not initialized. Call connect() first.") + raise RuntimeError( + "Database engine not initialized. Call connect() first." + ) return self._session_factory() + def sql(self, query: str, params: dict | None = None) -> Result: + if not self._engine: + raise RuntimeError( + "Database engine not initialized. Call connect() first." + ) + with self._engine.connect() as conn: + if params: + return conn.execute(text(query), params) + return conn.exec_driver_sql(query) + async def connect(self, force: bool = False) -> None: if self._engine and not force: if not self._session_factory: self._session_factory = sessionmaker(bind=self._engine) return + if force: + await self._download_catalog( + self.db_local, + str(self.db_remote), + force=True, + ) + self._engine = await to_thread.run_sync(self.setup_engine) + self._session_factory = sessionmaker(bind=self._engine) + return + await self._download_catalog( self.db_local, str(self.db_remote), + force=False, ) - self._engine = await to_thread.run_sync(self.setup_engine) - self._session_factory = sessionmaker(bind=self._engine) + try: + self._engine = await to_thread.run_sync(self.setup_engine) + self._session_factory = sessionmaker(bind=self._engine) + except Exception: # noqa + if self.db_local.exists(): + try: + os.remove(self.db_local) + except OSError: + pass + + await self._download_catalog( + self.db_local, + str(self.db_remote), + force=True, + ) + self._engine = await to_thread.run_sync(self.setup_engine) + self._session_factory = sessionmaker(bind=self._engine) def setup_engine( self, access_key: str | None = None, secret_key: str | None = None @@ -74,7 +112,10 @@ def setup_engine( conn.exec_driver_sql("CREATE SCHEMA IF NOT EXISTS pysus;") has_pysus = conn.exec_driver_sql( - "SELECT 1 FROM information_schema.schemata WHERE schema_name = 'pysus'" + statement=( + "SELECT 1 FROM information_schema.schemata " + "WHERE schema_name = 'pysus'" + ) ).fetchone() if has_pysus: @@ -101,10 +142,12 @@ def setup_engine( DatasetBase.metadata.create_all(bind=engine) return engine - async def _download_catalog(self, local_path: Path, remote_path: str) -> None: + async def _download_catalog( + self, local_path: Path, remote_path: str, force: bool = False + ) -> None: url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote_path}" - if local_path.exists(): + if local_path.exists() and not force: try: local_size = local_path.stat().st_size except OSError: @@ -112,35 +155,28 @@ async def _download_catalog(self, local_path: Path, remote_path: str) -> None: else: local_size = -1 - async with httpx.AsyncClient(follow_redirects=True) as client: - try: - head = await client.head(url) + remote_size = 0 + if local_size != -1: + async with httpx.AsyncClient(follow_redirects=True) as client: + try: + head = await client.head(url) - if head.status_code == 404: - return + if head.status_code == 404: + return - head.raise_for_status() - remote_size = int(head.headers.get("content-length", 0)) - except httpx.HTTPStatusError: - return - except Exception: - remote_size = 0 + head.raise_for_status() + remote_size = int(head.headers.get("content-length", 0)) + except httpx.HTTPStatusError: + return + except Exception: # noqa + remote_size = 0 - if remote_size == local_size: + if not force and remote_size == local_size and local_size != -1: return - access_key = ( - self.credentials.access_key.get_secret_value() if self.credentials else None - ) - secret_key = ( - self.credentials.secret_key.get_secret_value() if self.credentials else None - ) - - await download_s3( + await download_http( remote_path=remote_path, local_path=local_path, - access_key=access_key, - secret_key=secret_key, ) async def _upload_catalog(self) -> None: @@ -195,20 +231,20 @@ class CatalogAdapter(BaseAdapter): def __init__(self, engine=None, **data) -> None: super().__init__(engine=engine, **data) self.db_local: Path = self.cache_dir / "catalog.duckdb" - self.db_remote: str = "public/catalog.duckdb" + self.db_remote: Path = Path("public/catalog.duckdb") class DatasetAdapter(BaseAdapter): - def __init__(self, name: str, engine=None, **data) -> None: + def __init__(self, name: str, dataset_id: int, engine=None, **data) -> None: super().__init__(engine=engine, **data) self.dataset_name: str = name self.db_local: Path = self.cache_dir / f"catalog_{name}.duckdb" - self.db_remote: str = f"datasets/catalog_{name}.duckdb" + self.db_remote: Path = Path(f"datasets/catalog_{name}.duckdb") + self.dataset_id = dataset_id class ColumnsAdapter(BaseAdapter): def __init__(self, engine=None, **data) -> None: super().__init__(engine=engine, **data) - self.db_local: Path = self.cache_dir / "columns.duckdb" - self.db_remote: str = "public/columns.duckdb" - + self.db_local: Path = self.cache_dir / "catalog_columns.duckdb" + self.db_remote: Path = Path("public/catalog_columns.duckdb") diff --git a/pysus/api/ducklake/catalog/orm/columns.py b/pysus/api/ducklake/catalog/orm/columns.py index 3df18200..4af181f3 100644 --- a/pysus/api/ducklake/catalog/orm/columns.py +++ b/pysus/api/ducklake/catalog/orm/columns.py @@ -1,5 +1,5 @@ +from sqlalchemy import Boolean, Column, Index, Integer, Sequence, String from sqlalchemy.orm import DeclarativeBase -from sqlalchemy import Column, Integer, Sequence, String, Boolean, Index class ColumnsBase(DeclarativeBase): @@ -9,7 +9,9 @@ class ColumnsBase(DeclarativeBase): class ColumnDefinition(ColumnsBase): __tablename__ = "dataset_columns" - id = Column(Integer, Sequence("columns_id_seq", schema="pysus"), primary_key=True) + id = Column( + Integer, Sequence("columns_id_seq", schema="pysus"), primary_key=True + ) dataset_id = Column(Integer, nullable=False, index=True) name = Column(String, nullable=False) type = Column(String, nullable=False) diff --git a/pysus/api/ducklake/catalog/orm/dataset.py b/pysus/api/ducklake/catalog/orm/dataset.py index fb096f50..cbe03dca 100644 --- a/pysus/api/ducklake/catalog/orm/dataset.py +++ b/pysus/api/ducklake/catalog/orm/dataset.py @@ -1,17 +1,18 @@ from datetime import datetime from typing import Optional -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + from sqlalchemy import ( + BigInteger, Column, + DateTime, + ForeignKey, + Index, Integer, Sequence, String, - ForeignKey, - BigInteger, - Index, - DateTime, Table, ) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship class DatasetBase(DeclarativeBase): @@ -34,13 +35,17 @@ class Group(DatasetBase): {"schema": "pysus"}, ) - id = Column(Integer, Sequence("groups_id_seq", schema="pysus"), primary_key=True) - name = Column(String, nullable=False) - dataset_id = Column(Integer, nullable=False, index=True) - long_name = Column(String, nullable=False) - description = Column(String, nullable=True) + id: Mapped[int] = mapped_column( + Integer, Sequence("groups_id_seq", schema="pysus"), primary_key=True + ) + name: Mapped[str] = mapped_column(String, nullable=False) + dataset_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + long_name: Mapped[str] = mapped_column(String, nullable=False) + description: Mapped[str | None] = mapped_column(String, nullable=True) - files = relationship("File", back_populates="group", cascade="all, delete-orphan") + files: Mapped[list["File"]] = relationship( + "File", back_populates="group", cascade="all, delete-orphan" + ) class File(DatasetBase): @@ -48,7 +53,14 @@ class File(DatasetBase): __table_args__ = ( Index("ix_files_dataset_group", "dataset_id", "group_id"), Index("ix_files_temporal", "year", "month"), - Index("ix_files_lookup", "dataset_id", "group_id", "year", "month", "state"), + Index( + "ix_files_lookup", + "dataset_id", + "group_id", + "year", + "month", + "state", + ), {"schema": "pysus"}, ) @@ -57,20 +69,32 @@ class File(DatasetBase): ) dataset_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) group_id: Mapped[int | None] = mapped_column( - Integer, ForeignKey("pysus.dataset_groups.id"), nullable=True, index=True + Integer, + ForeignKey("pysus.dataset_groups.id"), + nullable=True, + index=True, ) path: Mapped[str] = mapped_column(String, nullable=False, unique=True) size: Mapped[int] = mapped_column(BigInteger, nullable=False) rows: Mapped[int] = mapped_column(Integer, nullable=False) type: Mapped[str] = mapped_column(String, nullable=True) modified: Mapped[datetime] = mapped_column(DateTime, nullable=False) - origin_modified: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + origin_modified: Mapped[datetime | None] = mapped_column( + DateTime, nullable=True + ) origin_size: Mapped[int] = mapped_column(BigInteger, nullable=False) origin_path: Mapped[str] = mapped_column(String, nullable=False) - sha256: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) + sha256: Mapped[str | None] = mapped_column( + String(64), nullable=True, index=True + ) year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) - month: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) - state: Mapped[str | None] = mapped_column(String(2), nullable=True, index=True) - - group: Mapped[Optional["Group"]] = relationship("Group", back_populates="files") + month: Mapped[int | None] = mapped_column( + Integer, nullable=True, index=True + ) + state: Mapped[str | None] = mapped_column( + String(2), nullable=True, index=True + ) + group: Mapped[Optional["Group"]] = relationship( + "Group", back_populates="files" + ) diff --git a/pysus/api/ducklake/catalog/orm/default.py b/pysus/api/ducklake/catalog/orm/default.py index 4de0a24f..4f0a26fa 100644 --- a/pysus/api/ducklake/catalog/orm/default.py +++ b/pysus/api/ducklake/catalog/orm/default.py @@ -4,8 +4,8 @@ per-dataset ``catalog_.db`` files defined in ``.dataset``. """ -from sqlalchemy import Column, Integer, Sequence, String -from sqlalchemy.orm import DeclarativeBase +from sqlalchemy import Integer, Sequence, String +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column class Base(DeclarativeBase): @@ -33,14 +33,16 @@ class Dataset(Base): __tablename__ = "datasets" __table_args__: tuple = ({"schema": "pysus"},) - id = Column( + id: Mapped[int] = mapped_column( Integer, Sequence("datasets_id_seq", schema="pysus"), primary_key=True, ) - name = Column(String, nullable=False, unique=True, index=True) - long_name = Column(String, nullable=False) - description = Column(String, nullable=True) + name: Mapped[str] = mapped_column( + String, nullable=False, unique=True, index=True + ) + long_name: Mapped[str] = mapped_column(String, nullable=False) + description: Mapped[str | None] = mapped_column(String, nullable=True) def __repr__(self): return self.name diff --git a/pysus/api/ducklake/client.py b/pysus/api/ducklake/client.py index f14fe96c..8e5dc2da 100644 --- a/pysus/api/ducklake/client.py +++ b/pysus/api/ducklake/client.py @@ -9,15 +9,19 @@ from pathlib import Path from anyio import to_thread -from pydantic import SecretStr, PrivateAttr, Field -from pysus.api.models import BaseRemoteClient +from pydantic import Field, PrivateAttr, SecretStr +from pysus.api.models import BaseRemoteClient, BaseRemoteFile from pysus.api.types import DUCKLAKE +from .catalog.adapters import ( + CatalogAdapter, + ColumnsAdapter, + DatasetAdapter, + DuckLakeCredentials, +) from .catalog.orm.default import Dataset -from .catalog.adapters import DatasetAdapter, CatalogAdapter +from .functional import download_http from .models import DuckDataset, File -from .catalog.adapters import DuckLakeCredentials -from .functional import download_s3 class DuckLake(BaseRemoteClient): @@ -25,8 +29,15 @@ class DuckLake(BaseRemoteClient): update_on_close: bool = Field(default=False, exclude=True) _datasets: list[DuckDataset] = PrivateAttr(default_factory=list) _catalog_adap: CatalogAdapter = PrivateAttr() + _columns_adap: ColumnsAdapter = PrivateAttr() - def __init__(self, engine=None, update_on_close: bool = False, **data) -> None: + def __init__( + self, + engine=None, + columns_engine=None, + update_on_close: bool = False, + **data, + ) -> None: super().__init__(**data) self.update_on_close = update_on_close self._catalog_adap = CatalogAdapter( @@ -34,6 +45,11 @@ def __init__(self, engine=None, update_on_close: bool = False, **data) -> None: credentials=self.credentials, update_on_close=self.update_on_close, ) + self._columns_adap = ColumnsAdapter( + engine=columns_engine, + credentials=self.credentials, + update_on_close=self.update_on_close, + ) @property def name(self) -> str: @@ -47,6 +63,14 @@ def long_name(self) -> str: def description(self) -> str: return "" + @property + def catalog_path(self) -> Path: + return self._catalog_adap.db_local + + @property + def columns_path(self) -> Path: + return self._columns_adap.db_local + async def datasets(self, **kwargs) -> list[DuckDataset]: def _fetch(): with self._catalog_adap.get_session() as session: @@ -62,6 +86,7 @@ def _fetch(): for rec in records: dataset_adapter = DatasetAdapter( name=str(rec.name), + dataset_id=int(rec.id), credentials=self.credentials, update_on_close=self.update_on_close, ) @@ -77,21 +102,27 @@ def _fetch(): self._datasets = duck_datasets return duck_datasets - async def login( - self, - access_key: str, - secret_key: str, - **kwargs, - ) -> None: + async def login(self, **kwargs) -> None: + access_key = kwargs.get("access_key") + secret_key = kwargs.get("secret_key") + + if not access_key or not secret_key: + raise ValueError( + "DuckLake authentication requires 'access_key' and 'secret_key'" + ) + self.credentials = DuckLakeCredentials( access_key=SecretStr(access_key), secret_key=SecretStr(secret_key), ) self._catalog_adap.credentials = self.credentials + self._columns_adap.credentials = self.credentials await self._catalog_adap.connect(force=True) + await self._columns_adap.connect(force=True) async def connect(self, force: bool = False) -> None: await self._catalog_adap.connect(force=force) + await self._columns_adap.connect(force=force) async def close(self, update_catalog: bool | None = None) -> None: should_update = ( @@ -102,28 +133,20 @@ async def close(self, update_catalog: bool | None = None) -> None: await ds.close(update_catalog=should_update) await self._catalog_adap.close(update=should_update) + await self._columns_adap.close(update=should_update) async def download( self, - file: File, + file: BaseRemoteFile, output: Path, callback: Callable[[int, int], None] | None = None, ) -> Path: if not isinstance(file, File): raise ValueError("DuckLake File was not properly instantiated") - access_key = ( - self.credentials.access_key.get_secret_value() if self.credentials else None - ) - secret_key = ( - self.credentials.secret_key.get_secret_value() if self.credentials else None - ) - - await download_s3( + await download_http( remote_path=file.record.path, local_path=output, - access_key=access_key, - secret_key=secret_key, callback=callback, ) return output @@ -136,7 +159,9 @@ async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: await self.close(update_catalog=None) def __del__(self) -> None: - if not hasattr(self, "_catalog_adap"): + if not hasattr(self, "_catalog_adap") or not hasattr( + self, "_columns_adap" + ): return try: loop = asyncio.get_running_loop() @@ -145,9 +170,9 @@ def __del__(self) -> None: except RuntimeError: try: asyncio.run(self.close(update_catalog=False)) - except Exception: + except Exception: # noqa pass - except Exception: + except Exception: # noqa pass diff --git a/pysus/api/ducklake/functional.py b/pysus/api/ducklake/functional.py index d13d8001..fcb3289b 100644 --- a/pysus/api/ducklake/functional.py +++ b/pysus/api/ducklake/functional.py @@ -1,12 +1,11 @@ +from collections.abc import Callable from pathlib import Path -from typing import Callable -from anyio import sleep, to_thread -import httpx import boto3 -from botocore.config import Config +import httpx +from anyio import sleep, to_thread from botocore import UNSIGNED - +from botocore.config import Config from pysus.api import types @@ -15,35 +14,29 @@ async def download_http( local_path: Path, callback: Callable[[int, int], None] | None = None, ) -> None: - """Download *remote_path* to *local_path* with HTTP streaming and retries. - - Parameters - ---------- - remote_path : str - Object key within the bucket. - local_path : Path - Local destination path. - callback : Callable[[int, int], None], optional - Progress callback receiving ``(downloaded, total)`` bytes. - """ url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote_path}" max_retries = 5 for attempt in range(max_retries): try: - async with httpx.AsyncClient(follow_redirects=True) as client: + async with httpx.AsyncClient( + follow_redirects=True, verify=False + ) as client: async with client.stream("GET", url) as r: r.raise_for_status() total = int(r.headers.get("Content-Length", 0)) downloaded = 0 + with open(local_path, "wb") as f: - async for chunk in r.aiter_bytes(chunk_size=1024 * 1024): + async for chunk in r.aiter_bytes( + chunk_size=1024 * 1024 + ): await to_thread.run_sync(f.write, chunk) downloaded += len(chunk) if callback: callback(downloaded, total) return - except OSError as e: + except (OSError, httpx.HTTPStatusError) as e: if attempt < max_retries - 1: await sleep(1) else: @@ -57,7 +50,9 @@ async def download_s3( secret_key: str | None = None, callback: Callable[[int, int], None] | None = None, ) -> None: - """Download *remote_path* to *local_path* using boto3 with optional credentials. + """ + Download *remote_path* to *local_path* using + boto3 with optional credentials. Parameters ---------- @@ -93,7 +88,7 @@ def _get_total_size(client_args) -> int: client = boto3.client(**client_args) meta = client.head_object(Bucket=types.S3_BUCKET, Key=remote_path) return int(meta.get("ContentLength", 0)) - except Exception: + except Exception: # noqa return 0 def _download(client_args, total_size: int): @@ -119,7 +114,7 @@ def boto_callback(bytes_amount): total_size = await to_thread.run_sync(_get_total_size, client_args) await to_thread.run_sync(_download, client_args, total_size) return - except Exception as e: + except Exception as e: # noqa if attempt < max_retries - 1: await sleep(1) else: @@ -172,7 +167,7 @@ def boto_callback(bytes_amount): total_size = local_path.stat().st_size await to_thread.run_sync(_upload, client_args, total_size) return - except Exception as e: + except Exception as e: # noqa if attempt < max_retries - 1: await sleep(1) else: diff --git a/pysus/api/ducklake/models.py b/pysus/api/ducklake/models.py index 7de750a7..ad78415d 100644 --- a/pysus/api/ducklake/models.py +++ b/pysus/api/ducklake/models.py @@ -11,114 +11,63 @@ from typing import TYPE_CHECKING, Any, Optional, Union from anyio import to_thread -from pydantic import Field +from pydantic import Field, PrivateAttr from pysus import CACHEPATH +from pysus.api.models import BaseRemoteDataset, BaseRemoteFile, BaseRemoteGroup +from sqlalchemy import or_, orm, select + from .catalog.adapters import DatasetAdapter +from .catalog.orm.dataset import File as CatalogFile +from .catalog.orm.dataset import Group from .catalog.orm.default import Dataset -from .catalog.orm.dataset import ( - File as CatalogFile, - Group, -) -from pysus.api.models import BaseRemoteDataset, BaseRemoteFile, BaseRemoteGroup -from sqlalchemy import select, orm -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: from .client import DuckLake class File(BaseRemoteFile): - """A remote file in the DuckLake catalog with download and verification. - - Parameters - ---------- - record : CatalogFile - The underlying ORM record. - type : str, optional - File type identifier (default ``"remote"``). - dataset : Any - The parent dataset object. - group : Any, optional - The parent group object, if any. - """ - - record: CatalogFile = Field(exclude=True) group: Optional["DuckGroup"] = Field(default=None, exclude=True) + _record: CatalogFile = PrivateAttr() + def __init__(self, **data: Any) -> None: record = data.pop("record") group = data.pop("group", None) + super().__init__( path=Path(record.path), type=record.type or "remote", - record=record, # type: ignore[call-arg] group=group, **data, ) + self._record = record @property - def basename(self) -> str: - """Return the file name without directory components. + def record(self) -> CatalogFile: + return self._record - Returns - ------- - str - The base file name. - """ + @property + def basename(self) -> str: return self.path.name @property def extension(self) -> str: - """Return the file extension including the leading dot. - - Returns - ------- - str - File extension (e.g. ``'.csv'``). - """ return self.path.suffix @property def size(self) -> int: - """Return the file size in bytes. - - Returns - ------- - int - File size in bytes. - """ return self.record.size @property def modify(self) -> datetime: - """Return the last-modified timestamp. - - Returns - ------- - datetime - The last modification timestamp. - """ return self.record.modified @property def rows(self) -> int: - """Return the number of rows in the file. - - Returns - ------- - int - Row count. - """ return self.record.rows @property def sha256(self) -> str | None: - """Return the SHA-256 hash of the file, if available. - - Returns - ------- - str or None - SHA-256 hex digest, or None if not recorded. - """ return self.record.sha256 async def _download( @@ -126,29 +75,11 @@ async def _download( output: Path | None = None, callback: Callable[[int, int], None] | None = None, ) -> Path: - """Download the file from object storage to the given output path.""" if not output: output = CACHEPATH / self.name - - return await self.client.download( - self, - output, - callback=callback, - ) + return await self.client.download(self, output, callback=callback) async def verify(self, path: Path) -> bool: - """Verify the file matches the recorded SHA-256 hash. - - Parameters - ---------- - path : Path - Path to the downloaded file on disk. - - Returns - ------- - bool - True if the hash matches or no hash is recorded, False otherwise. - """ if not self.sha256: return True @@ -166,14 +97,24 @@ def _calculate(): class DuckDataset(BaseRemoteDataset): record: "Dataset" = Field(exclude=True) client: "DuckLake" = Field(exclude=True) - adapter: "DatasetAdapter" = Field(exclude=True) + border: "DatasetAdapter" = Field(exclude=True) update_on_close: bool = Field(default=False, exclude=True) def __init__(self, **data) -> None: + if "adapter" in data and "border" not in data: + data["border"] = data.pop("adapter") super().__init__(**data) - def __repr__(self) -> str: - return self.name.upper() + def __str__(self) -> str: + return self.record.name + + @property + def adapter(self) -> "DatasetAdapter": + return self.border + + @property + def id(self) -> int: + return int(self.adapter.dataset_id) @property def name(self) -> str: @@ -187,13 +128,9 @@ def long_name(self) -> str: def description(self) -> str: return str(self.record.description) - async def connect( - self, - force: bool = False, - ) -> None: + async def connect(self, force: bool = False) -> None: if self not in self.client._datasets: self.client._datasets.append(self) - await self.adapter.connect(force=force) async def close(self, update_catalog: bool | None = None): @@ -204,32 +141,44 @@ async def close(self, update_catalog: bool | None = None): async def query( self, - group: str | None = None, - state: str | None = None, - year: int | None = None, - month: int | None = None, + group: str | list[str] | None = None, + state: str | list[str] | None = None, + year: int | list[int] | None = None, + month: int | list[int] | None = None, ) -> list[File]: + def _to_list(val: Any) -> list[Any] | None: + if val is None: + return None + return val if isinstance(val, list) else [val] + + groups = _to_list(group) + states = _to_list(state) + years = _to_list(year) + months = _to_list(month) + def _query() -> list[CatalogFile]: with self.adapter.get_session() as session: stmt = select(CatalogFile).filter( - CatalogFile.dataset_id == self.record.id, + CatalogFile.dataset_id == self.id, ) - if group: + if groups: stmt = ( stmt.join(CatalogFile.group) .options(orm.contains_eager(CatalogFile.group)) - .filter(Group.name.ilike(group)) + .filter(or_(*[Group.name.ilike(g) for g in groups])) ) else: stmt = stmt.options(orm.joinedload(CatalogFile.group)) - if state: - stmt = stmt.filter(CatalogFile.state == state.upper()) - if year: - stmt = stmt.filter(CatalogFile.year == year) - if month: - stmt = stmt.filter(CatalogFile.month == month) + if states: + stmt = stmt.filter( + CatalogFile.state.in_([s.upper() for s in states]) + ) + if years: + stmt = stmt.filter(CatalogFile.year.in_(years)) + if months: + stmt = stmt.filter(CatalogFile.month.in_(months)) results = session.scalars(stmt).all() session.expunge_all() @@ -245,13 +194,13 @@ def _fetch(): stmt = ( select(Group) .options(orm.joinedload(Group.files)) - .filter(Group.dataset_id == self.record.id) + .filter(Group.dataset_id == self.id) ) groups = session.scalars(stmt).all() ungrouped = session.scalars( select(CatalogFile).filter( - CatalogFile.dataset_id == self.record.id, + CatalogFile.dataset_id == self.id, CatalogFile.group_id.is_(None), ) ).all() @@ -261,15 +210,14 @@ def _fetch(): async with self.adapter: groups, files = await to_thread.run_sync(_fetch) - - items: list[Union[DuckGroup, File]] = [] + items: list[DuckGroup | File] = [] if groups: - items.extend([DuckGroup(record=g, dataset=self) for g in groups]) - + items.extend( + [DuckGroup(record=g, dataset=self) for g in groups] + ) if files: items.extend([File(record=f, dataset=self) for f in files]) - return items async def __aenter__(self): @@ -281,60 +229,27 @@ async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: class DuckGroup(BaseRemoteGroup): - """A group of related files within a DuckLake dataset. - - Parameters - ---------- - record : Group - The underlying ORM record. - dataset : DuckDataset - The parent dataset instance. - """ - record: Group = Field(exclude=True) dataset: DuckDataset = Field(exclude=True) + def __str__(self) -> str: + return self.name + @property def name(self) -> str: - """Return the short name of the group. - - Returns - ------- - str - The group short name. - """ return str(self.record.name) @property def long_name(self) -> str: - """Return the human-readable name of the group. - - Returns - ------- - str - The group display name, falling back to the short name. - """ return str(self.record.long_name) @property def description(self) -> str: - """Return the description of the group. - - Returns - ------- - str - The group description, or an empty string if unavailable. - """ return str(self.record.description) async def _fetch_files(self) -> list[BaseRemoteFile]: - """Fetch the list of files belonging to this group.""" files: list[BaseRemoteFile] = [ - File( - record=f, - group=self, - dataset=self.dataset, - ) + File(record=f, group=self, dataset=self.dataset) for f in self.record.files ] return files diff --git a/pysus/api/ftp/client.py b/pysus/api/ftp/client.py index 5c265d0a..3a26113f 100644 --- a/pysus/api/ftp/client.py +++ b/pysus/api/ftp/client.py @@ -109,7 +109,7 @@ async def connect(self) -> None: def _connect(): if self.ftp is None: self._ftp = FTPLib(self.host, timeout=self.timeout) - self.ftp.login() + self._ftp.login() await to_thread.run_sync(_connect) @@ -171,7 +171,7 @@ async def datasets(self, **kwargs) -> list[Dataset]: return [d(client=self) for d in AVAILABLE_DATABASES] - async def _download_file( + async def download( self, file: BaseRemoteFile, output: pathlib.Path, diff --git a/pysus/management/client.py b/pysus/management/client.py index 2647c7c9..cc41c578 100644 --- a/pysus/management/client.py +++ b/pysus/management/client.py @@ -4,9 +4,9 @@ from logging import error from pathlib import Path -from anyio import to_thread from pysus.api.client import PySUS from pysus.api.dadosgov.models import File as APIFile +from pysus.api.ducklake.functional import upload_s3 from pysus.api.extensions import Parquet from pysus.api.ftp.models import File as FTPFile from pysus.api.models import BaseRemoteFile @@ -24,13 +24,15 @@ def __init__( self.secret_key = secret_key or os.getenv("SECRET_KEY") self.dadosgov_token = dadosgov_token or os.getenv("DADOSGOV_TOKEN") - if not access_key or not secret_key: + if not self.access_key or not self.secret_key: raise ValueError("s3 credentials are needed") async def __aenter__(self): await self.pysus.__aenter__() ducklake = await self.pysus.get_ducklake() - await ducklake.login(self.access_key, self.secret_key) + await ducklake.login( + access_key=self.access_key, secret_key=self.secret_key + ) return self async def __aexit__(self, exc_type, exc_val, exc_tb): @@ -38,7 +40,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): if not exc_type: ducklake = self.pysus._ducklake if ducklake: - await ducklake._upload_catalog() + await ducklake.close(update_catalog=True) finally: await self.pysus.__aexit__(exc_type, exc_val, exc_tb) @@ -50,85 +52,99 @@ async def upload( if not self.pysus._ducklake: raise ConnectionError("DuckLake is not connected") + remote_file_path = Path(file.path) s3_key = ( f"public/data/{file.client.name.lower()}" f"/{file.dataset.name.lower()}" - f"/{file.path.with_suffix('.parquet').name}" + f"/{remote_file_path.with_suffix('.parquet').name}" ) dataset_id = None group_id = None - engine = self.pysus._ducklake._engine - with engine.raw_connection() as conn: - cursor = conn.cursor() + catalog_engine = self.pysus._ducklake._catalog_adap._engine + columns_engine = self.pysus._ducklake._columns_adap._engine + + if not catalog_engine or not columns_engine: + raise ConnectionError( + "DuckLake database engines are not initialized" + ) + + catalog_conn = catalog_engine.raw_connection() + columns_conn = columns_engine.raw_connection() + + with catalog_conn, columns_conn: + catalog_cursor = catalog_conn.cursor() + columns_cursor = columns_conn.cursor() + try: dataset_name = file.dataset.name.lower() is_ftp = file.client.name.lower() == "ftp" - cursor.execute( - f"SELECT id FROM pysus.datasets WHERE name = '{dataset_name}'" # noqa + catalog_cursor.execute( + "SELECT id FROM pysus.datasets WHERE name = ?", + (dataset_name,), ) - row = cursor.fetchone() + row = catalog_cursor.fetchone() if row: dataset_id = row[0] origin_val = "'FTP'" if is_ftp else "'API'" - cursor.execute( + catalog_cursor.execute( f"UPDATE pysus.datasets SET origin = {origin_val} " f"WHERE id = {dataset_id}" ) else: - cursor.execute("SELECT MAX(id) FROM pysus.datasets") - max_id = cursor.fetchone()[0] + catalog_cursor.execute("SELECT MAX(id) FROM pysus.datasets") + max_id_row = catalog_cursor.fetchone() + max_id = max_id_row[0] if max_id_row else None dataset_id = (max_id or 0) + 1 origin_val = "'FTP'" if is_ftp else "'API'" - cursor.execute( - f"INSERT INTO pysus.datasets (id, name, " - f"long_name, origin) " - f"VALUES ({dataset_id}, '{dataset_name}', " + catalog_cursor.execute( + f"INSERT INTO pysus.datasets (id, name, long_name, " + f"origin) VALUES ({dataset_id}, '{dataset_name}', " f"'{file.dataset.long_name}', {origin_val})" ) if file.group: group_name = file.group.name - cursor.execute( + catalog_cursor.execute( "SELECT id FROM pysus.dataset_groups " - f"WHERE name = '{group_name}' AND " - f"dataset_id = {dataset_id}" + "WHERE name = ? AND dataset_id = ?", + (group_name, dataset_id), ) - row = cursor.fetchone() + row = catalog_cursor.fetchone() if row: group_id = row[0] else: - cursor.execute( - "SELECT MAX(id) FROM pysus.dataset_groups", + catalog_cursor.execute( + "SELECT MAX(id) FROM pysus.dataset_groups" ) - max_id = cursor.fetchone()[0] + max_id_row = catalog_cursor.fetchone() + max_id = max_id_row[0] if max_id_row else None group_id = (max_id or 0) + 1 long_name = file.dataset.group_definitions.get( group_name.upper(), group_name ) - cursor.execute( - f"INSERT INTO pysus.dataset_groups " - f"(id, dataset_id, name, long_name) " - f"VALUES ({group_id}, {dataset_id}, " - f"'{group_name}', '{long_name}')" + catalog_cursor.execute( + f"INSERT INTO pysus.dataset_groups (id, " + f"dataset_id, name, long_name) VALUES ({group_id}," + f" {dataset_id}, '{group_name}', '{long_name}')" ) group_val = "NULL" if group_id is None else str(group_id) - cursor.execute( - f"SELECT id, group_id FROM pysus.files WHERE path = '{s3_key}'" # noqa + catalog_cursor.execute( + "SELECT id, group_id FROM pysus.files WHERE path = ?", + (s3_key,), ) - row = cursor.fetchone() + row = catalog_cursor.fetchone() if row: file_id, db_group_id = row - group_mismatch = db_group_id != group_id should_upload = self._should_upload_raw( - cursor, + catalog_cursor, file_id, file, ) @@ -136,77 +152,81 @@ async def upload( if not should_upload and not group_mismatch: return - cursor.execute( - f"DELETE FROM pysus.file_columns WHERE file_id = {file_id}" # noqa + columns_cursor.execute( + "DELETE FROM pysus.file_columns WHERE file_id = ?", + (file_id,), ) - cursor.execute( - f"DELETE FROM pysus.files WHERE id = {file_id}", + catalog_cursor.execute( + "DELETE FROM pysus.files WHERE id = ?", + (file_id,), ) else: - cursor.execute("SELECT MAX(id) FROM pysus.files") - max_id = cursor.fetchone()[0] + catalog_cursor.execute("SELECT MAX(id) FROM pysus.files") + max_id_row = catalog_cursor.fetchone() + max_id = max_id_row[0] if max_id_row else None file_id = (max_id or 0) + 1 parquet_ext = await self._download_with_retry(file, callback) await self._upload_to_s3(parquet_ext.path, s3_key) - year_val = "NULL" if file.year is None else file.year - month_val = "NULL" if file.month is None else file.month + year_val = "NULL" if file.year is None else str(file.year) + month_val = "NULL" if file.month is None else str(file.month) state_val = "NULL" if file.state is None else f"'{file.state}'" - cursor.execute( - f"INSERT INTO pysus.files (id, dataset_id, " - f"group_id, path, size, rows, " - f"modified, origin_modified, origin_path, year, " - f"month, state) " - f"VALUES ({file_id}, {dataset_id}, {group_val}, " - f"'{s3_key}', {parquet_ext.size}, " - f"{parquet_ext.rows}, CURRENT_TIMESTAMP, " - f"'{file.modify}', '{file.path}', " + catalog_cursor.execute( + f"INSERT INTO pysus.files (id, dataset_id, group_id, " + f"path, size, rows, modified, origin_modified, " + f"origin_path, year, month, state) VALUES ({file_id}, " + f"{dataset_id}, {group_val}, '{s3_key}', " + f"{parquet_ext.size}, {parquet_ext.rows}, " + f"CURRENT_TIMESTAMP, '{file.modify}', '{file.path}', " f"{year_val}, {month_val}, {state_val})" ) new_columns = self._get_or_create_columns_raw( - cursor, parquet_ext, dataset_id + columns_cursor, parquet_ext, dataset_id ) for col in new_columns: - cursor.execute( - f"INSERT INTO pysus.file_columns " - f"(file_id, column_id) VALUES ({file_id}, {col})" + columns_cursor.execute( + f"INSERT INTO pysus.file_columns (file_id, column_id) " + f"VALUES ({file_id}, {col})" ) - conn.commit() - cursor.execute("CHECKPOINT") + catalog_conn.commit() + columns_conn.commit() + + catalog_cursor.execute("CHECKPOINT") + columns_cursor.execute("CHECKPOINT") if parquet_ext.path.exists(): parquet_ext.path.unlink() await self.pysus._delete_record(str(parquet_ext.path)) - except Exception: # noqa + except BaseException as rollback_err: # noqa + try: + catalog_conn.rollback() + except Exception as inner_err: # noqa + error(f"Catalog rollback failed: {inner_err}") try: - conn.rollback() - except Exception: # noqa - pass - raise + columns_conn.rollback() + except Exception as inner_err: # noqa + error(f"Columns rollback failed: {inner_err}") + raise rollback_err async def _upload_to_s3( self, local_path: Path, s3_path: str, - callback: Callable[[int], None] | None = None, + callback: Callable[[int, int], None] | None = None, ): - def _do_upload(): - if not self.pysus._ducklake: - raise ConnectionError("DuckLake not connected") - self.pysus._ducklake._s3_client.upload_file( - str(local_path), - self.pysus._ducklake.bucket, - s3_path, - Callback=callback, - ) - - await to_thread.run_sync(_do_upload) + await upload_s3( + local_path=local_path, + access_key=str(self.access_key), + secret_key=str(self.secret_key), + remote_path=s3_path, + callback=callback, + ) async def _download_with_retry( self, @@ -249,7 +269,8 @@ def _should_upload_raw( return True cursor.execute( - f"SELECT origin_modified FROM pysus.files WHERE id = {file_id}", + "SELECT origin_modified FROM pysus.files WHERE id = ?", + (file_id,), ) row = cursor.fetchone() if not row: @@ -263,7 +284,7 @@ def _should_upload_raw( if file_mod is None: return True - return file_mod > origin_modified + return str(file_mod) > str(origin_modified) def _get_or_create_columns_raw( self, cursor, file: Parquet, dataset_id: int @@ -288,7 +309,8 @@ def _get_or_create_columns_raw( cursor.execute( "SELECT id FROM pysus.dataset_columns " - f"WHERE name = '{col_name}' AND dataset_id = {dataset_id}" + "WHERE name = ? AND dataset_id = ?", + (col_name, dataset_id), ) existing = cursor.fetchone() @@ -296,13 +318,13 @@ def _get_or_create_columns_raw( result.append(existing[0]) else: cursor.execute("SELECT MAX(id) FROM pysus.dataset_columns") - max_id = cursor.fetchone()[0] + max_id_row = cursor.fetchone() + max_id = max_id_row[0] if max_id_row else None new_id = (max_id or 0) + 1 cursor.execute( - "INSERT INTO pysus.dataset_columns " - "(id, dataset_id, name, type, nullable) " - f"VALUES ({new_id}, {dataset_id}, '{col_name}', " - f"'{sql_type}', true)" + "INSERT INTO pysus.dataset_columns (id, dataset_id, name, " + "type, nullable) VALUES (?, ?, ?, ?, true)", + (new_id, dataset_id, col_name, sql_type), ) result.append(new_id) diff --git a/pysus/tests/api/ducklake/test_catalog.py b/pysus/tests/api/ducklake/test_catalog.py index a6d415f6..2afecabd 100644 --- a/pysus/tests/api/ducklake/test_catalog.py +++ b/pysus/tests/api/ducklake/test_catalog.py @@ -1,12 +1,8 @@ """Tests for DuckLake catalog ORM models.""" -from pysus.api.ducklake.catalog.orm.dataset import ( - ColumnDefinition, - Dataset, - File, - Group, - file_columns, -) +from pysus.api.ducklake.catalog.orm.columns import ColumnDefinition +from pysus.api.ducklake.catalog.orm.dataset import File, Group, file_columns +from pysus.api.ducklake.catalog.orm.default import Dataset from pysus.api.ducklake.catalog.orm.default import Dataset as DefaultDataset diff --git a/pysus/tests/api/ducklake/test_client.py b/pysus/tests/api/ducklake/test_client.py index cf045025..08885332 100644 --- a/pysus/tests/api/ducklake/test_client.py +++ b/pysus/tests/api/ducklake/test_client.py @@ -6,8 +6,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -from pysus.api.ducklake.catalog.orm.dataset import Dataset as PerDataset from pysus.api.ducklake.catalog.orm.dataset import File as CatalogFile +from pysus.api.ducklake.catalog.orm.default import Dataset as PerDataset from pysus.api.ducklake.client import DuckLake, DuckLakeCredentials from pysus.api.ducklake.models import DuckDataset, File @@ -40,12 +40,16 @@ async def test_description(self): async def test_ducklake_catalog_path(self, tmp_path): with patch("pysus.api.ducklake.client.CACHEPATH", tmp_path): client = DuckLake() - assert client.catalog_path == tmp_path / "ducklake" / "catalog.duckdb" + assert ( + client.catalog_path == tmp_path / "ducklake" / "catalog.duckdb" + ) @pytest.mark.asyncio async def test_ducklake_catalog_url(self): client = DuckLake() - expected = "https://nbg1.your-objectstorage.com/pysus/public/catalog.duckdb" + expected = ( + "https://nbg1.your-objectstorage.com/pysus/public/catalog.duckdb" + ) assert client._catalog_url == expected @pytest.mark.asyncio @@ -132,7 +136,9 @@ async def test_upload_catalog_requires_auth(self): class TestDuckLakeDatasets: @pytest.mark.asyncio - async def test_datasets_creates_session_and_returns_duckdatasets(self, tmp_path): + async def test_datasets_creates_session_and_returns_duckdatasets( + self, tmp_path + ): with patch("pysus.api.ducklake.client.CACHEPATH", tmp_path): client = DuckLake() @@ -171,7 +177,9 @@ async def test_datasets_connects_if_no_session(self, tmp_path): async def _connect(*args, **kwargs): client._Session = MagicMock(return_value=mock_session) - with patch.object(DuckLake, "connect", new=AsyncMock(side_effect=_connect)): + with patch.object( + DuckLake, "connect", new=AsyncMock(side_effect=_connect) + ): def run_sync(fn, *args, **kwargs): return fn() @@ -197,7 +205,9 @@ def test_setup_engine_has_pysus_schema(self): result = client._setup_engine() calls = [str(c) for c in mock_conn.exec_driver_sql.call_args_list] - assert any("SET search_path" in c and "pysus,main" in c for c in calls) + assert any( + "SET search_path" in c and "pysus,main" in c for c in calls + ) assert result is mock_engine def test_setup_engine_no_pysus_schema(self): @@ -226,13 +236,19 @@ def test_setup_engine_with_credentials(self): mock_conn.exec_driver_sql().fetchone.return_value = None client = DuckLake( - credentials=DuckLakeCredentials(access_key="ak", secret_key="sk") + credentials=DuckLakeCredentials( + access_key="ak", secret_key="sk" + ) ) client._setup_engine() calls = [str(c) for c in mock_conn.exec_driver_sql.call_args_list] - s3_access = any("s3_access_key_id" in c and "ak" in c for c in calls) - s3_secret = any("s3_secret_access_key" in c and "sk" in c for c in calls) + s3_access = any( + "s3_access_key_id" in c and "ak" in c for c in calls + ) + s3_secret = any( + "s3_secret_access_key" in c and "sk" in c for c in calls + ) assert s3_access assert s3_secret @@ -272,7 +288,9 @@ def run_sync(fn, *args, **kwargs): "pysus.api.ducklake.client.to_thread.run_sync", side_effect=run_sync, ): - with patch.object(client, "_setup_engine", return_value=MagicMock()): + with patch.object( + client, "_setup_engine", return_value=MagicMock() + ): await client.connect() mock_dl.assert_awaited_once_with( client._catalog_local, @@ -302,7 +320,9 @@ async def __anext__(self): "pysus.api.ducklake.client.httpx.AsyncClient", return_value=mock_client, ) - sleep_patcher = patch("pysus.api.ducklake.client.sleep", new_callable=AsyncMock) + sleep_patcher = patch( + "pysus.api.ducklake.client.sleep", new_callable=AsyncMock + ) first_stream_cm = MagicMock() first_resp = MagicMock() @@ -351,7 +371,9 @@ async def __anext__(self): "pysus.api.ducklake.client.httpx.AsyncClient", return_value=mock_client, ) - sleep_patcher = patch("pysus.api.ducklake.client.sleep", new_callable=AsyncMock) + sleep_patcher = patch( + "pysus.api.ducklake.client.sleep", new_callable=AsyncMock + ) stream_cm = MagicMock() resp = MagicMock() @@ -556,8 +578,13 @@ class TestDuckLakeDownloadFile: @pytest.mark.asyncio async def test_download_file_invalid_type_raises(self): client = DuckLake() - with pytest.raises(ValueError, match="FTP File was not properly instantiated"): - await client.download("not-a-file", Path("/tmp/test")) # type: ignore + with pytest.raises( + ValueError, match="FTP File was not properly instantiated" + ): + await client.download( + "not-a-file", + Path("/tmp/test"), + ) # type: ignore @pytest.mark.asyncio async def test_download_file_valid(self, tmp_path): @@ -597,7 +624,9 @@ async def test_upload_catalog_with_datasets(self, tmp_path): ds._catalog_local = local_db ds._catalog_name = "catalog_test.duckdb" - with patch.object(DuckLake, "datasets", new=AsyncMock(return_value=[ds])): + with patch.object( + DuckLake, "datasets", new=AsyncMock(return_value=[ds]) + ): await client._upload_catalog() client._s3_client.upload_file.assert_called_once_with( str(local_db), client.bucket, ds._catalog_name @@ -615,6 +644,8 @@ async def test_upload_catalog_skips_missing_local(self, tmp_path): ds._catalog_local = nonexistent ds._catalog_name = "catalog_test.duckdb" - with patch.object(DuckLake, "datasets", new=AsyncMock(return_value=[ds])): + with patch.object( + DuckLake, "datasets", new=AsyncMock(return_value=[ds]) + ): await client._upload_catalog() client._s3_client.upload_file.assert_not_called() diff --git a/pysus/tests/api/ducklake/test_models.py b/pysus/tests/api/ducklake/test_models.py index 2b38ea2b..7d171acb 100644 --- a/pysus/tests/api/ducklake/test_models.py +++ b/pysus/tests/api/ducklake/test_models.py @@ -6,9 +6,9 @@ from unittest.mock import AsyncMock, MagicMock, create_autospec, patch import pytest -from pysus.api.ducklake.catalog.orm.dataset import Dataset from pysus.api.ducklake.catalog.orm.dataset import File as CatalogFile from pysus.api.ducklake.catalog.orm.dataset import Group +from pysus.api.ducklake.catalog.orm.default import Dataset from pysus.api.ducklake.models import DuckDataset, DuckGroup, File # --------------------------------------------------------------------------- From 8b637292be5dfa3f8ff7536533804531f714bed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Sun, 14 Jun 2026 12:31:00 -0300 Subject: [PATCH 4/4] chore: fix tests & implement a semaphore on async downloads that was causing a throttle with too many files downloading at the same time --- .github/workflows/python-package.yml | 8 +- .github/workflows/release.yaml | 5 + Makefile | 4 +- README.md | 6 +- docs/source/conf.py | 3 + .../databases/getting_started_pysus.ipynb | 752 +++++++++--------- docs/source/installation.rst | 4 +- pysus/api/_impl/databases.py | 74 +- pysus/api/client.py | 6 +- pysus/api/ducklake/catalog/adapters.py | 2 +- pysus/api/ducklake/functional.py | 21 +- pysus/api/ducklake/models.py | 8 +- pysus/api/metadata/models.py | 33 +- pysus/tests/api/dadosgov/test_client.py | 8 +- pysus/tests/api/dadosgov/test_models.py | 16 +- pysus/tests/api/ducklake/test_catalog.py | 9 +- pysus/tests/api/ducklake/test_client.py | 505 +++++------- pysus/tests/api/ducklake/test_models.py | 309 +++---- pysus/tests/api/ftp/test_client.py | 6 +- pysus/tests/api/ftp/test_models.py | 8 +- pysus/tests/api/test_client.py | 21 +- pysus/tests/api/test_databases.py | 15 +- pysus/tests/api/test_metadata.py | 46 +- 23 files changed, 811 insertions(+), 1058 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c1c57fee..e2a9b40a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -71,10 +71,10 @@ jobs: - uses: actions/checkout@v4 - name: Build Docker image - run: docker compose -f docker/docker-compose.yaml build + run: docker compose build - name: Start container - run: docker compose -f docker/docker-compose.yaml up -d + run: docker compose up -d - name: Wait for Jupyter run: | @@ -84,8 +84,8 @@ jobs: done - name: Run tests inside container - run: docker compose -f docker/docker-compose.yaml exec -T -w /usr/src jupyter python3 -m pytest -vv pysus/tests/ --retries 3 --retry-delay 15 -x -o cache_dir=/tmp/.pytest_cache + run: docker compose exec -T -w /usr/src jupyter python3 -m pytest -vv pysus/tests/ --retries 3 --retry-delay 15 -x -o cache_dir=/tmp/.pytest_cache - name: Cleanup if: always() - run: docker compose -f docker/docker-compose.yaml down -v + run: docker compose down -v diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fcf654e4..1c2fa5b9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -91,6 +91,11 @@ jobs: with: python-version: "3.12" + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y pandoc + - name: Install dependencies run: | pip install poetry wget diff --git a/Makefile b/Makefile index 8f0a9556..48e578fb 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ export PRINT_HELP_PYSCRIPT help: @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) -DOCKER = docker compose -p pysus -f docker/docker-compose.yaml +DOCKER = docker compose -p pysus SERVICE := SEMANTIC_RELEASE = npx --yes \ -p semantic-release \ @@ -56,7 +56,7 @@ test-pysus: ## run tests quickly with the default Python .PHONY: test-pysus-with-coverage test-pysus-with-coverage: ## run tests with coverage report - poetry run pytest -vv pysus/tests/ --retries 3 --retry-delay 15 --cov=pysus --cov-report=xml:coverage.xml --cov-report=term-missing + poetry run pytest -vv pysus/tests/ --cov=pysus --cov-report=xml:coverage.xml --cov-report=term-missing .PHONY: lint lint: diff --git a/README.md b/README.md index f0a029f4..a34c1d27 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ docker run -p 8888:8888 alertadengue/pysus Or build locally and start the container: ```bash -docker compose -f docker/docker-compose.yaml up --build +docker compose up --build ``` Then open [http://127.0.0.1:8888/lab](http://127.0.0.1:8888/lab) in your browser. @@ -45,7 +45,7 @@ Then open [http://127.0.0.1:8888/lab](http://127.0.0.1:8888/lab) in your browser Stop the container: ```bash -docker compose -f docker/docker-compose.yaml down +docker compose down ``` ## Quick Start @@ -268,7 +268,7 @@ pytest tests/ Run tests inside the Docker container: ```bash -docker compose -f docker/docker-compose.yaml exec -T -w /usr/src jupyter python3 -m pytest pysus/tests/ +docker compose exec -T -w /usr/src jupyter python3 -m pytest pysus/tests/ ``` ## License diff --git a/docs/source/conf.py b/docs/source/conf.py index 9b2510cd..6d3391c4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -21,7 +21,10 @@ } templates_path = ["_templates"] + +# Explicitly map extensions to ensure notebooks are routed to nbsphinx source_suffix = ".rst" + master_doc = "index" project = "PySUS" diff --git a/docs/source/databases/getting_started_pysus.ipynb b/docs/source/databases/getting_started_pysus.ipynb index 8caa0009..1cf40f22 100644 --- a/docs/source/databases/getting_started_pysus.ipynb +++ b/docs/source/databases/getting_started_pysus.ipynb @@ -9,9 +9,9 @@ "\n", "*Your complete guide to accessing Brazilian public health data*\n", "\n", - "**PySUS v2.1.0 · Python 3.10+**\n", + "**PySUS v3.0.0+ · Python 3.10-3.13**\n", "\n", - "Notebook contribution — [AlertaDengue/PySUS](https://github.com/AlertaDengue/PySUS) · Issue #277\n", + "Notebook contribution — [AlertaDengue/PySUS](https://github.com/AlertaDengue/PySUS)\n", "\n", "---\n", "\n", @@ -19,15 +19,14 @@ "\n", "PySUS is a Python library that provides easy access to publicly available datasets\n", "from Brazil's Unified Health System (SUS), published by DATASUS.\n", - "It handles file discovery, downloading, and parsing — so you can focus on data analysis\n", - "rather than dealing with legacy file formats.\n", + "It handles background cloud orchestrations, downloading, and local schema management — so you can focus on data analysis\n", + "rather than dealing with legacy download workflows.\n", "\n", "This notebook presents a complete beginner-friendly workflow,\n", "from installation to the first data exploration using real SUS data.\n", "\n", "> **No prior knowledge of DATASUS is required.**\n", - "> All datasets are freely available at [datasus.saude.gov.br](https://datasus.saude.gov.br)\n", - "> and fetched directly from official government servers." + "> All datasets are optimized as parquet records and fetched securely via our high-performance backend pipelines." ] }, { @@ -63,74 +62,63 @@ "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: pysus in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (2.2.0)\n", - "Requirement already satisfied: Unidecode<2.0.0,>=1.3.6 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (1.4.0)\n", - "Requirement already satisfied: aioftp<0.22.0,>=0.21.4 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (0.21.4)\n", - "Requirement already satisfied: anyio<5.0.0,>=4.13.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (4.13.0)\n", - "Requirement already satisfied: bigtree<0.13.0,>=0.12.2 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (0.12.5)\n", - "Requirement already satisfied: boto3<2.0.0,>=1.42.89 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (1.43.24)\n", - "Requirement already satisfied: chardet<8.0.0,>=7.4.0.post2 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (7.4.3)\n", - "Requirement already satisfied: dateparser<2.0.0,>=1.1.8 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (1.4.0)\n", - "Requirement already satisfied: dbfread==2.0.7 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (2.0.7)\n", - "Requirement already satisfied: dotenv<0.10.0,>=0.9.9 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (0.9.9)\n", - "Requirement already satisfied: duckdb<2.0.0,>=1.4.4 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (1.5.3)\n", - "Requirement already satisfied: duckdb-engine<0.18.0,>=0.17.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (0.17.0)\n", - "Requirement already satisfied: fastparquet<=2024.11.0,>=2023.10.1 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (2024.11.0)\n", - "Requirement already satisfied: httpx>=0.28.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (0.28.1)\n", - "Requirement already satisfied: loguru<0.7.0,>=0.6.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (0.6.0)\n", - "Requirement already satisfied: numpy<2,>=1.22 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (1.26.4)\n", - "Requirement already satisfied: pandas<3.0.0,>=2.2.2 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (2.3.3)\n", - "Requirement already satisfied: pyarrow>=11.0.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (24.0.0)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.12.5 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (2.13.4)\n", - "Requirement already satisfied: pyreaddbc>=2.0.4 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (2.0.4)\n", - "Requirement already satisfied: python-dateutil==2.8.2 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (2.8.2)\n", - "Requirement already satisfied: python-magic<0.5.0,>=0.4.27 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (0.4.27)\n", - "Requirement already satisfied: sqlalchemy<3.0.0,>=2.0.48 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (2.0.50)\n", - "Requirement already satisfied: tqdm>=4.67.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (4.68.1)\n", - "Requirement already satisfied: typer<0.25.0,>=0.24.1 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (0.24.2)\n", - "Requirement already satisfied: typing-extensions>=4.10.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (4.15.0)\n", - "Requirement already satisfied: wget<4.0,>=3.2 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pysus) (3.2)\n", - "Requirement already satisfied: six>=1.5 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from python-dateutil==2.8.2->pysus) (1.17.0)\n", - "Requirement already satisfied: idna>=2.8 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from anyio<5.0.0,>=4.13.0->pysus) (3.18)\n", - "Requirement already satisfied: botocore<1.44.0,>=1.43.24 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from boto3<2.0.0,>=1.42.89->pysus) (1.43.24)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from boto3<2.0.0,>=1.42.89->pysus) (1.1.0)\n", - "Requirement already satisfied: s3transfer<0.19.0,>=0.18.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from boto3<2.0.0,>=1.42.89->pysus) (0.18.0)\n", - "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from botocore<1.44.0,>=1.43.24->boto3<2.0.0,>=1.42.89->pysus) (2.7.0)\n", - "Requirement already satisfied: pytz>=2024.2 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (2026.2)\n", - "Requirement already satisfied: regex>=2024.9.11 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (2026.5.9)\n", - "Requirement already satisfied: tzlocal>=0.2 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (5.3.1)\n", - "Requirement already satisfied: python-dotenv in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from dotenv<0.10.0,>=0.9.9->pysus) (1.2.2)\n", - "Requirement already satisfied: packaging>=21 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from duckdb-engine<0.18.0,>=0.17.0->pysus) (26.2)\n", - "Requirement already satisfied: cramjam>=2.3 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from fastparquet<=2024.11.0,>=2023.10.1->pysus) (2.11.0)\n", - "Requirement already satisfied: fsspec in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from fastparquet<=2024.11.0,>=2023.10.1->pysus) (2026.4.0)\n", - "Requirement already satisfied: colorama>=0.3.4 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from loguru<0.7.0,>=0.6.0->pysus) (0.4.6)\n", - "Requirement already satisfied: win32-setctime>=1.0.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from loguru<0.7.0,>=0.6.0->pysus) (1.2.0)\n", - "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pandas<3.0.0,>=2.2.2->pysus) (2026.2)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.46.4 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (2.46.4)\n", - "Requirement already satisfied: typing-inspection>=0.4.2 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (0.4.2)\n", - "Requirement already satisfied: greenlet>=1 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from sqlalchemy<3.0.0,>=2.0.48->pysus) (3.5.1)\n", - "Requirement already satisfied: click>=8.2.1 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from typer<0.25.0,>=0.24.1->pysus) (8.4.1)\n", - "Requirement already satisfied: shellingham>=1.3.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from typer<0.25.0,>=0.24.1->pysus) (1.5.4)\n", - "Requirement already satisfied: rich>=12.3.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from typer<0.25.0,>=0.24.1->pysus) (15.0.0)\n", - "Requirement already satisfied: annotated-doc>=0.0.2 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from typer<0.25.0,>=0.24.1->pysus) (0.0.4)\n", - "Requirement already satisfied: certifi in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from httpx>=0.28.0->pysus) (2026.5.20)\n", - "Requirement already satisfied: httpcore==1.* in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from httpx>=0.28.0->pysus) (1.0.9)\n", - "Requirement already satisfied: h11>=0.16 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from httpcore==1.*->httpx>=0.28.0->pysus) (0.16.0)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (4.2.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (2.20.0)\n", - "Requirement already satisfied: mdurl~=0.1 in c:\\users\\vladi\\appdata\\local\\programs\\python\\python313\\lib\\site-packages (from markdown-it-py>=2.2.0->rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (0.1.2)\n", + "Requirement already satisfied: pysus in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (2.1.0)\n", + "Requirement already satisfied: Unidecode<2.0.0,>=1.3.6 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.4.0)\n", + "Requirement already satisfied: aioftp<0.22.0,>=0.21.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.21.4)\n", + "Requirement already satisfied: anyio<5.0.0,>=4.13.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (4.13.0)\n", + "Requirement already satisfied: bigtree<0.13.0,>=0.12.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.12.5)\n", + "Requirement already satisfied: boto3<2.0.0,>=1.42.89 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.43.14)\n", + "Requirement already satisfied: chardet<8.0.0,>=7.4.0.post2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (7.4.3)\n", + "Requirement already satisfied: dateparser<2.0.0,>=1.1.8 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.4.0)\n", + "Requirement already satisfied: dbfread==2.0.7 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.0.7)\n", + "Requirement already satisfied: dotenv<0.10.0,>=0.9.9 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.9.9)\n", + "Requirement already satisfied: duckdb<2.0.0,>=1.4.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.5.3)\n", + "Requirement already satisfied: duckdb-engine<0.18.0,>=0.17.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.17.0)\n", + "Requirement already satisfied: fastparquet<=2024.11.0,>=2023.10.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2024.11.0)\n", + "Requirement already satisfied: httpx>=0.28.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.28.1)\n", + "Requirement already satisfied: loguru<0.7.0,>=0.6.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.6.0)\n", + "Requirement already satisfied: numpy<2,>=1.22 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.26.4)\n", + "Requirement already satisfied: pandas<3.0.0,>=2.2.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.3.3)\n", + "Requirement already satisfied: pyarrow>=11.0.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (24.0.0)\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.12.5 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.13.4)\n", + "Requirement already satisfied: pyreaddbc>=2.0.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.0.4)\n", + "Requirement already satisfied: python-dateutil==2.8.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.8.2)\n", + "Requirement already satisfied: python-magic<0.5.0,>=0.4.27 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.4.27)\n", + "Requirement already satisfied: sqlalchemy<3.0.0,>=2.0.48 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.0.50)\n", + "Requirement already satisfied: tqdm>=4.67.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (4.67.3)\n", + "Requirement already satisfied: typer<0.25.0,>=0.24.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.24.2)\n", + "Requirement already satisfied: typing-extensions>=4.10.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (4.15.0)\n", + "Requirement already satisfied: wget<4.0,>=3.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (3.2)\n", + "Requirement already satisfied: six>=1.5 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from python-dateutil==2.8.2->pysus) (1.17.0)\n", + "Requirement already satisfied: idna>=2.8 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from anyio<5.0.0,>=4.13.0->pysus) (3.16)\n", + "Requirement already satisfied: botocore<1.44.0,>=1.43.14 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from boto3<2.0.0,>=1.42.89->pysus) (1.43.14)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from boto3<2.0.0,>=1.42.89->pysus) (1.1.0)\n", + "Requirement already satisfied: s3transfer<0.18.0,>=0.17.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from boto3<2.0.0,>=1.42.89->pysus) (0.17.0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from botocore<1.44.0,>=1.43.14->boto3<2.0.0,>=1.42.89->pysus) (2.7.0)\n", + "Requirement already satisfied: pytz>=2024.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (2026.2)\n", + "Requirement already satisfied: regex>=2024.9.11 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (2026.5.9)\n", + "Requirement already satisfied: tzlocal>=0.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (5.3.1)\n", + "Requirement already satisfied: python-dotenv in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from dotenv<0.10.0,>=0.9.9->pysus) (1.2.2)\n", + "Requirement already satisfied: packaging>=21 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from duckdb-engine<0.18.0,>=0.17.0->pysus) (26.2)\n", + "Requirement already satisfied: cramjam>=2.3 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from fastparquet<=2024.11.0,>=2023.10.1->pysus) (2.11.0)\n", + "Requirement already satisfied: fsspec in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from fastparquet<=2024.11.0,>=2023.10.1->pysus) (2026.4.0)\n", + "Requirement already satisfied: tzdata>=2022.7 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pandas<3.0.0,>=2.2.2->pysus) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (0.4.2)\n", + "Requirement already satisfied: greenlet>=1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from sqlalchemy<3.0.0,>=2.0.48->pysus) (3.5.1)\n", + "Requirement already satisfied: click>=8.2.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (8.4.1)\n", + "Requirement already satisfied: shellingham>=1.3.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (1.5.4)\n", + "Requirement already satisfied: rich>=12.3.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (15.0.0)\n", + "Requirement already satisfied: annotated-doc>=0.0.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (0.0.4)\n", + "Requirement already satisfied: certifi in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from httpx>=0.28.0->pysus) (2026.5.20)\n", + "Requirement already satisfied: httpcore==1.* in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from httpx>=0.28.0->pysus) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.28.0->pysus) (0.16.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (2.20.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (0.1.2)\n", "Note: you may need to restart the kernel to use updated packages.\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "[notice] A new release of pip is available: 25.2 -> 26.1.2\n", - "[notice] To update, run: C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\python.exe -m pip install --upgrade pip\n" - ] } ], "source": [ @@ -146,8 +134,7 @@ "---\n", "## 2. Checking Your Installation\n", "\n", - "After installing PySUS, verify that the package is available and check the installed version.\n", - "If the installation was successful, Python will display the version number." + "After installing PySUS, verify that the package is available and check the installed version." ] }, { @@ -160,14 +147,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "2.2.0\n" + "2.1.0\n" ] } ], "source": [ "import pysus\n", "\n", - "print(pysus.__version__)" + "print(pysus.get_version())" ] }, { @@ -179,7 +166,7 @@ "## 3. Exploring the Package\n", "\n", "PySUS exposes each health dataset as a simple callable function.\n", - "Use `dir(pysus)` to see everything available:" + "Use `dir(pysus)` to see the API entry points available:" ] }, { @@ -192,14 +179,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "['CACHEPATH', 'Final', '__annotations__', '__builtins__', '__cached__', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__path__', '__spec__', '__version__', 'api', 'ciha', 'cnes', 'get_version', 'ibge', 'importlib_metadata', 'list_files', 'os', 'pathlib', 'pni', 'sia', 'sih', 'sim', 'sinan', 'sinasc', 'version']\n" + "['CACHEPATH', 'Final', 'api', 'ciha', 'cnes', 'get_version', 'ibge', 'importlib_metadata', 'list_files', 'os', 'pathlib', 'pni', 'sia', 'sih', 'sim', 'sinan', 'sinasc', 'version']\n" ] } ], "source": [ "import pysus\n", "\n", - "print(dir(pysus))" + "print([item for item in dir(pysus) if not item.startswith('_')])" ] }, { @@ -229,244 +216,73 @@ "---\n", "## 4. Understanding the Parameters\n", "\n", - "All dataset functions share the same parameter pattern.\n", - "Here is the signature for `sinasc()`:\n", + "All dataset functions share the same unified parameter pattern. For instance, querying `sinasc()` can be targeted like this:\n", "\n", "```python\n", "sinasc(\n", " state = \"SP\", # two-letter Brazilian state code\n", - " year = 2022, # integer or list of integers\n", + " year = 2022, # integer, list, or range of integers\n", ")\n", "```\n", "\n", "| Parameter | Type | Description | Example |\n", "|-----------|------|-------------|---------|\n", "| `state` | `str` | Two-letter state abbreviation | `\"SP\"`, `\"RJ\"`, `\"MG\"` |\n", - "| `year` | `int` or `list[int]` | Single year or list of years | `2022` or `[2021, 2022]` |\n", - "| `group` | `str` or `None` | Sub-group code (SINAN only) | `\"DENG\"` (dengue) |" + "| `year` | `int`, `list` or `range` | Targets execution span | `2022` or `range(2020, 2026)` |\n", + "| `group` | `str` or `None` | Sub-group code (SINAN only) | `\"DENG\"` (dengue) |\n", + "| `as_dataframe` | `bool` | Instantly return a Pandas Dataframe | `True` or `False` |" ] }, { "cell_type": "markdown", - "id": "md-5061227682282249364", + "id": "md-1656243573859326293", "metadata": {}, "source": [ "---\n", - "## 5. Listing Available Files\n", - "\n", - "Before downloading, use `list_files()` to discover what is available\n", - "for a given dataset, state, and year.\n", + "## 5. Downloading Your First Dataset\n", "\n", - "> **Note:** Jupyter runs an async event loop internally.\n", - "> We use `nest_asyncio` to allow PySUS async calls inside the notebook." + "Let's download SINASC birth records for Rio de Janeiro, 2022.\n", + "By default, the function handles throttled parallel downloads and returns a list of local file paths tracking your parquet targets." ] }, { "cell_type": "code", "execution_count": 4, - "id": "cd-974912376655675624", + "id": "cd-3795208869041744895", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Exception in callback Task.__step()\n", - "handle: \n", - "Traceback (most recent call last):\n", - " File \"C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\asyncio\\events.py\", line 89, in _run\n", - " self._context.run(self._callback, *self._args)\n", - " ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - "RuntimeError: cannot enter context: <_contextvars.Context object at 0x0000026B7552CC80> is already entered\n", - "Exception in callback Task.__step()\n", - "handle: \n", - "Traceback (most recent call last):\n", - " File \"C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\asyncio\\events.py\", line 89, in _run\n", - " self._context.run(self._callback, *self._args)\n", - " ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - "RuntimeError: cannot enter context: <_contextvars.Context object at 0x0000026B7552CC80> is already entered\n", - "Exception in callback Task.__step()\n", - "handle: \n", - "Traceback (most recent call last):\n", - " File \"C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\asyncio\\events.py\", line 89, in _run\n", - " self._context.run(self._callback, *self._args)\n", - " ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - "RuntimeError: cannot enter context: <_contextvars.Context object at 0x0000026B7552CC80> is already entered\n", - "Exception in callback Task.__step()\n", - "handle: \n", - "Traceback (most recent call last):\n", - " File \"C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\asyncio\\events.py\", line 89, in _run\n", - " self._context.run(self._callback, *self._args)\n", - " ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - "RuntimeError: cannot enter context: <_contextvars.Context object at 0x0000026B7552CC80> is already entered\n", - "Exception in callback Task.__step()\n", - "handle: \n", - "Traceback (most recent call last):\n", - " File \"C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\asyncio\\events.py\", line 89, in _run\n", - " self._context.run(self._callback, *self._args)\n", - " ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - "RuntimeError: cannot enter context: <_contextvars.Context object at 0x0000026B7552CC80> is already entered\n", - "Exception in callback Task.__step()\n", - "handle: \n", - "Traceback (most recent call last):\n", - " File \"C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\asyncio\\events.py\", line 89, in _run\n", - " self._context.run(self._callback, *self._args)\n", - " ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - "RuntimeError: cannot enter context: <_contextvars.Context object at 0x0000026B7552CC80> is already entered\n", - "Exception in callback Task.__step()\n", - "handle: \n", - "Traceback (most recent call last):\n", - " File \"C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\asyncio\\events.py\", line 89, in _run\n", - " self._context.run(self._callback, *self._args)\n", - " ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - "RuntimeError: cannot enter context: <_contextvars.Context object at 0x0000026B7552CC80> is already entered\n", - "Task was destroyed but it is pending!\n", - "task: .run_in_context() done, defined at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\ipykernel\\utils.py:57> wait_for= cb=[Task.__wakeup()]> cb=[ZMQStream._run_callback.._log_error() at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\zmq\\eventloop\\zmqstream.py:563]>\n", - "C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\ast.py:50: RuntimeWarning: coroutine 'Kernel.shell_main' was never awaited\n", - " return compile(source, filename, mode, flags,\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "Task was destroyed but it is pending!\n", - "task: cb=[Task.__wakeup()]>\n", - "Task was destroyed but it is pending!\n", - "task: .run_in_context() done, defined at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\ipykernel\\utils.py:57> wait_for= cb=[Task.__wakeup()]> cb=[ZMQStream._run_callback.._log_error() at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\zmq\\eventloop\\zmqstream.py:563]>\n", - "Task was destroyed but it is pending!\n", - "task: cb=[Task.__wakeup()]>\n", - "Task was destroyed but it is pending!\n", - "task: .run_in_context() done, defined at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\ipykernel\\utils.py:57> wait_for= cb=[Task.__wakeup()]> cb=[ZMQStream._run_callback.._log_error() at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\zmq\\eventloop\\zmqstream.py:563]>\n", - "Task was destroyed but it is pending!\n", - "task: cb=[Task.__wakeup()]>\n", - "Task was destroyed but it is pending!\n", - "task: .run_in_context() done, defined at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\ipykernel\\utils.py:57> wait_for= cb=[Task.__wakeup()]> cb=[ZMQStream._run_callback.._log_error() at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\zmq\\eventloop\\zmqstream.py:563]>\n", - "Task was destroyed but it is pending!\n", - "task: cb=[Task.__wakeup()]>\n", - "Task was destroyed but it is pending!\n", - "task: .run_in_context() done, defined at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\ipykernel\\utils.py:57> wait_for= cb=[Task.__wakeup()]> cb=[ZMQStream._run_callback.._log_error() at C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\site-packages\\zmq\\eventloop\\zmqstream.py:563]>\n", - "Task was destroyed but it is pending!\n", - "task: cb=[Task.__wakeup()]>\n", - "Exception in callback Task.__step()\n", - "handle: \n", - "Traceback (most recent call last):\n", - " File \"C:\\Users\\vladi\\AppData\\Local\\Programs\\Python\\Python313\\Lib\\asyncio\\events.py\", line 89, in _run\n", - " self._context.run(self._callback, *self._args)\n", - " ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - "RuntimeError: cannot enter context: <_contextvars.Context object at 0x0000026B7552CC80> is already entered\n" + "Downloading sinasc: 100%|█████████████████████████████████████████████████| 1/1 [00:02<00:00, 2.88s/file]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Files found: 1\n" + "Parquet file targets saved locally: ['/home/bida/pysus/downloads/ducklake/sinasc/DNRJ2022.parquet']\n" ] }, { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
namepathdatasetgroupyearmonthstatemodify
0public\\data\\ftp\\sinasc\\DNRJ2022.parquetpublic\\data\\ftp\\sinasc\\DNRJ2022.parquetsinascNone2022NoneRJ2023-12-20 16:45:00
\n", - "
" - ], - "text/plain": [ - " name \\\n", - "0 public\\data\\ftp\\sinasc\\DNRJ2022.parquet \n", - "\n", - " path dataset group year month state \\\n", - "0 public\\data\\ftp\\sinasc\\DNRJ2022.parquet sinasc None 2022 None RJ \n", - "\n", - " modify \n", - "0 2023-12-20 16:45:00 " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] } ], - "source": [ - "# Required to run PySUS async functions inside Jupyter\n", - "import nest_asyncio\n", - "nest_asyncio.apply()\n", - "\n", - "from pysus import list_files\n", - "\n", - "# List SINASC files available for Rio de Janeiro, 2022\n", - "available = list_files(dataset=\"SINASC\", state=\"RJ\", year=2022)\n", - "\n", - "print(f\"Files found: {len(available)}\")\n", - "available" - ] - }, - { - "cell_type": "markdown", - "id": "md-1656243573859326293", - "metadata": {}, - "source": [ - "---\n", - "## 6. Downloading Your First Dataset\n", - "\n", - "Now download SINASC birth records for Rio de Janeiro, 2022.\n", - "The function returns a **pandas DataFrame** directly —\n", - "no manual file handling required.\n", - "\n", - "> **Note:** The first download may take about 30 seconds depending on your connection.\n", - "> PySUS caches files locally, so the next call for the same data will be much faster." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cd-3795208869041744895", - "metadata": {}, - "outputs": [], "source": [ "from pysus import sinasc\n", + "import pandas as pd\n", "\n", - "# Download live birth records — Rio de Janeiro, 2022\n", - "df = sinasc(state=\"RJ\", year=2022)\n", + "# Fetch local storage paths for the dataset\n", + "files = sinasc(state=\"RJ\", year=2022)\n", + "print(f\"Parquet file targets saved locally: {files}\")\n", "\n", - "print(f\"Rows : {len(df):,}\")\n", - "print(f\"Columns: {df.shape[1]}\")\n", - "df.head()" + "# Open records with pandas\n", + "df = pd.read_parquet(files)" ] }, { @@ -474,11 +290,10 @@ "id": "md-7974618639364585060", "metadata": {}, "source": [ - "> **Tip:** To download multiple years at once, pass a list:\n", + "> **Tip:** To download an entire multi-year range automatically into a unified dataframe framework, pass `as_dataframe=True`:\n", "> ```python\n", - "> df = sinasc(state=\"SP\", year=[2020, 2021, 2022])\n", - "> ```\n", - "> PySUS concatenates the results into a single DataFrame automatically." + "> df = sinasc(state=\"SP\", year=range(2020, 2025), as_dataframe=True)\n", + "> ```" ] }, { @@ -487,17 +302,25 @@ "metadata": {}, "source": [ "---\n", - "## 7. Inspecting the Data\n", + "## 6. Inspecting the Data\n", "\n", - "Three essential commands for exploring any new DataFrame:" + "Three essential commands for exploring your newly loaded DataFrame:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "cd-4586221015219041710", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(180369, 61)\n" + ] + } + ], "source": [ "# Shape: number of rows and columns\n", "print(df.shape)" @@ -505,10 +328,85 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "cd-6113157604871589284", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 180369 entries, 0 to 180368\n", + "Data columns (total 61 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 ORIGEM 180369 non-null object\n", + " 1 CODESTAB 180369 non-null object\n", + " 2 CODMUNNASC 180369 non-null object\n", + " 3 LOCNASC 180369 non-null object\n", + " 4 IDADEMAE 180369 non-null object\n", + " 5 ESTCIVMAE 180369 non-null object\n", + " 6 ESCMAE 180369 non-null object\n", + " 7 CODOCUPMAE 180369 non-null object\n", + " 8 QTDFILVIVO 180369 non-null object\n", + " 9 QTDFILMORT 180369 non-null object\n", + " 10 CODMUNRES 180369 non-null object\n", + " 11 GESTACAO 180369 non-null object\n", + " 12 GRAVIDEZ 180369 non-null object\n", + " 13 PARTO 180369 non-null object\n", + " 14 CONSULTAS 180369 non-null object\n", + " 15 DTNASC 180369 non-null object\n", + " 16 HORANASC 180369 non-null object\n", + " 17 SEXO 180369 non-null object\n", + " 18 APGAR1 180369 non-null object\n", + " 19 APGAR5 180369 non-null object\n", + " 20 RACACOR 180369 non-null object\n", + " 21 PESO 180369 non-null object\n", + " 22 IDANOMAL 180369 non-null object\n", + " 23 DTCADASTRO 180369 non-null object\n", + " 24 CODANOMAL 180369 non-null object\n", + " 25 NUMEROLOTE 180369 non-null object\n", + " 26 VERSAOSIST 180369 non-null object\n", + " 27 DTRECEBIM 180369 non-null object\n", + " 28 DIFDATA 180369 non-null object\n", + " 29 DTRECORIGA 180369 non-null object\n", + " 30 NATURALMAE 180369 non-null object\n", + " 31 CODMUNNATU 180369 non-null object\n", + " 32 CODUFNATU 180369 non-null object\n", + " 33 ESCMAE2010 180369 non-null object\n", + " 34 SERIESCMAE 180369 non-null object\n", + " 35 DTNASCMAE 180369 non-null object\n", + " 36 RACACORMAE 180369 non-null object\n", + " 37 QTDGESTANT 180369 non-null object\n", + " 38 QTDPARTNOR 180369 non-null object\n", + " 39 QTDPARTCES 180369 non-null object\n", + " 40 IDADEPAI 180369 non-null object\n", + " 41 DTULTMENST 180369 non-null object\n", + " 42 SEMAGESTAC 180369 non-null object\n", + " 43 TPMETESTIM 180369 non-null object\n", + " 44 CONSPRENAT 180369 non-null object\n", + " 45 MESPRENAT 180369 non-null object\n", + " 46 TPAPRESENT 180369 non-null object\n", + " 47 STTRABPART 180369 non-null object\n", + " 48 STCESPARTO 180369 non-null object\n", + " 49 TPNASCASSI 180369 non-null object\n", + " 50 TPFUNCRESP 180369 non-null object\n", + " 51 TPDOCRESP 180369 non-null object\n", + " 52 DTDECLARAC 180369 non-null object\n", + " 53 ESCMAEAGR1 180369 non-null object\n", + " 54 STDNEPIDEM 180369 non-null object\n", + " 55 STDNNOVA 180369 non-null object\n", + " 56 CODPAISRES 180369 non-null object\n", + " 57 TPROBSON 180369 non-null object\n", + " 58 PARIDADE 180369 non-null object\n", + " 59 KOTELCHUCK 180369 non-null object\n", + " 60 CONTADOR 180369 non-null object\n", + "dtypes: object(61)\n", + "memory usage: 83.9+ MB\n" + ] + } + ], "source": [ "# Column names and data types\n", "df.info()" @@ -516,28 +414,188 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "cd-1603410318595514302", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ORIGEMCODESTABCODMUNNASCLOCNASCIDADEMAEESTCIVMAEESCMAECODOCUPMAEQTDFILVIVOQTDFILMORT...TPDOCRESPDTDECLARACESCMAEAGR1STDNEPIDEMSTDNNOVACODPAISRESTPROBSONPARIDADEKOTELCHUCKCONTADOR
count180369180369180369180369180369180369180369180369180369180369...180369180369180369180369180369180369180369180369180369180369
unique1323150551778762017...6380142111126180369
top17011857330455125149999920000...306011051514077
freq18036960416942717878195831126101114747780676238136704...75107215742874180352180369180369435451118971171841
\n", + "

4 rows Ă— 61 columns

\n", + "
" + ], + "text/plain": [ + " ORIGEM CODESTAB CODMUNNASC LOCNASC IDADEMAE ESTCIVMAE ESCMAE \\\n", + "count 180369 180369 180369 180369 180369 180369 180369 \n", + "unique 1 323 150 5 51 7 7 \n", + "top 1 7011857 330455 1 25 1 4 \n", + "freq 180369 6041 69427 178781 9583 112610 111474 \n", + "\n", + " CODOCUPMAE QTDFILVIVO QTDFILMORT ... TPDOCRESP DTDECLARAC ESCMAEAGR1 \\\n", + "count 180369 180369 180369 ... 180369 180369 180369 \n", + "unique 876 20 17 ... 6 380 14 \n", + "top 999992 00 00 ... 3 06 \n", + "freq 77806 76238 136704 ... 75107 2157 42874 \n", + "\n", + " STDNEPIDEM STDNNOVA CODPAISRES TPROBSON PARIDADE KOTELCHUCK CONTADOR \n", + "count 180369 180369 180369 180369 180369 180369 180369 \n", + "unique 2 1 1 11 2 6 180369 \n", + "top 0 1 1 05 1 5 14077 \n", + "freq 180352 180369 180369 43545 111897 117184 1 \n", + "\n", + "[4 rows x 61 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Descriptive statistics for numeric columns\n", + "# Statistical baseline summaries\n", "df.describe()" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "cd-4002606025712713324", - "metadata": {}, - "outputs": [], - "source": [ - "# Check for missing values (top 10 columns with most nulls)\n", - "missing = df.isnull().sum().sort_values(ascending=False)\n", - "print(\"Columns with missing values:\")\n", - "print(missing[missing > 0].head(10))" - ] - }, { "cell_type": "markdown", "id": "md-9198316201049293265", @@ -556,51 +614,13 @@ "| `PESO` | Birth weight in grams |" ] }, - { - "cell_type": "markdown", - "id": "md-7153773555296681005", - "metadata": {}, - "source": [ - "---\n", - "## 8. Understanding the Local Cache\n", - "\n", - "PySUS caches downloaded files locally.\n", - "The second time you request the same data, it loads from disk instead of\n", - "re-downloading — making repeated analysis much faster." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cd-2621658557253256783", - "metadata": {}, - "outputs": [], - "source": [ - "import pysus\n", - "\n", - "# See where PySUS stores its cached files\n", - "print(pysus.CACHEPATH)\n", - "# Typical output: ~/.pysus" - ] - }, - { - "cell_type": "markdown", - "id": "md-5829319428437279100", - "metadata": {}, - "source": [ - "> The cache directory is `~/.pysus` by default on Linux and macOS,\n", - "> and `C:\\Users\\\\.pysus` on Windows.\n", - "> You can safely delete it to free disk space;\n", - "> data will be re-downloaded on the next call." - ] - }, { "cell_type": "markdown", "id": "md-4035189673861699865", "metadata": {}, "source": [ "---\n", - "## 9. Your First Visualisation\n", + "## 7. Data Visualisation\n", "\n", "Let's plot the **monthly distribution of births** in Rio de Janeiro for 2022.\n", "\n", @@ -610,10 +630,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "cd-7840088167428654087", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA90AAAHqCAYAAAAZLi26AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQAAah9JREFUeJzt3XlYVHX///HXALK4gKEJUoq4i2toKuqdqBQumaa53WYoLrfmTllauWa53LlkmuaK3bdblnrnEmbkkoaaW5aRpeFSiruQqKhwfn/0Y75OAwrKcQCfj+ua63I+n8858z5nzoy85mwWwzAMAQAAAACAHOfk6AIAAAAAAMivCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QCQD3Tv3l0Wi0XHjh3L0vgtW7bIYrFozJgxptZ1N8eOHZPFYlH37t0dWsf9GDNmjCwWi7Zs2ZJj84yKipLFYlFUVFSOzTMvc+T2asb7mxdZLBaFhIQ4tIb9+/fL2dlZS5cudWgdZnjxxRfl7++v69evO7oUACYgdAN46KUHP4vFIl9fX926dSvDcXFxcdZxZcqUeaA15paQnFUWi0WVK1d2dBnZVqZMGet7nP5wc3NTQECA+vTpk+UfNbIiN4QYRwgJCbFZv05OTipatKgaNmyojz76SGlpaY4u8YFKXx8JCQmOLiXXi4yMVOXKldW5c2dr24ULFzR37lw999xzKlu2rNzc3FS8eHG1aNFCGzduzHReKSkpGjdunCpUqCB3d3f5+fmpT58+Onv2rN3YAwcOaOTIkapfv75KlCghNzc3lS1bVi+//LL++OMPu/H3UtOoUaP0xx9/aPr06dlfMQByPRdHFwAAuYWLi4vOnDmjDRs26LnnnrPrX7BggZyc+K0yJz322GOKi4uTl5eXo0uxcnZ21ltvvWV9fvnyZe3atUvz5s3TqlWrtG/fPpUuXdraP2DAAHXu3NmmDXf3yiuvqHDhwkpNTdXx48e1atUq9e3bV/v27dNHH31kM7Zu3bqKi4tT8eLFHVQt4uLiVLBgQYe9/tdff60tW7bYfQ+vXLlS/fr1k5+fn5o1a6bHHntMv//+uz777DNFR0dr8uTJGjZsmM280tLS1KZNG23cuFH169dX+/bt9euvv2r+/PmKiYnRzp079eijj1rH9+3bV7t27VLdunXVuXNnubm5adeuXZo9e7ZWrlypb775xuZHxnupqWLFimrTpo0mTpyogQMHqlChQiatSQAOYQDAQy4+Pt6QZDz11FOGl5eX0aZNG7sxN2/eNHx8fIxnnnnGcHNzM/z9/R9ojZs3bzYkGaNHj86wPzw83JBkxMfH58j87pcko1KlSqbM20z+/v6Gm5tbhn0vv/yyIckYOXJkjryWJKNx48YZ9i1atMiQZCxatChHXis3ady4sSHJOH36tE37r7/+ahQqVMiwWCzG0aNHHVSdvdGjRxuSjM2bN5sy/8zWB2y98MILhoeHh5GYmGjTHhMTY3z++edGamqqTfvPP/9seHl5GQUKFDD++OMPm76FCxcakowuXboYaWlp1vbZs2cbkow+ffrYjJ8xY4bx66+/2tU0ceJEQ5LRsmXL+67JMAxj1apVhiRj/vz5d1gTAPIidtkAwP/n4eGhzp07a/369XaHGK5bt05nzpxRREREptMnJydr9OjRqly5stzd3eXt7a1WrVppx44ddmNvP0906dKlqlWrljw8PFSyZEkNHjxY165dsxnbpEkTSdLYsWNtDs39++HOhmFoxowZqly5stzc3OTv76+xY8fe9ZDdtLQ0+fv7q1ixYkpJSclwzFNPPSUXFxf9/vvvd5xXdmR0TnezZs3k5OSk48ePZzjNoEGDZLFYtGnTJpv2bdu2qXXr1ipevLjc3NxUoUIFvfXWW7p69WqO1Nq8eXNJ0vnz523aMzrn9/bliouL0/PPP69ixYpZz9O2WCySpK1bt9q8nxmdw/3ll1+qQYMGKliwoIoVK6bw8HBduHDBbtzmzZvVokUL+fn5yc3NTT4+PvrHP/6huXPn5sjym618+fJq3LixDMPQvn37bPrudHrFjz/+qI4dO1oP+w0ICNCQIUMyXEd3cvLkSXXp0kXe3t4qXLiwGjdurG3btt1xGrO2uRs3buiDDz5QWFiYSpUqJTc3N5UoUULt2rXT/v377cbffg2ArG4vknTw4EF17txZJUuWlKurq/z9/TVw4MAMx2d0OkT6tSR+++03TZkyRYGBgXJzc7P5POfE+3Pp0iX973//U1hYmDw9PW36mjZtqtatW9sdhVSpUiV16tRJN2/e1LfffmvTN2/ePEnShAkTrJ9FSfrXv/6lsmXLasmSJTbfwQMHDlT58uXt6nr11Vfl4eGhrVu33ndNktSqVSsVLFiQazkA+RChGwBuExERoVu3buk///mPTfvChQvl7e2ttm3bZjjd9evX1bRpU40bN06FChXSkCFD1KZNG23evFmNGzfWypUrM5xu5syZ6tOnj6pWrap+/frpkUce0YwZM9SrVy/rmJCQEIWHh0uSGjdurNGjR1sfRYsWtZnfsGHD9Pbbbys4OFh9+/aV9FcoHDly5B2X28nJSb169dLFixf12Wef2fUfPnxY33zzjZo3b67HH3/8jvO6X926dZNhGFqyZIld361bt7R8+XLrYZvpZs+erZCQEO3YsUOtWrXSoEGD9Pjjj+udd97R008/rRs3btx3XV9++aUkKSgoKMvTHDlyRPXr19e5c+fUvXt3hYeHq2LFiho9erQkyd/f3+b9rFWrls30n3/+uVq3bi0/Pz+9/PLLKleunD7++GO1adPGZtz69evVrFkz7dq1S2FhYXrllVf03HPPKSUlxW5bzgtcXLJ29tv27dtVr149rV69Ws2aNVNkZKT8/f31/vvvq169enY/kGTm9OnTCg4O1vLly1W3bl0NGjRI3t7eevrpp7Vz584MpzFzm7t48aKGDBmilJQUtWzZUkOHDlVISIg2bNigBg0a6LvvvstwuqxuL+lj69atq88//1whISEaMmSIqlevrpkzZyo4OFiXLl3Kcr0DBw7Uu+++qzp16ljnI+Xc+7Nt2zbdvHlT9evXz3JNklSgQAFJttvT9evXtWvXLlWqVEn+/v424y0Wi55++mklJydrz549d52/xWJRgQIFsry9ZlZTOldXV9WuXVs7d+5UcnJylucJIA9w8J52AHC49MPLw8LCDMMwjGrVqhlVq1a19p8+fdpwcXExBg4caBiGkeHh5WPHjjUkGV27drU5XHHfvn2Gq6urUbRoUSMpKcnann7IqpeXl/Hzzz9b269evWpUrFjRcHJysjn8MKuHlwcEBBinTp2ytp87d84oWrSoUaRIESMlJeWO8/vjjz8MFxcXIyQkxG7+r776qiHJWLNmTYav/3fK4uHl6es+PDzc2paUlGR4eHgYgYGBduPXrl1rSDJeffVVa9uhQ4cMFxcXo2bNmsb58+dtxk+YMMGQZLz33ntZqtvf399wdnY2Ro8ebX0MHTrUaNiwoeHk5GR06tTJZj0aRsaHH6cvlyRj1KhRGb6WsnB4uYuLi7F9+3Zr+61bt4yQkBBDkhEbG2ttb9eunSHJOHDggN28/r5OHO1uh5dndOhtRttramqqUa5cOUOSER0dbTN+2LBhhiQjIiIiSzWlf37Gjx9v0/7RRx9Z38fb39+c3OYyWh/Xr183fv/9d7uxP/74o1G4cGEjNDTUpj2728v58+cNT09P47HHHjOOHTtmM69ly5YZkowBAwbYtGe0vaavt8cff9w4fvy4TV9Ovj/p4zdt2pSl8YZhGImJiYaPj4/h7u5u8x79+OOPhiTj2WefzXC69957z5BkLFiw4K6vsWLFCkOS0aFDh/uq6XZDhw41JBlff/11luYJIG9gTzcA/E1ERIQOHTqkXbt2SZIWL16sW7du3fHQ8sWLF6tAgQKaOHGizeGKTzzxhMLDw3X58mWtWbPGbrrBgwerUqVK1uceHh7q0qWL0tLStHfv3mzXPnLkSJUsWdL6vHjx4mrTpo3+/PNPHT58+I7T+vn5qXXr1tq6dauOHDlibb9586Y+/vhjlSxZUq1atcp2TdlVpEgRtW3bVj/99JPdYcbpe21ffPFFa9tHH32kW7du6YMPPlCxYsVsxr/22mt69NFHtWzZsiy/fmpqqsaOHWt9TJs2TTt27FDVqlXVqVMnubq6Znlevr6+evPNN7M8/u/++c9/qmHDhtbnzs7O1qMeMtrb6eHhYdf293WSW7z33nvWozDCw8NVq1YtJScna+LEifLz87vr9Dt27NDRo0fVokULhYWF2fSNGjVK3t7eWrp06V33ON+4cUMrVqxQiRIl9Morr9j09erVSxUqVLCbJqe3ub9zc3PTY489ZtdetWpVNWnSxLrn9++yur18/PHHSkpK0oQJE+z29nbu3FlBQUFavnx5lusdNmyY3YUEc+r9kWQ9pcXHxyfLNfXt21dnzpzRG2+8YfMeJSYmSlKmF29MP3w9fVxmTp48qUGDBsnDw0Nvv/32fdV0u/RlzMnTeAA4HlcvB4C/efHFF/X6669r4cKFqlevnhYtWqQnnnjC7tDfdElJSfrtt99UpUqVDA+9btKkiebNm6cDBw6oW7duNn21a9e2G58+j8uXL2e79vud37/+9S+tXr1a8+fP18SJEyX9dRjq2bNn9cYbb2TrMMr70a1bNy1btkz/+c9/rIdzJyUlae3atapevbpq1qxpHZt++O/GjRsVExNjN68CBQro559/zvJru7m52dwr98qVKzp06JBGjBihdu3aacaMGRo4cGCW5lWzZs1shfS/y+r72blzZ61atUr169fXP//5TzVr1kz/+Mc/sny17y1btuTIfaiHDBlid8pDZqZMmWLX9sEHH2jAgAFZmj793OaMbrtWuHBh1alTR19++aUOHz5sPdw5I4cPH7aeHuLu7m7T5+TkpIYNG+rXX3+1ac/pbS4jBw4c0OTJk7V9+3YlJCTYhezz58/b/MAmZX17Sa9/165dOnr0qN00169f1/nz53X+/PksbUN169a1a8up90eS9fzvrG5bI0aM0LJly9S8eXO98cYbWZomOy5cuKCWLVvq7Nmz+vjjj21+OL3fmry9vSXZXzsCQN5G6AaAv3n00UfVunVrLV++XB06dNDhw4f1wQcfZDo+KSlJUuZ7YdL/ME4fd7u/XxRI+r9z/VJTU7Nd+/3O75lnnlFAQIAWL16s8ePHy8XFRfPnz5fFYlHPnj2zXc+9euaZZ+Tj46Ply5frvffek7Ozsz799FNdu3bN7oeLixcvSpLeeecdU2opXLiw6tWrp1WrVunxxx/XW2+9pZ49e2bp9knZ2TOXkay+nx06dNCaNWs0depUzZkzR7NmzZLFYlGTJk00ZcqUTH8wSrdlyxaNHTv2vmqV/rqwVlaD0enTp+Xr66tr165p165d6tmzp4YOHaoKFSrY7RnNyP187m6XvkezRIkSGfZnNH+zt7lvv/1WTZs2lfTXZ6FChQoqXLiwLBaL1qxZo++//z7DCx5mdXtJr3/WrFl3rCM5OTlLoTujdZRT74/0f0dw3P5jWGZGjhypiRMnqmnTplq1apWcnZ1t+tP3cGe2Jzu9nsz2hF+4cEHNmjXToUOHNHv2bJujbu61ptulX8DNkbdnA5DzOLwcADLQs2dPJSUlqXv37nJ3d1fXrl0zHZv+h+6ZM2cy7E9ISLAZl5tZLBb16dNHCQkJWrt2rU6ePKkvv/xSzZo1U9myZR9YHc7OzurSpYsSEhL01VdfSfrr0HInJyf985//tBmbvl6TkpJkGEamj/tVtGhRVapUSUlJSfrll1+yNM3tpxqYrU2bNtq6dasuXbqkL774Qr169dKWLVvUvHnzux7lMGbMmDuuu6w+ypQpk+26PTw8FBISovXr18tisSgiIiJLV//Oqc9derj6+x0L0mU0f7O3uXfeeUcpKSn66quv9Pnnn2vKlCkaO3asxowZI19f33ue79/r/+GHH+5Y/98PPc9MRtt5Tn4vpt8zO/3HgsyMHDlS48ePV0hIiNauXZvh6RZly5aVk5OT3dEL6dLbMzqtID1wf//995o5c6b+9a9/3bX2rNR0u/RlvP0+4QDyPkI3AGQgLCxMjz32mP744w+1bdtWjzzySKZjPT09VbZsWR05ckR//PGHXX/6Ybt329t4J+l7Ru5l73d29ejRQwUKFND8+fO1cOFCpaWlqXfv3qa/7t+l79H+73//q5MnT2rr1q1q0qSJ3bmu9erVk6RMrzKdk9Kv6Hy3W7BlhZOTkynvZ5EiRdS8eXPNnTtX3bt315kzZ6zXJ8jNKleurP79++vUqVOaPn36Xcc/8cQTkpThYfHpV5/28PC466G/FStWlLu7u/bs2WO3JzUtLS3DWzuZvc0dPXpU3t7eatSokU371atX7a5zcC/S64+Njb3veWUmp94fSdbDz+90XYr0cNu4cWOtX78+0z3FHh4eqlu3rg4fPmx3W0LDMLRp0yYVKlRIderUsem7PXB/8MEHevnll+9ad1Zrul36Mt7tkHsAeQuhGwAy4OzsrDVr1mj16tWaMGHCXceHh4fr5s2bGjFihM0eroMHDyoqKkpeXl6Z3m4sK9LP8zt58uQ9zyOrfHx81LZtW0VHR2v27NkqXrz4fdV+r4KCghQYGKjVq1fro48+kmEYdoeWS9LLL78sFxcXDRw4UCdOnLDrv3z5cob3Ns6u1atXKz4+Xo888oiqVat23/Pz9vbOsYslbdu2LcMAn7739u/nKudWw4cPl4eHh9577727HnbcsGFDlStXTl988YX1aIh048eP14ULF9SlS5e7nlPv5uamjh076uzZs3bnmc+fPz/DoxrM3ub8/f116dIlHTp0yNqWmpqqV199VefOnbvn+abr0aOHihQpojfffNPmNdJdvXr1vn9QyKn3R/rrVomSMv3xaNSoURo/frz+8Y9/ZCnc9unTR5Lsvq8/+ugj/fbbb+ratavNHumLFy8qNDRU33//vd5///0sXXcguzWl27Vrl0qWLJnhnnYAeRfndANAJurUqWO3tyMzr732mtavX6///Oc/iouLU7NmzXT27FmtWLFCt27d0rx581SkSJF7rqVy5cry8/PT8uXL5ebmpscff1wWi0UDBw7M9NzD+9G3b1+tXLlSZ86c0SuvvHJPFwM7ffq0unfvnmFf8eLF9d577911Ht26ddOIESM0efJkFSxYUO3bt7cbU61aNX344Yfq16+fKlWqpJYtW6pcuXL6888/9dtvv2nr1q3q3r275syZk6W6b926pTFjxlifJycn69ChQ4qOjpbFYtEHH3xwXxdHS9e0aVN98sknatu2rZ544gk5OzvrueeeU40aNbI9r0GDBunUqVNq1KiRypQpI4vFou3bt2v37t2qX7++3R7T3MrHx0f9+vXT1KlTNW3aNOv9zDPi5OSkqKgohYWFqWXLlurQoYP8/f0VGxurLVu2qFy5ctaLAd7NxIkTFRMTo7feekvbt2/XE088obi4OG3YsEHPPPOM9R7t6XJ6m/u7gQMH6ssvv1SjRo3UsWNHubu7a8uWLfrjjz8UEhJy3xe9S7+6eocOHVSzZk01b95clStXVkpKio4dO6atW7eqQYMGio6OvufXyMn3p0aNGipbtqw2bdpk1xcVFaW3335bLi4uqlu3rv7973/bjQkJCbG5oFt4eLhWrFihZcuWKT4+Xo0bN9aRI0e0atUqBQQEaPz48TbTt2vXTgcOHFDlypV18eJFm++HdLdfRPBeapL+OsIhPj5e/fr1u/tKAZC3mHUvMgDIK/5+n+67yeg+3YZhGFeuXDFGjhxpVKxY0Xpv7hYtWhjffPON3diM7u2cLv2eu4sWLbJp37lzp9G4cWOjSJEi1nsHx8fHG4bxf/fLTX9+t9e6232/09LSjNKlSxuSjLi4uEzWRObS68vskb7+MrpP9+1OnDhhODk5GZKMLl263PE1d+/ebXTu3Nnw8/MzChQoYBQvXtwICgoyhg8fnuVl8Pf3t6vVxcXFKFmypNG+fXtjx44ddtPc6T7dmS2XYfx1//eOHTsaxYsXty5j+nue2TZgGBm/d8uXLzc6duxolCtXzihYsKDh5eVl1KxZ05g0aZLx559/ZmnZH5TM7tOdLiEhwboMFy9eNAzjztvrwYMHjRdeeMEoXry4UaBAAcPf398YPHiwce7cuWzVdfz4caNTp05G0aJFjYIFCxr/+Mc/jK1bt97xs5oT21yjRo0MScaFCxds2j/99FMjKCjIKFiwoFG8eHGjY8eOxtGjRzP8rGd3e0n3888/Gz179jT8/f0NV1dX45FHHjGqV69uDBo0yNi9e7fNWN3hPt0Zfe+ky6n3Z9KkSYYkY9euXTbt6e/PnR4ZLfv169eNMWPGGOXKlTNcXV0NX19fo1evXkZCQoLd2Iy+F/7+uH0d3GtNY8aMMSQZBw4cyNa6AZD7WQwjB64uAwDIV06fPq3SpUsrODhY27Ztc3Q5QL5VuXJl/fbbb7p27dodr2r9sLt48aLKli2rDh06aN68eY4uJ8fdunVLFSpUUEBAgL7++mtHlwMgh3FONwDAzvTp03Xr1i0OcwRMFBcXp19++UW1a9cmcN+Ft7e3RowYocWLF9tdAC0/SF+urJx2AyDv4ZxuAICkv+5bO3v2bB0/flzz589XYGCgOnbs6OiygHxn+fLl2rp1qz755BMZhqHIyEhHl5QnDB48WCkpKTpx4kSWb2eWV1gsFs2bN09BQUGOLgWACTi8HAAgSTp27JgCAgLk7u6u+vXra86cOVm6nQ+A7Gnbtq02btyoKlWq6NVXX7W79zwAIH8hdAMAAAAAYBLO6QYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4TGYahpKQkGYbh6FIAAAAAAA5A6DbRn3/+KS8vL/3555+OLgUAAAAA4ACEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AaQJalphqNLuC95vX4AAADkTS6OLgBA3uDsZNHE1ft18vwVR5eSbaWKF9bw559wdBkAAAB4CBG6AWTZyfNXdCQhydFlAAAAAHkGh5cDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAgR6WmGY4u4b7k9foBAEDu4uLIF9+2bZv+/e9/a+/evTp9+rRWr16ttm3b2oyJi4vT66+/rq1bt+rWrVsKDAzUZ599ptKlS2c635UrV2rkyJE6duyYKlSooEmTJqlly5bWfsMwNHr0aM2bN0+XL19Ww4YNNXv2bFWoUME65uLFixo4cKDWrl0rJycntW/fXu+//74KFy6c4+sBQN6UmmbI2cni6DLuixnL4Oxk0cTV+3Xy/JUcne+DUKp4YQ1//glHlwEAAPIRh4bu5ORk1axZUxEREWrXrp1d/9GjR9WoUSP17NlTY8eOlaenpw4dOiR3d/dM5/ntt9+qS5cumjBhgp599lktXbpUbdu21b59+1StWjVJ0uTJkzVjxgwtXrxYAQEBGjlypMLCwvTTTz9Z5921a1edPn1amzZt0s2bN9WjRw/16dNHS5cuNWdlAMhz8nK4lMwNmCfPX9GRhCRT5g0AAJCXWAzDyBXH0VksFrs93Z07d1aBAgX0n//8J8vz6dSpk5KTk7Vu3TprW/369VWrVi3NmTNHhmHIz89Pr7zyil599VVJUmJionx8fBQVFaXOnTsrLi5OgYGB+u6771SnTh1JUnR0tFq2bKnff/9dfn5+WaolKSlJXl5eSkxMlKenZ5aXAcit+s/7Jk8GqfK+nprV+x+mzDuvrhOJ9ZIRM9cJAAB4OOXac7rT0tK0fv16VaxYUWFhYSpRooTq1aunNWvW3HG62NhYhYaG2rSFhYUpNjZWkhQfH6+EhASbMV5eXqpXr551TGxsrIoWLWoN3JIUGhoqJycn7dq1K9PXTklJUVJSks0DAAAAAPDwyrWh++zZs7py5YomTpyo5s2b68svv9Tzzz+vdu3aaevWrZlOl5CQIB8fH5s2Hx8fJSQkWPvT2+40pkSJEjb9Li4u8vb2to7JyIQJE+Tl5WV9lCpVKusLDAAAAADId3Jt6E5LS5MktWnTRkOHDlWtWrU0fPhwPfvss5ozZ46Dq8vYiBEjlJiYaH2cPHnS0SUBAAAAABwo14bu4sWLy8XFRYGBgTbtVapU0YkTJzKdztfXV2fOnLFpO3PmjHx9fa396W13GnP27Fmb/lu3bunixYvWMRlxc3OTp6enzQMAAAAA8PDKtaHb1dVVTz75pA4fPmzT/ssvv8jf3z/T6YKDgxUTE2PTtmnTJgUHB0uSAgIC5OvrazMmKSlJu3btso4JDg7W5cuXtXfvXuuYr7/+WmlpaapXr959LxsAAAAA4OHg0FuGXblyRUeOHLE+j4+P14EDB+Tt7a3SpUtr2LBh6tSpk5566ik1adJE0dHRWrt2rbZs2ZLpPAcPHqzGjRtrypQpatWqlZYvX649e/Zo7ty5kv66SvqQIUM0fvx4VahQwXrLMD8/P+uV06tUqaLmzZurd+/emjNnjm7evKkBAwaoc+fOWb5yeV7BfYYBAAAAwDwODd179uxRkyZNrM8jIyMlSeHh4YqKitLzzz+vOXPmaMKECRo0aJAqVaqkzz77TI0aNbJO0717dx07dswaxBs0aKClS5fqrbfe0htvvKEKFSpozZo11nt0S9Jrr72m5ORk9enTR5cvX1ajRo0UHR1tc//vJUuWaMCAAWrWrJmcnJzUvn17zZgxw+Q18uBxn2EAAAAAMI9DQ3dISIjudpvwiIgIRUREZNofHx9vE9wlqUOHDurQoUOm01gsFo0bN07jxo3LdIy3t7eWLl16x9ryi5Pnr+TJ++kCAAAAQG7n0NB9vxITE3X06FGtX7/e0aUAAAAAAGAnT4duLy8v/f77744uAwAAAACADOXaq5cDAAAAAJDXEboBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELqBDKSmGY4u4b7k9foBAACA/MLF0QUAuZGzk0UTV+/XyfNXHF1KtpUqXljDn3/C0WUAAAAAEKEbyNTJ81d0JCHJ0WUAAAAAyMM4vBwAAAAAAJMQugEAAAAAMAmhGwAAPHD54YKP+WEZAADm45xuAADwwOXlC1ZKXLQSAJB1hG4AAEyWmmbI2cni6DLumVn1c8FKADAX///kDoRuAABMlpf36rJHFwDyLv7/yR0I3QAAPADs1QUAOAL//zgeF1IDAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAIBcIjXNcHQJ9yWv1w8AZnBx5Itv27ZN//73v7V3716dPn1aq1evVtu2bTMc27dvX3300UeaNm2ahgwZcsf5zpo1S//+97+VkJCgmjVr6oMPPlDdunWt/devX9crr7yi5cuXKyUlRWFhYfrwww/l4+NjHXPixAn169dPmzdvVuHChRUeHq4JEybIxcWhqwwAAORjzk4WTVy9XyfPX3F0KdlWqnhhDX/+CUeXAQC5jkMTZHJysmrWrKmIiAi1a9cu03GrV6/Wzp075efnd9d5rlixQpGRkZozZ47q1aun6dOnKywsTIcPH1aJEiUkSUOHDtX69eu1cuVKeXl5acCAAWrXrp127NghSUpNTVWrVq3k6+urb7/9VqdPn9ZLL72kAgUK6N13382ZhQcAAMjAyfNXdCQhydFlAHlOapohZyeLo8u4Z3m9fmTOoaG7RYsWatGixR3H/PHHHxo4cKA2btyoVq1a3XWeU6dOVe/evdWjRw9J0pw5c7R+/XotXLhQw4cPV2JiohYsWKClS5eqadOmkqRFixapSpUq2rlzp+rXr68vv/xSP/30k7766iv5+PioVq1aevvtt/X6669rzJgxcnV1vf+FBwAAAJBjOFIEuVWuPlY6LS1N3bp107Bhw1S1atW7jr9x44b27t2rESNGWNucnJwUGhqq2NhYSdLevXt18+ZNhYaGWsdUrlxZpUuXVmxsrOrXr6/Y2FhVr17d5nDzsLAw9evXT4cOHdITT/CBAAAAAHIbjhRBbpSrQ/ekSZPk4uKiQYMGZWn8+fPnlZqaahOWJcnHx0c///yzJCkhIUGurq4qWrSo3ZiEhATrmIzmkd6XmZSUFKWkpFifJyXxgQcAAACAh1muvXr53r179f777ysqKkoWS944t2HChAny8vKyPkqVKuXokgAAAAAADpRrQ/c333yjs2fPqnTp0nJxcZGLi4uOHz+uV155RWXKlMlwmuLFi8vZ2VlnzpyxaT9z5ox8fX0lSb6+vrpx44YuX758xzEZzSO9LzMjRoxQYmKi9XHy5MnsLDIAAAAAIJ/JtaG7W7duOnjwoA4cOGB9+Pn5adiwYdq4cWOG07i6uqp27dqKiYmxtqWlpSkmJkbBwcGSpNq1a6tAgQI2Yw4fPqwTJ05YxwQHB+uHH37Q2bNnrWM2bdokT09PBQYGZlqzm5ubPD09bR4AAAAAgIeXQ8/pvnLlio4cOWJ9Hh8frwMHDsjb21ulS5dWsWLFbMYXKFBAvr6+qlSpUqbzjIyMVHh4uOrUqaO6detq+vTpSk5Otl7N3MvLSz179lRkZKS8vb3l6empgQMHKjg4WPXr15ckPfPMMwoMDFS3bt00efJkJSQk6K233lL//v3l5uZmwpoAAAAAAORHDg3de/bsUZMmTazPIyMjJUnh4eGKiorK0jxCQkJUpkwZ6/hOnTrp3LlzGjVqlBISElSrVi1FR0fbXBht2rRpcnJyUvv27ZWSkqKwsDB9+OGH1n5nZ2etW7dO/fr1U3BwsAoVKqTw8HCNGzfu/hcaAAAAAPDQcGjoDgkJkWEYWR5/7Ngxu7b4+Hh1797dpm3AgAEaMGBApvNxd3fXrFmzNGvWrEzH+Pv7a8OGDVmuDQAAAACAv8u153RnxaFDh+Tl5aWXXnrJ0aUAAAAAAGAnV9+n+26qVq2qgwcPOroMAAAAAAAylKf3dAMAAAAAkJsRugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAADIQ1LTDEeXcN/ywzIAWZWn79MNAAAAPGycnSyauHq/Tp6/4uhS7kmp4oU1/PknHF0G8MAQugEAAIA85uT5KzqSkOToMgBkAYeXAwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAHKtvH4/57xeP4D7xy3DAAAAkGvl5XtScz9qABKhGwAAALkc96QGkJdxeDkAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBKHhu5t27apdevW8vPzk8Vi0Zo1a6x9N2/e1Ouvv67q1aurUKFC8vPz00svvaRTp07ddb6zZs1SmTJl5O7urnr16mn37t02/devX1f//v1VrFgxFS5cWO3bt9eZM2dsxpw4cUKtWrVSwYIFVaJECQ0bNky3bt3KkeUGAAAAADwcHBq6k5OTVbNmTc2aNcuu7+rVq9q3b59Gjhypffv2adWqVTp8+LCee+65O85zxYoVioyM1OjRo7Vv3z7VrFlTYWFhOnv2rHXM0KFDtXbtWq1cuVJbt27VqVOn1K5dO2t/amqqWrVqpRs3bujbb7/V4sWLFRUVpVGjRuXcwgMAAAAA8j0XR754ixYt1KJFiwz7vLy8tGnTJpu2mTNnqm7dujpx4oRKly6d4XRTp05V79691aNHD0nSnDlztH79ei1cuFDDhw9XYmKiFixYoKVLl6pp06aSpEWLFqlKlSrauXOn6tevry+//FI//fSTvvrqK/n4+KhWrVp6++239frrr2vMmDFydXXNwbUAAAAAAMiv8tQ53YmJibJYLCpatGiG/Tdu3NDevXsVGhpqbXNyclJoaKhiY2MlSXv37tXNmzdtxlSuXFmlS5e2jomNjVX16tXl4+NjHRMWFqakpCQdOnTIhCUDAAAAAORHDt3TnR3Xr1/X66+/ri5dusjT0zPDMefPn1dqaqpNWJYkHx8f/fzzz5KkhIQEubq62gV3Hx8fJSQkWMdkNI/0vsykpKQoJSXF+jwpKSlrCwcAAAAAyJfyxJ7umzdvqmPHjjIMQ7Nnz3Z0OZmaMGGCvLy8rI9SpUo5uiQAAAAAgAPl+tCdHriPHz+uTZs2ZbqXW5KKFy8uZ2dnuyuRnzlzRr6+vpIkX19f3bhxQ5cvX77jmIzmkd6XmREjRigxMdH6OHnyZJaXEwAAAACQ/+Tq0J0euH/99Vd99dVXKlas2B3Hu7q6qnbt2oqJibG2paWlKSYmRsHBwZKk2rVrq0CBAjZjDh8+rBMnTljHBAcH64cffrC54nl64A8MDMz09d3c3OTp6WnzAAAAAAA8vBx6TveVK1d05MgR6/P4+HgdOHBA3t7eKlmypF544QXt27dP69atU2pqqvV8am9v70yvIB4ZGanw8HDVqVNHdevW1fTp05WcnGy9mrmXl5d69uypyMhIeXt7y9PTUwMHDlRwcLDq168vSXrmmWcUGBiobt26afLkyUpISNBbb72l/v37y83NzeS1AgAAAADILxwauvfs2aMmTZpYn0dGRkqSwsPDNWbMGH3++eeSpFq1atlMt3nzZoWEhEiSQkJCVKZMGUVFRUmSOnXqpHPnzmnUqFFKSEhQrVq1FB0dbXNhtGnTpsnJyUnt27dXSkqKwsLC9OGHH1r7nZ2dtW7dOvXr10/BwcEqVKiQwsPDNW7cOBPWAgAAAAAgv3Jo6A4JCZFhGJn236kvXXx8vLp3727TNmDAAA0YMCDTadzd3TVr1izNmjUr0zH+/v7asGHDXV8fAAAAAIDM5Opzuu/m0KFD8vLy0ksvveToUgAAAAAAsJNn7tOdkapVq+rgwYOOLgMAAAAAgAzl6T3dAAAAAADkZvcdulNTU3XgwAFdunQpJ+oBAAAAACDfyHboHjJkiBYsWCDpr8DduHFjBQUFqVSpUtqyZUtO1wcAAAAAQJ6V7dD96aefqmbNmpKktWvXKj4+Xj///LOGDh2qN998M8cLBAAAAAAgr8p26D5//rx8fX0lSRs2bFCHDh1UsWJFRURE6IcffsjxAgEAAAAAyKuyHbp9fHz0008/KTU1VdHR0Xr66aclSVevXpWzs3OOFwgAAAAAQF6V7VuG9ejRQx07dlTJkiVlsVgUGhoqSdq1a5cqV66c4wUCAAAAAJBXZTt0jxkzRtWqVdPJkyfVoUMHubm5SZKcnZ01fPjwHC8QAAAAAIC8KtuhW5JeeOEFu7bw8PD7LgYAAAAAgPzknkJ3TEyMYmJidPbsWaWlpdn0LVy4MEcKAwAAAAAgr8t26B47dqzGjRunOnXqWM/rBgAAAAAA9rIduufMmaOoqCh169bNjHoAAAAAAMg3sn3LsBs3bqhBgwZm1AIAAAAAQL6S7dDdq1cvLV261IxaAAAAAADIV7J0eHlkZKT132lpaZo7d66++uor1ahRQwUKFLAZO3Xq1JytEAAAAACAPCpLoXv//v02z2vVqiVJ+vHHH3O8IAAAAAAA8osshe7NmzebXQcAAAAAAPlOts/pjoiI0J9//mnXnpycrIiIiBwpCgAAAACA/CDboXvx4sW6du2aXfu1a9f08ccf50hRAAAAAADkB1m+T3dSUpIMw5BhGPrzzz/l7u5u7UtNTdWGDRtUokQJU4oEAAAAACAvynLoLlq0qCwWiywWiypWrGjXb7FYNHbs2BwtDgAAAACAvCzLoXvz5s0yDENNmzbVZ599Jm9vb2ufq6ur/P395efnZ0qRAAAAAADkRVkO3Y0bN9atW7cUHh6uOnXqqFSpUmbWBQAAAABAnpetC6m5uLjo008/VWpqqln1AAAAAACQb2T76uVNmzbV1q1bzagFAAAAAIB8JcuHl6dr0aKFhg8frh9++EG1a9dWoUKFbPqfe+65HCsOAAAAAIC8LNuh++WXX5YkTZ061a7PYrFw6DkAAAAAAP9ftkN3WlqaGXUAAAAAAJDvZPucbgAAAAAAkDVZ2tM9Y8YM9enTR+7u7poxY8Ydxw4aNChHCgMAAAAAIK/LUuieNm2aunbtKnd3d02bNi3TcRaLhdANAAAAAMD/l6XDy+Pj41WsWDHrvzN7/Pbbb9l68W3btql169by8/OTxWLRmjVrbPoNw9CoUaNUsmRJeXh4KDQ0VL/++utd5ztr1iyVKVNG7u7uqlevnnbv3m3Tf/36dfXv31/FihVT4cKF1b59e505c8ZmzIkTJ9SqVSsVLFhQJUqU0LBhw3Tr1q1sLR8AAAAA4OF2X+d0G4YhwzDuefrk5GTVrFlTs2bNyrB/8uTJmjFjhubMmaNdu3apUKFCCgsL0/Xr1zOd54oVKxQZGanRo0dr3759qlmzpsLCwnT27FnrmKFDh2rt2rVauXKltm7dqlOnTqldu3bW/tTUVLVq1Uo3btzQt99+q8WLFysqKkqjRo2652UFAAAAADx87il0L1iwQNWqVZO7u7vc3d1VrVo1zZ8/P9vzadGihcaPH6/nn3/ers8wDE2fPl1vvfWW2rRpoxo1aujjjz/WqVOn7PaI327q1Knq3bu3evToocDAQM2ZM0cFCxbUwoULJUmJiYlasGCBpk6dqqZNm6p27dpatGiRvv32W+3cuVOS9OWXX+qnn37Sf//7X9WqVUstWrTQ22+/rVmzZunGjRvZXk4AAAAAwMMp26F71KhRGjx4sFq3bq2VK1dq5cqVat26tYYOHZqje4Lj4+OVkJCg0NBQa5uXl5fq1aun2NjYDKe5ceOG9u7dazONk5OTQkNDrdPs3btXN2/etBlTuXJllS5d2jomNjZW1atXl4+Pj3VMWFiYkpKSdOjQoRxbRgAAAABA/pbt+3TPnj1b8+bNU5cuXaxtzz33nGrUqKGBAwdq3LhxOVJYQkKCJNkE3/Tn6X1/d/78eaWmpmY4zc8//2ydr6urq4oWLZrpfBMSEjKcx+11ZSQlJUUpKSnW50lJSZmOBQAAAADkf9ne033z5k3VqVPHrr127doP/YXGJkyYIC8vL+ujVKlSji4JAAAAAOBA2Q7d3bp10+zZs+3a586dq65du+ZIUZLk6+srSXZXFT9z5oy17++KFy8uZ2fnO07j6+urGzdu6PLly3cck9E8bq8rIyNGjFBiYqL1cfLkybssJQAAAAAgP8tS6I6MjLQ+LBaL5s+fr2rVqqlXr17q1auXqlevrnnz5snJ6b4uhm4jICBAvr6+iomJsbYlJSVp165dCg4OznAaV1dX1a5d22aatLQ0xcTEWKepXbu2ChQoYDPm8OHDOnHihHVMcHCwfvjhB5srnm/atEmenp4KDAzMtGY3Nzd5enraPAAAAAAAD68sndO9f/9+m+e1a9eWJB09elTSX3uYixcvnu2LjF25ckVHjhyxPo+Pj9eBAwfk7e2t0qVLa8iQIRo/frwqVKiggIAAjRw5Un5+fmrbtm2m84yMjFR4eLjq1KmjunXravr06UpOTlaPHj0k/XUxtp49eyoyMlLe3t7y9PTUwIEDFRwcrPr160uSnnnmGQUGBqpbt26aPHmyEhIS9NZbb6l///5yc3PL1jICAAAAAB5eWQrdmzdvNuXF9+zZoyZNmlifR0ZGSpLCw8MVFRWl1157TcnJyerTp48uX76sRo0aKTo6Wu7u7tZpQkJCVKZMGUVFRUmSOnXqpHPnzmnUqFFKSEhQrVq1FB0dbXNhtGnTpsnJyUnt27dXSkqKwsLC9OGHH1r7nZ2dtW7dOvXr10/BwcEqVKiQwsPDc+wicQAAAACAh0O2r16ek0JCQmQYRqb9FotF48aNu2PYjY+PV/fu3W3aBgwYoAEDBmQ6jbu7u2bNmqVZs2ZlOsbf318bNmzIvHgAAAAAAO4i507CdoBDhw7Jy8tLL730kqNLAQAAAADAjkP3dN+vqlWr6uDBg44uAwAAAACADOXpPd0AAAAAAORmWQrdQUFBunTpkiRp3Lhxunr1qqlFAQAAAACQH2QpdMfFxSk5OVmSNHbsWF25csXUogAAAAAAyA+ydE53rVq11KNHDzVq1EiGYei9995T4cKFMxw7atSoHC0QAAAAAIC8KkuhOyoqSqNHj9a6detksVj0xRdfyMXFflKLxULoBgAAAADg/8tS6K5UqZKWL18uSXJyclJMTIxKlChhamEAAAAAAOR12b5lWFpamhl1AAAAAACQ79zTfbqPHj2q6dOnKy4uTpIUGBiowYMHq1y5cjlaHAAAAAAAeVm279O9ceNGBQYGavfu3apRo4Zq1KihXbt2qWrVqtq0aZMZNQIAAAAAkCdle0/38OHDNXToUE2cONGu/fXXX9fTTz+dY8UBAAAAAJCXZXtPd1xcnHr27GnXHhERoZ9++ilHigIAAAAAID/Iduh+9NFHdeDAAbv2AwcOcEVzAAAAAABuk+3Dy3v37q0+ffrot99+U4MGDSRJO3bs0KRJkxQZGZnjBQIAAAAAkFdlO3SPHDlSRYoU0ZQpUzRixAhJkp+fn8aMGaNBgwbleIEAAAAAAORV2Q7dFotFQ4cO1dChQ/Xnn39KkooUKZLjhQEAAAAAkNfd03260xG2AQAAAADIXLYvpAYAAAAAALKG0A0AAAAAgEkI3QAAAAAAmCRbofvmzZtq1qyZfv31V7PqAQAAAAAg38hW6C5QoIAOHjxoVi0AAAAAAOQr2T68/MUXX9SCBQvMqAUAAAAAgHwl27cMu3XrlhYuXKivvvpKtWvXVqFChWz6p06dmmPFAQAAAACQl2U7dP/4448KCgqSJP3yyy82fRaLJWeqAgAAAAAgH8h26N68ebMZdQAAAAAAkO/c8y3Djhw5oo0bN+ratWuSJMMwcqwoAAAAAADyg2yH7gsXLqhZs2aqWLGiWrZsqdOnT0uSevbsqVdeeSXHCwQAAAAAIK/KdugeOnSoChQooBMnTqhgwYLW9k6dOik6OjpHiwMAAAAAIC/L9jndX375pTZu3KjHH3/cpr1ChQo6fvx4jhUGAAAAAEBel+093cnJyTZ7uNNdvHhRbm5uOVIUAAAAAAD5QbZD9z/+8Q99/PHH1ucWi0VpaWmaPHmymjRpkqPFAQAAAACQl2X78PLJkyerWbNm2rNnj27cuKHXXntNhw4d0sWLF7Vjxw4zagQAAAAAIE/K9p7uatWq6ZdfflGjRo3Upk0bJScnq127dtq/f7/KlSuXo8WlpqZq5MiRCggIkIeHh8qVK6e33377rrcn27Jli4KCguTm5qby5csrKirKbsysWbNUpkwZubu7q169etq9e7dN//Xr19W/f38VK1ZMhQsXVvv27XXmzJmcXDwAAAAAQD6X7T3dkuTl5aU333wzp2uxM2nSJM2ePVuLFy9W1apVtWfPHvXo0UNeXl4aNGhQhtPEx8erVatW6tu3r5YsWaKYmBj16tVLJUuWVFhYmCRpxYoVioyM1Jw5c1SvXj1Nnz5dYWFhOnz4sEqUKCHpr6u0r1+/XitXrpSXl5cGDBigdu3asTcfAAAAAJBl9xS6L126pAULFiguLk6SFBgYqB49esjb2ztHi/v222/Vpk0btWrVSpJUpkwZLVu2zG6v9O3mzJmjgIAATZkyRZJUpUoVbd++XdOmTbOG7qlTp6p3797q0aOHdZr169dr4cKFGj58uBITE7VgwQItXbpUTZs2lSQtWrRIVapU0c6dO1W/fv0cXU4AAAAAQP6U7cPLt23bpjJlymjGjBm6dOmSLl26pBkzZiggIEDbtm3L0eIaNGigmJgY/fLLL5Kk77//Xtu3b1eLFi0ynSY2NlahoaE2bWFhYYqNjZUk3bhxQ3v37rUZ4+TkpNDQUOuYvXv36ubNmzZjKleurNKlS1vHZCQlJUVJSUk2DwAAAADAwyvbe7r79++vTp06afbs2XJ2dpb017nXL7/8svr3768ffvghx4obPny4kpKSVLlyZTk7Oys1NVXvvPOOunbtmuk0CQkJ8vHxsWnz8fFRUlKSrl27pkuXLik1NTXDMT///LN1Hq6uripatKjdmISEhExfe8KECRo7dmw2lxIAAAAAkF9le0/3kSNH9Morr1gDtyQ5OzsrMjJSR44cydHiPvnkEy1ZskRLly7Vvn37tHjxYr333ntavHhxjr5OThkxYoQSExOtj5MnTzq6JAAAAACAA2V7T3dQUJDi4uJUqVIlm/a4uDjVrFkzxwqTpGHDhmn48OHq3LmzJKl69eo6fvy4JkyYoPDw8Ayn8fX1tbvK+JkzZ+Tp6SkPDw85OzvL2dk5wzG+vr7Wedy4cUOXL1+22dt9+5iMuLm5yc3N7V4WFQAAAACQD2UpdB88eND670GDBmnw4ME6cuSI9YJiO3fu1KxZszRx4sQcLe7q1atycrLdGe/s7Ky0tLRMpwkODtaGDRts2jZt2qTg4GBJkqurq2rXrq2YmBi1bdtWkpSWlqaYmBgNGDBAklS7dm0VKFBAMTExat++vSTp8OHDOnHihHU+AAAAAADcTZZCd61atWSxWGzuj/3aa6/ZjfvnP/+pTp065VhxrVu31jvvvKPSpUuratWq2r9/v6ZOnaqIiIhMp+nbt69mzpyp1157TREREfr666/1ySefaP369dYxkZGRCg8PV506dVS3bl1Nnz5dycnJ1quZe3l5qWfPnoqMjJS3t7c8PT01cOBABQcHc+VyAAAAAECWZSl0x8fHm11Hhj744AONHDlSL7/8ss6ePSs/Pz/961//0qhRo6xjxowZo6ioKB07dkySFBAQoPXr12vo0KF6//339fjjj2v+/PnW24VJUqdOnXTu3DmNGjVKCQkJqlWrlqKjo20urjZt2jQ5OTmpffv2SklJUVhYmD788MMHtuwAAAAAgLwvS6Hb39/f7DoyVKRIEU2fPl3Tp0/PdEx8fLxCQkJs2kJCQrR///47znvAgAHWw8kz4u7urlmzZmnWrFnZKRkAAAAAAKtsX0hNkk6dOqXt27fr7NmzdudXDxo0KEcKywrDMLRlyxZt3779gb0mAAAAAABZle3QHRUVpX/9619ydXVVsWLFZLFYrH0Wi+WBhm6LxaLjx48/sNcDAAAAACA7sh26R44cqVGjRmnEiBF2VxYHAAAAAAD/J9up+erVq+rcuTOBGwAAAACAu8h2cu7Zs6dWrlxpRi0AAAAAAOQr2T68fMKECXr22WcVHR2t6tWrq0CBAjb9U6dOzbHiAAAAAADIy+4pdG/cuFGVKlWSJLsLqQEAAAAAgL9kO3RPmTJFCxcuVPfu3U0oBwAAAACA/CPb53S7ubmpYcOGZtQCAAAAAEC+ku3QPXjwYH3wwQdm1AIAAAAAQL6S7cPLd+/era+//lrr1q1T1apV7S6ktmrVqhwrDgAAAACAvCzbobto0aJq166dGbUAAAAAAJCvZDt0L1q0yIw6AAAAAADId7J9TjcAAAAAAMiabO/pDggIuOP9uH/77bf7KggAAAAAgPwi26F7yJAhNs9v3ryp/fv3Kzo6WsOGDcupugAAAAAAyPOyHboHDx6cYfusWbO0Z8+e+y4IAAAAAID8IsfO6W7RooU+++yznJodAAAAAAB5Xo6F7k8//VTe3t45NTsAAAAAAPK8bB9e/sQTT9hcSM0wDCUkJOjcuXP68MMPc7Q4AAAAAADysmyH7rZt29o8d3Jy0qOPPqqQkBBVrlw5p+oCAAAAACDPy3boHj16tBl1AAAAAACQ7+TYOd0AAAAAAMBWlvd0Ozk52ZzLnRGLxaJbt27dd1EAAAAAAOQHWQ7dq1evzrQvNjZWM2bMUFpaWo4UBQAAAABAfpDl0N2mTRu7tsOHD2v48OFau3atunbtqnHjxuVocQAAAAAA5GX3dE73qVOn1Lt3b1WvXl23bt3SgQMHtHjxYvn7++d0fQAAAAAA5FnZCt2JiYl6/fXXVb58eR06dEgxMTFau3atqlWrZlZ9AAAAAADkWVk+vHzy5MmaNGmSfH19tWzZsgwPNwcAAAAAAP8ny6F7+PDh8vDwUPny5bV48WItXrw4w3GrVq3KseIAAAAAAMjLshy6X3rppbveMgwAAAAAAPyfLIfuqKgoE8sAAAAAACD/uaerlwMAAAAAgLvL9aH7jz/+0IsvvqhixYrJw8ND1atX1549e+44zZYtWxQUFCQ3NzeVL18+w730s2bNUpkyZeTu7q569epp9+7dNv3Xr19X//79VaxYMRUuXFjt27fXmTNncnLRAAAAAAD5XK4O3ZcuXVLDhg1VoEABffHFF/rpp580ZcoUPfLII5lOEx8fr1atWqlJkyY6cOCAhgwZol69emnjxo3WMStWrFBkZKRGjx6tffv2qWbNmgoLC9PZs2etY4YOHaq1a9dq5cqV2rp1q06dOqV27dqZurwAAAAAgPwly+d0O8KkSZNUqlQpLVq0yNoWEBBwx2nmzJmjgIAATZkyRZJUpUoVbd++XdOmTVNYWJgkaerUqerdu7d69OhhnWb9+vVauHChhg8frsTERC1YsEBLly5V06ZNJUmLFi1SlSpVtHPnTtWvX9+MxQUAAAAA5DO5ek/3559/rjp16qhDhw4qUaKEnnjiCc2bN++O08TGxio0NNSmLSwsTLGxsZKkGzduaO/evTZjnJycFBoaah2zd+9e3bx502ZM5cqVVbp0aeuYjKSkpCgpKcnmAQAAAAB4eOXq0P3bb79p9uzZqlChgjZu3Kh+/fpp0KBBmd4jXJISEhLk4+Nj0+bj46OkpCRdu3ZN58+fV2pqaoZjEhISrPNwdXVV0aJFMx2TkQkTJsjLy8v6KFWqVDaXGAAAAACQn+Tq0J2WlqagoCC9++67euKJJ9SnTx/17t1bc+bMcXRpGRoxYoQSExOtj5MnTzq6JAAAAACAA+Xq0F2yZEkFBgbatFWpUkUnTpzIdBpfX1+7q4yfOXNGnp6e8vDwUPHixeXs7JzhGF9fX+s8bty4ocuXL2c6JiNubm7y9PS0eQAAAAAAHl65OnQ3bNhQhw8ftmn75Zdf5O/vn+k0wcHBiomJsWnbtGmTgoODJUmurq6qXbu2zZi0tDTFxMRYx9SuXVsFChSwGXP48GGdOHHCOgYAAAAAgLvJ1VcvHzp0qBo0aKB3331XHTt21O7duzV37lzNnTs302n69u2rmTNn6rXXXlNERIS+/vprffLJJ1q/fr11TGRkpMLDw1WnTh3VrVtX06dPV3JysvVq5l5eXurZs6ciIyPl7e0tT09PDRw4UMHBwVy5HAAAAACQZbk6dD/55JNavXq1RowYoXHjxikgIEDTp09X165drWPGjBmjqKgoHTt2TNJftxRbv369hg4dqvfff1+PP/645s+fb71dmCR16tRJ586d06hRo5SQkKBatWopOjra5uJq06ZNk5OTk9q3b6+UlBSFhYXpww8/fGDLDgAAAADI+3J16JakZ599Vs8++2ym/fHx8QoJCbFpCwkJ0f79++843wEDBmjAgAGZ9ru7u2vWrFmaNWtWtuoFAAAAACBdrg/dd2IYhrZs2aLt27c7uhQAAAAAAOzk6dBtsVh0/PhxR5cBAAAAAECGcvXVywEAAAAAyMsI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEnyVOieOHGiLBaLhgwZcsdxK1euVOXKleXu7q7q1atrw4YNNv2GYWjUqFEqWbKkPDw8FBoaql9//dVmzMWLF9W1a1d5enqqaNGi6tmzp65cuZLTiwQAAAAAyMfyTOj+7rvv9NFHH6lGjRp3HPftt9+qS5cu6tmzp/bv36+2bduqbdu2+vHHH61jJk+erBkzZmjOnDnatWuXChUqpLCwMF2/ft06pmvXrjp06JA2bdqkdevWadu2berTp49pywcAAAAAyH/yROi+cuWKunbtqnnz5umRRx6549j3339fzZs317Bhw1SlShW9/fbbCgoK0syZMyX9tZd7+vTpeuutt9SmTRvVqFFDH3/8sU6dOqU1a9ZIkuLi4hQdHa358+erXr16atSokT744AMtX75cp06dMntxAQAAAAD5RJ4I3f3791erVq0UGhp617GxsbF248LCwhQbGytJio+PV0JCgs0YLy8v1atXzzomNjZWRYsWVZ06daxjQkND5eTkpF27dmX62ikpKUpKSrJ5AAAAAAAeXi6OLuBuli9frn379um7777L0viEhAT5+PjYtPn4+CghIcHan952pzElSpSw6XdxcZG3t7d1TEYmTJigsWPHZqlOAAAAAED+l6v3dJ88eVKDBw/WkiVL5O7u7uhy7mrEiBFKTEy0Pk6ePOnokgAAAAAADpSr93Tv3btXZ8+eVVBQkLUtNTVV27Zt08yZM5WSkiJnZ2ebaXx9fXXmzBmbtjNnzsjX19fan95WsmRJmzG1atWyjjl79qzNPG7duqWLFy9ap8+Im5ub3Nzcsr+gAAAAAIB8KVfv6W7WrJl++OEHHThwwPqoU6eOunbtqgMHDtgFbkkKDg5WTEyMTdumTZsUHBwsSQoICJCvr6/NmKSkJO3atcs6Jjg4WJcvX9bevXutY77++mulpaWpXr16ZiwqAAAAACAfytV7uosUKaJq1arZtBUqVEjFihWza083ePBgNW7cWFOmTFGrVq20fPly7dmzR3PnzpUk632+x48frwoVKiggIEAjR46Un5+f2rZtK0mqUqWKmjdvrt69e2vOnDm6efOmBgwYoM6dO8vPz8/UZQYAAAAA5B+5ek93VnTv3l0hISHW5w0aNNDSpUs1d+5c1axZU59++qnWrFljE9Jfe+01DRw4UH369NGTTz6pK1euKDo62ua88SVLlqhy5cpq1qyZWrZsqUaNGlmDOwAAAAAAWZGr93RnZMuWLTbP4+Pj1aRJE5u2Dh06qEOHDpnOw2KxaNy4cRo3blymY7y9vbV06dL7qhUAAAAA8HDLc6H7domJiTp69KjWr1/v6FIAAAAAALCTp0O3l5eXfv/9d0eXAQAAAABAhvL8Od0AAAAAAORWhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkuT50T5gwQU8++aSKFCmiEiVKqG3btjp8+PBdp1u5cqUqV64sd3d3Va9eXRs2bLDpNwxDo0aNUsmSJeXh4aHQ0FD9+uuvNmMuXryorl27ytPTU0WLFlXPnj115cqVHF0+AAAAAED+letD99atW9W/f3/t3LlTmzZt0s2bN/XMM88oOTk502m+/fZbdenSRT179tT+/fvVtm1btW3bVj/++KN1zOTJkzVjxgzNmTNHu3btUqFChRQWFqbr169bx3Tt2lWHDh3Spk2btG7dOm3btk19+vQxdXkBAAAAAPmHi6MLuJvo6Gib51FRUSpRooT27t2rp556KsNp3n//fTVv3lzDhg2TJL399tvatGmTZs6cqTlz5sgwDE2fPl1vvfWW2rRpI0n6+OOP5ePjozVr1qhz586Ki4tTdHS0vvvuO9WpU0eS9MEHH6hly5Z677335OfnZ+JSAwAAAADyg1y/p/vvEhMTJUne3t6ZjomNjVVoaKhNW1hYmGJjYyVJ8fHxSkhIsBnj5eWlevXqWcfExsaqaNGi1sAtSaGhoXJyctKuXbtybHkAAAAAAPlXrt/Tfbu0tDQNGTJEDRs2VLVq1TIdl5CQIB8fH5s2Hx8fJSQkWPvT2+40pkSJEjb9Li4u8vb2to75u5SUFKWkpFifJyUlZXHJAAAAAAD5UZ7a092/f3/9+OOPWr58uaNLydCECRPk5eVlfZQqVcrRJQEAAAAAHCjPhO4BAwZo3bp12rx5sx5//PE7jvX19dWZM2ds2s6cOSNfX19rf3rbncacPXvWpv/WrVu6ePGidczfjRgxQomJidbHyZMns76AAAAAAIB8J9eHbsMwNGDAAK1evVpff/21AgIC7jpNcHCwYmJibNo2bdqk4OBgSVJAQIB8fX1txiQlJWnXrl3WMcHBwbp8+bL27t1rHfP1118rLS1N9erVy/B13dzc5OnpafMAAAAAADy8cv053f3799fSpUv1v//9T0WKFLGeT+3l5SUPD48Mpxk8eLAaN26sKVOmqFWrVlq+fLn27NmjuXPnSpIsFouGDBmi8ePHq0KFCgoICNDIkSPl5+entm3bSpKqVKmi5s2bq3fv3pozZ45u3rypAQMGqHPnzly5HAAAAACQJbl+T/fs2bOVmJiokJAQlSxZ0vpYsWKFdUz37t0VEhJifd6gQQMtXbpUc+fOVc2aNfXpp59qzZo1Nhdfe+211zRw4ED16dNHTz75pK5cuaLo6Gi5u7tbxyxZskSVK1dWs2bN1LJlSzVq1Mga3AEAAAAAuJtcv6fbMIy7jomPj1eTJk1s2jp06KAOHTpkOo3FYtG4ceM0bty4TMd4e3tr6dKlWS8WAAAAAIDb5PrQfTeJiYk6evSo1q9f7+hSAAAAAACwkedDt5eXl37//XdHlwEAAAAAgJ1cf043AAAAAAB5FaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaH7LmbNmqUyZcrI3d1d9erV0+7dux1dEgAAAAAgjyB038GKFSsUGRmp0aNHa9++fapZs6bCwsJ09uxZR5cGAAAAAMgDCN13MHXqVPXu3Vs9evRQYGCg5syZo4IFC2rhwoWOLg0AAAAAkAcQujNx48YN7d27V6GhodY2JycnhYaGKjY21oGVAQAAAADyChdHF5BbnT9/XqmpqfLx8bFp9/Hx0c8//5zhNCkpKUpJSbE+T0xMlCQlJSWZV2gOeLSgdMPL2dFl3JNHC5q3fvPqemGd2GOdZIz1Yo91Yo91kjHWiz3WiT3WScZYL/ZYJ/bMXCc5rUiRIrJYLJn2WwzDMB5gPXnGqVOn9Nhjj+nbb79VcHCwtf21117T1q1btWvXLrtpxowZo7Fjxz7IMgEAAAAADpSYmChPT89M+9nTnYnixYvL2dlZZ86csWk/c+aMfH19M5xmxIgRioyMtD5PS0vTxYsXVaxYsTv+8pFfJSUlqVSpUjp58uQdN8KHDevFHuvEHuskY6wXe6wTe6yTjLFe7LFOMsZ6scc6scc6+T9FihS5Yz+hOxOurq6qXbu2YmJi1LZtW0l/heiYmBgNGDAgw2nc3Nzk5uZm01a0aFGTK839PD09H/oPYkZYL/ZYJ/ZYJxljvdhjndhjnWSM9WKPdZIx1os91ok91sndEbrvIDIyUuHh4apTp47q1q2r6dOnKzk5WT169HB0aQAAAACAPIDQfQedOnXSuXPnNGrUKCUkJKhWrVqKjo62u7gaAAAAAAAZIXTfxYABAzI9nBx35ubmptGjR9sdcv+wY73YY53YY51kjPVij3Vij3WSMdaLPdZJxlgv9lgn9lgnWcfVywEAAAAAMImTowsAAAAAACC/InQDAAAAAGASQjeQC1gsFq1Zs8bRZQBAvsZ3LQDAEQjduGfdu3e33sMcf60Pi8Vi9zhy5IijS3OI9PXRt29fu77+/fvLYrGoe/fuD76wXCQ2NlbOzs5q1aqVo0txGLaTu+O7NnOsm7/wXWLv3Llz6tevn0qXLi03Nzf5+voqLCxMO3bscHRpDnfy5ElFRETIz89Prq6u8vf31+DBg3XhwoUsTb9lyxZZLBZdvnzZ3EIfgPT/gyZOnGjTvmbNGlksFgdV5Vi3/z1boEAB+fj46Omnn9bChQuVlpbm6PLyLEI3kIOaN2+u06dP2zwCAgIcXZbDlCpVSsuXL9e1a9esbdevX9fSpUtVunTp+5r3zZs377c8h1uwYIEGDhyobdu26dSpU/c1r9TU1Dz7n6GZ2wnwMMjJ75L8on379tq/f78WL16sX375RZ9//rlCQkKyHCzzq99++0116tTRr7/+qmXLlunIkSOaM2eOYmJiFBwcrIsXLzq6xAfO3d1dkyZN0qVLlxxdSq6R/vfssWPH9MUXX6hJkyYaPHiwnn32Wd26dcvR5eVJhG7kiOjoaDVq1EhFixZVsWLF9Oyzz+ro0aPW/mPHjslisWjVqlVq0qSJChYsqJo1ayo2NtaBVee89F/Tb384Ozvrf//7n4KCguTu7q6yZctq7Nixdl9ap0+fVosWLeTh4aGyZcvq008/ddBS5JygoCCVKlVKq1atsratWrVKpUuX1hNPPGFty+r2s2LFCjVu3Fju7u5asmTJA12WnHblyhWtWLFC/fr1U6tWrRQVFWXtS9+LsH79etWoUUPu7u6qX7++fvzxR+uYqKgoFS1aVJ9//rkCAwPl5uamEydOOGBJ7l9ObSdNmza1u8XjuXPn5OrqqpiYGPMX5AEoU6aMpk+fbtNWq1YtjRkzxvrcYrFo/vz5ev7551WwYEFVqFBBn3/++YMt1AGysm7yozt9l6R/T9wuoz1448ePV4kSJVSkSBH16tVLw4cPV61atcwv3iSXL1/WN998o0mTJqlJkyby9/dX3bp1NWLECD333HPWMb169dKjjz4qT09PNW3aVN9//711HmPGjFGtWrX00UcfqVSpUipYsKA6duyoxMRERy1Wjujfv79cXV315ZdfqnHjxipdurRatGihr776Sn/88YfefPNNSVJKSopef/11lSpVSm5ubipfvrwWLFigY8eOqUmTJpKkRx55JF8cjRQaGipfX19NmDAh0zGfffaZqlatKjc3N5UpU0ZTpkyx9r3xxhuqV6+e3TQ1a9bUuHHjTKnZbOl/zz722GMKCgrSG2+8of/973/64osvrN8xd/sMSdLatWv15JNPyt3dXcWLF9fzzz/vgKXJHQjdyBHJycmKjIzUnj17FBMTIycnJz3//PN2e97efPNNvfrqqzpw4IAqVqyoLl265PtfzL755hu99NJLGjx4sH766Sd99NFHioqK0jvvvGMzbuTIkWrfvr2+//57de3aVZ07d1ZcXJyDqs45ERERWrRokfX5woUL1aNHD5sxWd1+hg8frsGDBysuLk5hYWEPpH6zfPLJJ6pcubIqVaqkF198UQsXLtTf7+A4bNgwTZkyRd99950effRRtW7d2mYP/9WrVzVp0iTNnz9fhw4dUokSJR70YuSYnNhOevXqpaVLlyolJcU6zX//+1899thjatq06YNZkFxi7Nix6tixow4ePKiWLVuqa9euD+UerIdBVr5L7mTJkiV65513NGnSJO3du1elS5fW7NmzTazYfIULF1bhwoW1Zs0am++D23Xo0EFnz57VF198ob179yooKEjNmjWz+ZwcOXJEn3zyidauXavo6Gjt379fL7/88oNajBx38eJFbdy4US+//LI8PDxs+nx9fdW1a1etWLFChmHopZde0rJlyzRjxgzFxcXpo48+UuHChVWqVCl99tlnkqTDhw/r9OnTev/99x2xODnG2dlZ7777rj744AP9/vvvdv179+5Vx44d1blzZ/3www8aM2aMRo4caQ2fXbt21e7du21+BD506JAOHjyof/7znw9qMUzXtGlT1axZ0/oD+d0+Q+vXr9fzzz+vli1bav/+/YqJiVHdunUduQiOZQD3KDw83GjTpk2GfefOnTMkGT/88INhGIYRHx9vSDLmz59vHXPo0CFDkhEXF/cgyjVdeHi44ezsbBQqVMj6eOGFF4xmzZoZ7777rs3Y//znP0bJkiWtzyUZffv2tRlTr149o1+/fg+kdjOkbx9nz5413NzcjGPHjhnHjh0z3N3djXPnzhlt2rQxwsPDM5w2s+1n+vTpD3AJzNWgQQPr8ty8edMoXry4sXnzZsMwDGPz5s2GJGP58uXW8RcuXDA8PDyMFStWGIZhGIsWLTIkGQcOHHjgteeknNxOrl27ZjzyyCPWdWQYhlGjRg1jzJgxD2JRTHP7d62/v78xbdo0m/6aNWsao0ePtj6XZLz11lvW51euXDEkGV988cUDqPbBupd1s3r16gdW34Nwp++SRYsWGV5eXjbjV69ebdz+51+9evWM/v3724xp2LChUbNmTTPLNt2nn35qPPLII4a7u7vRoEEDY8SIEcb3339vGIZhfPPNN4anp6dx/fp1m2nKlStnfPTRR4ZhGMbo0aMNZ2dn4/fff7f2f/HFF4aTk5Nx+vTpB7cgOWjnzp13/AxMnTrVkGTs2rXLkGRs2rQpw3Hp/0ddunTJvGIfkNu/Q+rXr29EREQYhmH7OfnnP/9pPP300zbTDRs2zAgMDLQ+r1mzpjFu3Djr8xEjRhj16tUzuXpz3Onv+06dOhlVqlTJ0mcoODjY6Nq1q9nl5hns6UaO+PXXX9WlSxeVLVtWnp6eKlOmjCTZHe5ao0YN679LliwpSTp79uwDq9NsTZo00YEDB6yPGTNm6Pvvv9e4ceOsv7wXLlxYvXv31unTp3X16lXrtMHBwTbzCg4Ozhd7uh999FHrIY+LFi1Sq1atVLx4cZsxWd1+6tSp86DKNtXhw4e1e/dudenSRZLk4uKiTp06acGCBTbjbt8mvL29ValSJZttwtXV1eYzlZflxHbi7u6ubt26aeHChZKkffv26ccff8zzhz7ei9u3i0KFCsnT0zNffdfiL1n9LrnbPP6+9yk/7I1q3769Tp06pc8//1zNmzfXli1bFBQUpKioKH3//fe6cuWKihUrZvN/c3x8vM3eytKlS+uxxx6zPg8ODlZaWpoOHz7siEXKMcZdjoQ4duyYnJ2d1bhx4wdUUe4wadIkLV682O5vr7i4ODVs2NCmrWHDhvr111+Vmpoq6a+93UuXLpX01/pdtmyZunbt+mAKf4AMw5DFYsnSZ+jAgQNq1qyZgyvOPVwcXQDyh9atW8vf31/z5s2Tn5+f0tLSVK1aNd24ccNmXIECBaz/Tj+nLK9e/CkjhQoVUvny5W3arly5orFjx6pdu3Z2493d3R9UaQ4VERFhPdd21qxZdv1Z3X4KFSr0QOo124IFC3Tr1i35+flZ2wzDkJubm2bOnJnl+Xh4eOSrq6vmxHbSq1cv1apVS7///rsWLVqkpk2byt/f/4Etg9mcnJzs/mDO6KKCt3/XSn993+an79qMZHXd5Cd3+y55GNfJ7dzd3fX000/r6aef1siRI9WrVy+NHj1aL7/8skqWLKktW7bYTfP3c+Dzk/Lly8tisSguLi7Dc2vj4uL0yCOP2B16/rB46qmnFBYWphEjRmT7x9ouXbro9ddf1759+3Tt2jWdPHlSnTp1MqdQB4qLi1NAQICuXLly18/Qw7odZYbQjft24cIFHT58WPPmzdM//vEPSdL27dsdXFXuERQUpMOHD9uF8b/buXOnXnrpJZvnt19EKi9r3ry5bty4IYvFYncu9sO2/dy6dUsff/yxpkyZomeeecamr23btlq2bJkqV64s6a9tIP3q3ZcuXdIvv/yiKlWqPPCaH5Sc2E6qV6+uOnXqaN68eVq6dGm2fsTICx599FGdPn3a+jwpKUnx8fEOrCj3eNjWTVa+S/z9/fXnn38qOTnZ+qPlgQMHbMZWqlRJ3333nc3/P999953p9TtCYGCg1qxZo6CgICUkJMjFxcV6xExGTpw4oVOnTll/1Ni5c6ecnJxUqVKlB1RxzipWrJiefvppffjhhxo6dKhNKEpISNCSJUv00ksvqXr16kpLS9PWrVsVGhpqNx9XV1dJsu7lzU8mTpyoWrVq2bzHVapUsbvV3I4dO1SxYkU5OztLkh5//HE1btxYS5Ys0bVr1/T000/n6eusZOTrr7/WDz/8oKFDh+rxxx+/62eoRo0aiomJsbs+y8OK0I379sgjj6hYsWKaO3euSpYsqRMnTmj48OGOLivXGDVqlJ599lmVLl1aL7zwgpycnPT999/rxx9/1Pjx463jVq5cqTp16qhRo0ZasmSJdu/ena1DBHMzZ2dn6+Fa6f9BpXvYtp9169bp0qVL6tmzp7y8vGz62rdvrwULFujf//63JGncuHEqVqyYfHx89Oabb6p48eL5+p7EObWd9OrVSwMGDFChQoXy3ZVSmzZtqqioKLVu3VpFixbVqFGj7NbVw+phWzdZ+S7ZuHGjChYsqDfeeEODBg3Srl27bK5uLkkDBw5U7969VadOHTVo0EArVqzQwYMHVbZs2Qe4NDnrwoUL6tChgyIiIlSjRg0VKVJEe/bs0eTJk9WmTRuFhoYqODhYbdu21eTJk1WxYkWdOnXKeuGn9FOZ3N3dFR4ervfee09JSUkaNGiQOnbsKF9fXwcv4b2bOXOmGjRooLCwMI0fP14BAQE6dOiQhg0bpscee0zvvPOOvL29FR4eroiICM2YMUM1a9bU8ePHdfbsWXXs2FH+/v6yWCxat26dWrZsKQ8PDxUuXNjRi5Yjqlevrq5du2rGjBnWtldeeUVPPvmk3n77bXXq1EmxsbGaOXOmPvzwQ5tpu3btqtGjR+vGjRuaNm3agy49R6WkpCghIUGpqak6c+aMoqOjNWHCBD377LN66aWX5OTkdNfP0OjRo9WsWTOVK1dOnTt31q1bt7Rhwwa9/vrrjl48h+CcbtyztLQ0ubi4yMnJScuXL9fevXtVrVo1DR061BoaIIWFhWndunX68ssv9eSTT6p+/fqaNm2a3SGvY8eO1fLly1WjRg19/PHHWrZsmQIDAx1Udc7z9PSUp6enXfvDtv0sWLBAoaGhdn8kS3/9obxnzx4dPHhQ0l+/uA8ePFi1a9dWQkKC1q5da93DkF/lxHbSpUsXubi4qEuXLvniFI7071pJGjFihBo3bqxnn31WrVq1Utu2bVWuXDkHV+g4D/O6ycp3ye+//67//ve/2rBhg6pXr65ly5bZ3UKta9euGjFihF599VUFBQUpPj5e3bt3z9OfncKFC6tevXqaNm2annrqKVWrVk0jR45U7969NXPmTFksFm3YsEFPPfWUevTooYoVK6pz5846fvy4fHx8rPMpX7682rVrp5YtW+qZZ55RjRo17IJWXlOhQgXt2bNHZcuWVceOHVWuXDn16dNHTZo0UWxsrLy9vSVJs2fP1gsvvKCXX35ZlStXVu/evZWcnCxJeuyxxzR27FgNHz5cPj4+drdqzOvGjRtnczpOUFCQPvnkEy1fvlzVqlXTqFGjNG7cOLtD0F944QVduHBBV69ezfM/kEdHR6tkyZIqU6aMmjdvrs2bN2vGjBn63//+J2dn5yx9hkJCQrRy5Up9/vnnqlWrlpo2bardu3c7eMkcx2Lc7WoKQCaaN2+u8uXL57vDNwFH27Jli5o0aaJLly7l6/MLzXLs2DGVK1dO3333nYKCghxdzn3juzZzrBtzPP300/L19dV//vMfR5fiMGPGjNGaNWvsDscHgHvB4eXItkuXLmnHjh3asmWL+vbt6+hyAEDSXxeIunDhgt566y3Vr18/zwduvmszx7rJOVevXtWcOXMUFhYmZ2dnLVu2TF999ZU2bdrk6NIAIN8gdCPbIiIi9N133+mVV15RmzZtHF0OAEj668I2TZo0UcWKFfXpp586upz7xndt5lg3OSf9MNF33nlH169fV6VKlfTZZ59leAEtAMC94fByAAAAAABMwoXUAAAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAOJTFYtGaNWscXQYAAKYgdAMA8JDq3r27LBaL+vbta9fXv39/WSwWde/ePcdeb8yYMapVq1aOzQ8AgLyA0A0AwEOsVKlSWr58ua5du2Ztu379upYuXarSpUs7sDIAAPIHQjcAAA+xoKAglSpVSqtWrbK2rVq1SqVLl9YTTzxhbUtJSdGgQYNUokQJubu7q1GjRvruu++s/Vu2bJHFYlFMTIzq1KmjggULqkGDBjp8+LAkKSoqSmPHjtX3338vi8Uii8WiqKgo6/Tnz5/X888/r4IFC6pChQr6/PPPzV94AAAeAEI3AAAPuYiICC1atMj6fOHCherRo4fNmNdee02fffaZFi9erH379ql8+fIKCwvTxYsXbca9+eabmjJlivbs2SMXFxdFRERIkjp16qRXXnlFVatW1enTp3X69Gl16tTJOt3YsWPVsWNHHTx4UC1btlTXrl3t5g0AQF5E6AYA4CH34osvavv27Tp+/LiOHz+uHTt26MUXX7T2Jycna/bs2fr3v/+tFi1aKDAwUPPmzZOHh4cWLFhgM6933nlHjRs3VmBgoIYPH65vv/1W169fl4eHhwoXLiwXFxf5+vrK19dXHh4e1um6d++uLl26qHz58nr33Xd15coV7d69+4GtAwAAzOLi6AIAAIBjPfroo2rVqpWioqJkGIZatWql4sWLW/uPHj2qmzdvqmHDhta2AgUKqG7duoqLi7OZV40aNaz/LlmypCTp7Nmzdz0//PbpChUqJE9PT509e/a+lgsAgNyA0A0AABQREaEBAwZIkmbNmnXP8ylQoID13xaLRZKUlpaWrenSp83KdAAA5HYcXg4AANS8eXPduHFDN2/eVFhYmE1fuXLl5Orqqh07dljbbt68qe+++06BgYFZfg1XV1elpqbmWM0AAOQF7OkGAABydna2Hiru7Oxs01eoUCH169dPw4YNk7e3t0qXLq3Jkyfr6tWr6tmzZ5Zfo0yZMoqPj9eBAwf0+OOPq0iRInJzc8vR5QAAILchdAMAAEmSp6dnpn0TJ05UWlqaunXrpj///FN16tTRxo0b9cgjj2R5/u3bt9eqVavUpEkTXb58WYsWLVL37t1zoHIAAHIvi2EYhqOLAAAAAAAgP+KcbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCT/Dzwf+rK8Sa/4AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Total births plotted: 180,369\n" + ] + } + ], "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", @@ -632,7 +671,6 @@ " \"Jul\", \"Aug\", \"Sep\", \"Oct\", \"Nov\", \"Dec\"\n", "]\n", "\n", - "# Plot\n", "fig, ax = plt.subplots(figsize=(10, 5))\n", "\n", "ax.bar(\n", @@ -665,9 +703,9 @@ "metadata": {}, "source": [ "---\n", - "## 10. Available Data Sources\n", + "## 8. Available Data Sources\n", "\n", - "PySUS gives access to the following official DATASUS datasets:\n", + "PySUS provides accelerated access to the following official DATASUS structures:\n", "\n", "| Dataset | Full name | Coverage |\n", "|---------|-----------|----------|\n", @@ -680,32 +718,6 @@ "| PNI | Programa Nacional de Imunizações | Vaccination data |\n", "| IBGE | Instituto Brasileiro de Geografia e EstatĂ­stica | Population and demographic data |" ] - }, - { - "cell_type": "markdown", - "id": "md-334929490140752845", - "metadata": {}, - "source": [ - "---\n", - "## 11. Next Steps\n", - "\n", - "You now have a working PySUS workflow. Here are some directions to explore next:\n", - "\n", - "- **Analyse maternal age** — use the `IDADEMAE` column\n", - "- **Compare delivery types** — vaginal vs. caesarean using `PARTO`\n", - "- **Explore prenatal visits** — `CONSULTAS` column\n", - "- **Download mortality data** — `sim(state=\"SP\", year=2022)`\n", - "- **Investigate notifiable diseases** — `sinan(state=\"RJ\", year=2022, group=\"DENG\")`\n", - "- **Compare multiple states** — loop over a list of state codes\n", - "\n", - "---\n", - "\n", - "**Useful resources:**\n", - "\n", - "- [PySUS documentation](https://pysus.readthedocs.io)\n", - "- [PySUS GitHub repository](https://github.com/AlertaDengue/PySUS)\n", - "- [DATASUS portal](https://datasus.saude.gov.br) — official Brazilian health data portal" - ] } ], "metadata": { @@ -724,7 +736,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.7" + "version": "3.12.13" } }, "nbformat": 4, diff --git a/docs/source/installation.rst b/docs/source/installation.rst index d4d64c58..dc994612 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -34,7 +34,7 @@ Or build locally and start the container: .. code-block:: bash - docker compose -f docker/docker-compose.yaml up --build + docker compose up --build Then open http://127.0.0.1:8888/lab in your browser. @@ -42,7 +42,7 @@ Stop the container with: .. code-block:: bash - docker compose -f docker/docker-compose.yaml down + docker compose down Development ----------- diff --git a/pysus/api/_impl/databases.py b/pysus/api/_impl/databases.py index c5321c9f..4f15630c 100644 --- a/pysus/api/_impl/databases.py +++ b/pysus/api/_impl/databases.py @@ -8,12 +8,12 @@ """ import asyncio -from typing import Literal +from typing import Literal, cast import pandas as pd from pysus.api import types from pysus.api.client import PySUS -from tqdm import tqdm +from tqdm.asyncio import tqdm __all__ = [ "sinan", @@ -57,10 +57,9 @@ def _fetch_data( month : int | list[int], optional Month or list of months to fetch. show_progress : bool, optional - Whether to display a tqdm progress bar during download. Default is True. + Whether to display a tqdm progress bar during download. as_dataframe : bool, optional Whether to concatenate and return the data as a pandas DataFrame. - Default is False. **kwargs Additional arguments forwarded to :meth:`PySUS.read_parquet`. @@ -71,48 +70,41 @@ def _fetch_data( as_dataframe is True, returns a concatenated DataFrame. """ - async def _fetch(): - + async def _fetch() -> list[str] | pd.DataFrame: async with PySUS() as pysus: - years = [year] if isinstance(year, int) else (year or [None]) - months = [month] if isinstance(month, int) else (month or [None]) + files = await pysus.query( + dataset=dataset, + group=group, + state=state, + year=year, + month=month, + ) - files = [] - for y in years: - for m in months: - files.extend( - await pysus.query( - dataset=dataset, - group=group, - state=state, - year=y, - month=m, - ) - ) + if not files: + return pd.DataFrame() if as_dataframe else cast(list[str], []) + + sem = asyncio.Semaphore(3) + + async def _throttled_download(f): + async with sem: + return await pysus.download(f) + + tasks = [_throttled_download(f) for f in files] - paths = [] if show_progress: - for file in tqdm( - files, + downloaded_files = await tqdm.gather( + *tasks, desc=f"Downloading {dataset}", unit="file", - ): - f = await pysus.download(file) - paths.append(str(f.path)) + ) else: - for file in files: - f = await pysus.download(file) - paths.append(str(f.path)) + downloaded_files = await asyncio.gather(*tasks) + + paths: list[str] = [str(f.path) for f in downloaded_files] if as_dataframe: - return ( - pysus.read_parquet( - paths, - **kwargs, - ).df() - if paths - else pd.DataFrame() - ) + res = pysus.read_parquet(paths, **kwargs).df() + return cast(pd.DataFrame, res) return paths @@ -132,9 +124,11 @@ async def _fetch(): "Install it with: pip install nest_asyncio" ) raise RuntimeError(msg) from None - return loop.run_until_complete(_fetch()) - else: - return asyncio.run(_fetch()) + result = loop.run_until_complete(_fetch()) + return cast(list[str] | pd.DataFrame, result) + + result = asyncio.run(_fetch()) + return cast(list[str] | pd.DataFrame, result) def sinan( diff --git a/pysus/api/client.py b/pysus/api/client.py index 2854f075..7ef4f2a2 100644 --- a/pysus/api/client.py +++ b/pysus/api/client.py @@ -314,7 +314,11 @@ async def download( ) return await ExtensionFactory.instantiate(local_path) - except Exception as e: # noqa: B902 + except Exception as e: # noqa + import traceback + + traceback.print_exc() + await self._update_state( local_path, str(remote_path), diff --git a/pysus/api/ducklake/catalog/adapters.py b/pysus/api/ducklake/catalog/adapters.py index a2a14808..61e64230 100644 --- a/pysus/api/ducklake/catalog/adapters.py +++ b/pysus/api/ducklake/catalog/adapters.py @@ -239,7 +239,7 @@ def __init__(self, name: str, dataset_id: int, engine=None, **data) -> None: super().__init__(engine=engine, **data) self.dataset_name: str = name self.db_local: Path = self.cache_dir / f"catalog_{name}.duckdb" - self.db_remote: Path = Path(f"datasets/catalog_{name}.duckdb") + self.db_remote: Path = Path(f"public/catalog_{name}.duckdb") self.dataset_id = dataset_id diff --git a/pysus/api/ducklake/functional.py b/pysus/api/ducklake/functional.py index fcb3289b..aad9139a 100644 --- a/pysus/api/ducklake/functional.py +++ b/pysus/api/ducklake/functional.py @@ -17,10 +17,16 @@ async def download_http( url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote_path}" max_retries = 5 + timeout = httpx.Timeout(15.0, read=60.0, write=20.0, connect=15.0) + limits = httpx.Limits(max_keepalive_connections=5, max_connections=10) + for attempt in range(max_retries): try: async with httpx.AsyncClient( - follow_redirects=True, verify=False + follow_redirects=True, + verify=False, + limits=limits, + timeout=timeout, ) as client: async with client.stream("GET", url) as r: r.raise_for_status() @@ -28,17 +34,20 @@ async def download_http( downloaded = 0 with open(local_path, "wb") as f: - async for chunk in r.aiter_bytes( - chunk_size=1024 * 1024 - ): + async for chunk in r.aiter_bytes(chunk_size=64 * 1024): await to_thread.run_sync(f.write, chunk) downloaded += len(chunk) if callback: callback(downloaded, total) return - except (OSError, httpx.HTTPStatusError) as e: + except ( + OSError, + httpx.HTTPStatusError, + httpx.ConnectError, + httpx.ReadError, + ) as e: if attempt < max_retries - 1: - await sleep(1) + await sleep(2 * (attempt + 1)) else: raise e diff --git a/pysus/api/ducklake/models.py b/pysus/api/ducklake/models.py index ad78415d..b0f3dde7 100644 --- a/pysus/api/ducklake/models.py +++ b/pysus/api/ducklake/models.py @@ -97,7 +97,7 @@ def _calculate(): class DuckDataset(BaseRemoteDataset): record: "Dataset" = Field(exclude=True) client: "DuckLake" = Field(exclude=True) - border: "DatasetAdapter" = Field(exclude=True) + border: Any = Field(exclude=True) update_on_close: bool = Field(default=False, exclude=True) def __init__(self, **data) -> None: @@ -143,12 +143,14 @@ async def query( self, group: str | list[str] | None = None, state: str | list[str] | None = None, - year: int | list[int] | None = None, - month: int | list[int] | None = None, + year: int | list[int] | range | None = None, + month: int | list[int] | range | None = None, ) -> list[File]: def _to_list(val: Any) -> list[Any] | None: if val is None: return None + if isinstance(val, range): + return list(val) return val if isinstance(val, list) else [val] groups = _to_list(group) diff --git a/pysus/api/metadata/models.py b/pysus/api/metadata/models.py index 62a20bc9..44bd44c4 100644 --- a/pysus/api/metadata/models.py +++ b/pysus/api/metadata/models.py @@ -3,30 +3,6 @@ from pysus.api.types import ColumnType, Origin -def lookup_column_meta(name: str) -> dict[str, str] | None: - """Look up column metadata from the global columns.py constants. - - Returns the {dataset: description} dict if the column name exists - as a constant in columns.py, or None if not found. - """ - try: - from pysus.api.ducklake.catalog import columns as _cols - - return getattr(_cols, name.upper(), None) - except ImportError: - return None - - -def pick_description(meta: dict[str, str] | None) -> str: - """Pick the best description from a column metadata dict.""" - if meta is None: - return "" - for desc in meta.values(): - if desc: - return desc - return "" - - @dataclass class Dataset: name: str @@ -70,11 +46,12 @@ class Column: dtype: ColumnType @classmethod - def from_schema(cls, name: str, dtype: ColumnType) -> "Column": - """Create a Column from a file schema, looking up description from - columns.py.""" + def from_schema( + cls, name: str, dtype: ColumnType, description: str = "" + ) -> "Column": + """Create a Column with a description provided from the database.""" return cls( name=name, - description=pick_description(lookup_column_meta(name)), + description=description, dtype=dtype, ) diff --git a/pysus/tests/api/dadosgov/test_client.py b/pysus/tests/api/dadosgov/test_client.py index 9c728040..29c71e12 100644 --- a/pysus/tests/api/dadosgov/test_client.py +++ b/pysus/tests/api/dadosgov/test_client.py @@ -464,7 +464,7 @@ async def test_download_file_connection_error(self): mock_file = MagicMock() mock_file.path = "http://example.com/file.csv" with pytest.raises(ConnectionError, match="Client not connected"): - await client._download_file(mock_file, Path("/tmp/out.csv")) + await client.download(mock_file, Path("/tmp/out.csv")) @pytest.mark.asyncio async def test_download_file_success(self, tmp_path): @@ -493,9 +493,7 @@ async def _aiter_bytes(): callback = MagicMock() try: - result = await client._download_file( - mock_file, output, callback=callback - ) + result = await client.download(mock_file, output, callback=callback) assert result == output mock_http.stream.assert_called_once_with( @@ -534,7 +532,7 @@ async def _aiter_bytes(): output = tmp_path / "test_download_nocb.csv" try: - result = await client._download_file(mock_file, output) + result = await client.download(mock_file, output) assert result == output mock_http.stream.assert_called_once_with( diff --git a/pysus/tests/api/dadosgov/test_models.py b/pysus/tests/api/dadosgov/test_models.py index dfbcc005..e09a06b9 100644 --- a/pysus/tests/api/dadosgov/test_models.py +++ b/pysus/tests/api/dadosgov/test_models.py @@ -415,11 +415,9 @@ async def test_download_delegates_to_client(self): output = Path("/tmp/test_out.csv") callback = MagicMock() - with patch.object( - ds.client, "_download_file", new_callable=AsyncMock - ) as mock_dl: - mock_dl.return_value = output - result = await f._download(output=output, callback=callback) + mock_dl = AsyncMock(return_value=output) + object.__setattr__(ds.client, "download", mock_dl) + result = await f._download(output=output, callback=callback) assert result == output mock_dl.assert_awaited_once_with(f, output, callback=callback) @@ -432,11 +430,9 @@ async def test_download_default_output(self): expected = CACHEPATH / f.name - with patch.object( - ds.client, "_download_file", new_callable=AsyncMock - ) as mock_dl: - mock_dl.return_value = expected - result = await f._download() + mock_dl = AsyncMock(return_value=expected) + object.__setattr__(ds.client, "download", mock_dl) + result = await f._download() assert result == expected diff --git a/pysus/tests/api/ducklake/test_catalog.py b/pysus/tests/api/ducklake/test_catalog.py index 2afecabd..1fe25346 100644 --- a/pysus/tests/api/ducklake/test_catalog.py +++ b/pysus/tests/api/ducklake/test_catalog.py @@ -36,9 +36,7 @@ def test_schema(self): assert Dataset.__table_args__[0]["schema"] == "pysus" def test_relationships(self): - assert hasattr(Dataset, "groups") - assert hasattr(Dataset, "files") - assert hasattr(Dataset, "columns") + assert hasattr(Dataset, "__tablename__") class TestColumnDefinition: @@ -68,7 +66,6 @@ def test_columns(self): assert "description" in cols def test_relationships(self): - assert hasattr(Group, "dataset") assert hasattr(Group, "files") @@ -94,9 +91,7 @@ def test_columns(self): assert "origin_path" in cols def test_relationships(self): - assert hasattr(File, "dataset") assert hasattr(File, "group") - assert hasattr(File, "columns") class TestFileColumns: @@ -111,6 +106,4 @@ def test_file_columns_primary_keys(self): def test_file_columns_foreign_keys(self): file_id_col = file_columns.c.file_id - column_id_col = file_columns.c.column_id assert file_id_col.foreign_keys - assert column_id_col.foreign_keys diff --git a/pysus/tests/api/ducklake/test_client.py b/pysus/tests/api/ducklake/test_client.py index 08885332..208d19e7 100644 --- a/pysus/tests/api/ducklake/test_client.py +++ b/pysus/tests/api/ducklake/test_client.py @@ -1,11 +1,11 @@ """Tests for DuckLake client module.""" -import errno from datetime import datetime from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import pytest +from pysus.api.ducklake.catalog.adapters import CatalogAdapter, DatasetAdapter from pysus.api.ducklake.catalog.orm.dataset import File as CatalogFile from pysus.api.ducklake.catalog.orm.default import Dataset as PerDataset from pysus.api.ducklake.client import DuckLake, DuckLakeCredentials @@ -28,8 +28,10 @@ async def test_ducklake_init(self): client = DuckLake() assert client.name == "DuckLake" assert client.long_name == "PySUS s3 Client" - assert client.endpoint == "nbg1.your-objectstorage.com" - assert client.bucket == "pysus" + assert client.credentials is None + assert client.update_on_close is False + assert isinstance(client._catalog_adap, CatalogAdapter) + assert client._datasets == [] @pytest.mark.asyncio async def test_description(self): @@ -37,118 +39,66 @@ async def test_description(self): assert client.description == "" @pytest.mark.asyncio - async def test_ducklake_catalog_path(self, tmp_path): - with patch("pysus.api.ducklake.client.CACHEPATH", tmp_path): + async def test_ducklake_catalog_path(self): + with patch("pathlib.Path.mkdir"): client = DuckLake() - assert ( - client.catalog_path == tmp_path / "ducklake" / "catalog.duckdb" - ) - - @pytest.mark.asyncio - async def test_ducklake_catalog_url(self): - client = DuckLake() - expected = ( - "https://nbg1.your-objectstorage.com/pysus/public/catalog.duckdb" - ) - assert client._catalog_url == expected - - @pytest.mark.asyncio - async def test_is_authenticated_false_no_credentials(self): - client = DuckLake() - assert client._is_authenticated is False + assert isinstance(client.catalog_path, Path) + assert client.catalog_path.name == "catalog.duckdb" @pytest.mark.asyncio - async def test_is_authenticated_with_credentials(self): + async def test_is_authenticated_without_credentials(self): client = DuckLake() - with patch.object(client, "_download_catalog"): - await client.login(access_key="key", secret_key="secret") - assert client._is_authenticated is True + assert client.credentials is None @pytest.mark.asyncio async def test_login_sets_credentials(self): client = DuckLake() - with patch.object(client, "_download_catalog"): - await client.login(access_key="key", secret_key="secret") + client._catalog_adap = AsyncMock() + client._columns_adap = AsyncMock() + await client.login(access_key="key", secret_key="secret") assert client.credentials is not None - @pytest.mark.asyncio - async def test_login_creates_s3_client(self): - client = DuckLake() - with patch.object(client, "_download_catalog"): - await client.login(access_key="key", secret_key="secret") - assert client._s3_client is not None - - @pytest.mark.asyncio - async def test_login_clears_credentials(self): - client = DuckLake() - client.credentials = DuckLakeCredentials( - access_key="test_key", - secret_key="test_secret", - ) - with patch.object(client, "_download_catalog"): - await client.login() - assert client.credentials is None - assert client._s3_client is None - - @pytest.mark.asyncio - async def test_close_clears_state(self): - client = DuckLake() - client._engine = MagicMock() - with patch( - "pysus.api.ducklake.client.to_thread.run_sync", - side_effect=lambda fn, *a, **kw: fn(), - ): - await client.close() - assert client._engine is None - assert client._Session is None - assert client._s3_client is None - @pytest.mark.asyncio async def test_close_with_datasets(self): client = DuckLake() + client._catalog_adap = AsyncMock() + client._columns_adap = AsyncMock() ds = AsyncMock(spec=DuckDataset) client._datasets.append(ds) await client.close() ds.close.assert_awaited_once_with(update_catalog=False) - assert client._datasets == [] @pytest.mark.asyncio async def test_close_with_update_catalog(self): client = DuckLake() + client._catalog_adap = AsyncMock() + client._columns_adap = AsyncMock() ds = AsyncMock(spec=DuckDataset) client._datasets.append(ds) - with patch.object(client, "_upload_catalog") as mock_upload: - await client.close(update_catalog=True) - mock_upload.assert_awaited_once() - - @pytest.mark.asyncio - async def test_get_s3_client_requires_credentials(self): - client = DuckLake() - with pytest.raises(ConnectionError): - client._get_s3_client() - - @pytest.mark.asyncio - async def test_upload_catalog_requires_auth(self): - client = DuckLake() - with pytest.raises(PermissionError): - await client._upload_catalog() + await client.close(update_catalog=True) + ds.close.assert_awaited_once_with(update_catalog=True) class TestDuckLakeDatasets: @pytest.mark.asyncio - async def test_datasets_creates_session_and_returns_duckdatasets( - self, tmp_path - ): - with patch("pysus.api.ducklake.client.CACHEPATH", tmp_path): - client = DuckLake() + async def test_datasets_returns_duckdatasets(self, tmp_path): + with patch("pysus.api.ducklake.catalog.adapters.CACHEPATH", tmp_path): + with patch("pathlib.Path.mkdir"): + client = DuckLake() mock_session = MagicMock() mock_session.__enter__.return_value = mock_session record = PerDataset(name="sinan", long_name="SINAN", description="Test") + record.id = 1 mock_session.query.return_value.all.return_value = [record] - client._Session = MagicMock(return_value=mock_session) + + mock_catalog_adap = MagicMock() + mock_catalog_adap.__aenter__.return_value = mock_catalog_adap + mock_catalog_adap.__aexit__ = AsyncMock() + mock_catalog_adap.get_session.return_value = mock_session + client._catalog_adap = mock_catalog_adap def run_sync(fn, *args, **kwargs): return fn() @@ -157,7 +107,8 @@ def run_sync(fn, *args, **kwargs): "pysus.api.ducklake.client.to_thread.run_sync", side_effect=run_sync, ): - result = await client.datasets() + with patch("pathlib.Path.mkdir"): + result = await client.datasets() assert len(result) == 1 assert isinstance(result[0], DuckDataset) @@ -165,147 +116,55 @@ def run_sync(fn, *args, **kwargs): @pytest.mark.asyncio async def test_datasets_connects_if_no_session(self, tmp_path): - with patch("pysus.api.ducklake.client.CACHEPATH", tmp_path): - client = DuckLake() - - assert client._Session is None + with patch("pysus.api.ducklake.catalog.adapters.CACHEPATH", tmp_path): + with patch("pathlib.Path.mkdir"): + client = DuckLake() mock_session = MagicMock() mock_session.__enter__.return_value = mock_session mock_session.query.return_value.all.return_value = [] - async def _connect(*args, **kwargs): - client._Session = MagicMock(return_value=mock_session) - - with patch.object( - DuckLake, "connect", new=AsyncMock(side_effect=_connect) - ): + mock_catalog_adap = MagicMock() + mock_catalog_adap.__aenter__.return_value = mock_catalog_adap + mock_catalog_adap.__aexit__ = AsyncMock() + mock_catalog_adap.get_session.return_value = mock_session + client._catalog_adap = mock_catalog_adap - def run_sync(fn, *args, **kwargs): - return fn() + def run_sync(fn, *args, **kwargs): + return fn() - with patch( - "pysus.api.ducklake.client.to_thread.run_sync", - side_effect=run_sync, - ): + with patch( + "pysus.api.ducklake.client.to_thread.run_sync", + side_effect=run_sync, + ): + with patch("pathlib.Path.mkdir"): await client.datasets() -class TestDuckLakeSetupEngine: - def test_setup_engine_has_pysus_schema(self): - with patch("pysus.api.ducklake.client.create_engine") as mock_create: - mock_engine = MagicMock() - mock_conn = MagicMock() - mock_engine.connect.return_value.__enter__.return_value = mock_conn - mock_create.return_value = mock_engine - - mock_conn.exec_driver_sql().fetchone.return_value = (1,) - - client = DuckLake() - result = client._setup_engine() - - calls = [str(c) for c in mock_conn.exec_driver_sql.call_args_list] - assert any( - "SET search_path" in c and "pysus,main" in c for c in calls - ) - assert result is mock_engine - - def test_setup_engine_no_pysus_schema(self): - with patch("pysus.api.ducklake.client.create_engine") as mock_create: - mock_engine = MagicMock() - mock_conn = MagicMock() - mock_engine.connect.return_value.__enter__.return_value = mock_conn - mock_create.return_value = mock_engine - - mock_conn.exec_driver_sql().fetchone.return_value = None - - client = DuckLake() - result = client._setup_engine() - - calls = [str(c) for c in mock_conn.exec_driver_sql.call_args_list] - assert any("SET search_path" in c and "'main'" in c for c in calls) - assert result is mock_engine - - def test_setup_engine_with_credentials(self): - with patch("pysus.api.ducklake.client.create_engine") as mock_create: - mock_engine = MagicMock() - mock_conn = MagicMock() - mock_engine.connect.return_value.__enter__.return_value = mock_conn - mock_create.return_value = mock_engine - - mock_conn.exec_driver_sql().fetchone.return_value = None - - client = DuckLake( - credentials=DuckLakeCredentials( - access_key="ak", secret_key="sk" - ) - ) - client._setup_engine() - - calls = [str(c) for c in mock_conn.exec_driver_sql.call_args_list] - s3_access = any( - "s3_access_key_id" in c and "ak" in c for c in calls - ) - s3_secret = any( - "s3_secret_access_key" in c and "sk" in c for c in calls - ) - assert s3_access - assert s3_secret - - class TestDuckLakeConnect: @pytest.mark.asyncio - async def test_connect_already_connected_returns_early(self): + async def test_connect_delegates_to_adapters(self): client = DuckLake() - client._engine = MagicMock() - client._Session = MagicMock() - with patch.object(client, "_download_catalog") as mock_dl: - await client.connect() - mock_dl.assert_not_called() + client._catalog_adap = AsyncMock() + client._columns_adap = AsyncMock() + await client.connect() + client._catalog_adap.connect.assert_awaited_once_with(force=False) + client._columns_adap.connect.assert_awaited_once_with(force=False) @pytest.mark.asyncio - async def test_connect_creates_session_if_missing(self): + async def test_connect_force(self): client = DuckLake() - client._engine = MagicMock() - client._Session = None - with patch.object(client, "_download_catalog") as mock_dl: - await client.connect() - assert client._Session is not None - mock_dl.assert_not_called() - - @pytest.mark.asyncio - async def test_connect_downloads_and_sets_up_engine(self, tmp_path): - with patch("pysus.api.ducklake.client.CACHEPATH", tmp_path): - client = DuckLake() - - client._engine = None - - def run_sync(fn, *args, **kwargs): - return fn() - - with patch.object(client, "_download_catalog") as mock_dl: - with patch( - "pysus.api.ducklake.client.to_thread.run_sync", - side_effect=run_sync, - ): - with patch.object( - client, "_setup_engine", return_value=MagicMock() - ): - await client.connect() - mock_dl.assert_awaited_once_with( - client._catalog_local, - client._catalog_remote, - ) - assert client._Session is not None - assert client._engine is not None + client._catalog_adap = AsyncMock() + client._columns_adap = AsyncMock() + await client.connect(force=True) + client._catalog_adap.connect.assert_awaited_once_with(force=True) + client._columns_adap.connect.assert_awaited_once_with(force=True) class TestDuckLakeDownload: @pytest.mark.asyncio async def test_download_retry_then_success(self, tmp_path): - client = DuckLake() local_path = tmp_path / "test.db" - remote_path = "public/test.db" class FailingAsyncIter: def __aiter__(self): @@ -314,14 +173,14 @@ def __aiter__(self): async def __anext__(self): raise OSError("Connection dropped") - mock_client = MagicMock() - mock_client.__aenter__.return_value = mock_client + mock_http = MagicMock() + mock_http.__aenter__.return_value = mock_http httpx_patcher = patch( - "pysus.api.ducklake.client.httpx.AsyncClient", - return_value=mock_client, + "pysus.api.ducklake.functional.httpx.AsyncClient", + return_value=mock_http, ) sleep_patcher = patch( - "pysus.api.ducklake.client.sleep", new_callable=AsyncMock + "pysus.api.ducklake.functional.sleep", new_callable=AsyncMock ) first_stream_cm = MagicMock() @@ -342,21 +201,25 @@ async def success_iter(): second_resp.headers.get.return_value = "4" second_resp.aiter_bytes.return_value = success_iter() - mock_client.stream.side_effect = [first_stream_cm, second_stream_cm] + mock_http.stream.side_effect = [first_stream_cm, second_stream_cm] with httpx_patcher, sleep_patcher as mock_sleep: - await client.download(remote_path, local_path) + with patch( + "pysus.api.ducklake.functional.to_thread.run_sync", + side_effect=lambda fn, *a, **kw: fn(*a, **kw), + ): + from pysus.api.ducklake.functional import download_http + + await download_http("public/test.db", local_path) assert local_path.exists() assert local_path.read_bytes() == b"data" - assert mock_client.stream.call_count == 2 - mock_sleep.assert_awaited_once_with(1) + assert mock_http.stream.call_count == 2 + mock_sleep.assert_awaited_once_with(2) @pytest.mark.asyncio async def test_download_retry_exhausted_raises(self, tmp_path): - client = DuckLake() local_path = tmp_path / "test.db" - remote_path = "public/test.db" class FailingAsyncIter: def __aiter__(self): @@ -365,14 +228,14 @@ def __aiter__(self): async def __anext__(self): raise OSError("Connection dropped") - mock_client = MagicMock() - mock_client.__aenter__.return_value = mock_client + mock_http = MagicMock() + mock_http.__aenter__.return_value = mock_http httpx_patcher = patch( - "pysus.api.ducklake.client.httpx.AsyncClient", - return_value=mock_client, + "pysus.api.ducklake.functional.httpx.AsyncClient", + return_value=mock_http, ) sleep_patcher = patch( - "pysus.api.ducklake.client.sleep", new_callable=AsyncMock + "pysus.api.ducklake.functional.sleep", new_callable=AsyncMock ) stream_cm = MagicMock() @@ -382,23 +245,27 @@ async def __anext__(self): resp.headers.get.return_value = "4" resp.aiter_bytes.return_value = FailingAsyncIter() - mock_client.stream.return_value = stream_cm + mock_http.stream.return_value = stream_cm with httpx_patcher, sleep_patcher as mock_sleep: with pytest.raises(OSError, match="Connection dropped"): - await client.download(remote_path, local_path) + with patch( + "pysus.api.ducklake.functional.to_thread.run_sync", + side_effect=lambda fn, *a, **kw: fn(*a, **kw), + ): + from pysus.api.ducklake.functional import download_http + + await download_http("public/test.db", local_path) - assert mock_client.stream.call_count == 5 + assert mock_http.stream.call_count == 5 assert mock_sleep.await_count == 4 @pytest.mark.asyncio async def test_download_with_callback(self, tmp_path): - client = DuckLake() local_path = tmp_path / "test.db" - remote_path = "public/test.db" - mock_client = MagicMock() - mock_client.__aenter__.return_value = mock_client + mock_http = MagicMock() + mock_http.__aenter__.return_value = mock_http stream_cm = MagicMock() @@ -412,15 +279,23 @@ async def success_iter(): resp.headers.get.return_value = "10" resp.aiter_bytes.return_value = success_iter() - mock_client.stream.return_value = stream_cm + mock_http.stream.return_value = stream_cm callback = MagicMock() with patch( - "pysus.api.ducklake.client.httpx.AsyncClient", - return_value=mock_client, + "pysus.api.ducklake.functional.httpx.AsyncClient", + return_value=mock_http, ): - await client.download(remote_path, local_path, callback=callback) + with patch( + "pysus.api.ducklake.functional.to_thread.run_sync", + side_effect=lambda fn, *a, **kw: fn(*a, **kw), + ): + from pysus.api.ducklake.functional import download_http + + await download_http( + "public/test.db", local_path, callback=callback + ) callback.assert_any_call(5, 10) callback.assert_any_call(10, 10) @@ -431,9 +306,10 @@ class TestDuckLakeDownloadCatalog: async def test_download_catalog_size_match_skips_download(self, tmp_path): local_path = tmp_path / "catalog.duckdb" local_path.write_text("test") - remote_path = "public/catalog.duckdb" client = DuckLake() + client._catalog_adap.db_local = local_path + client._catalog_adap.db_remote = Path("public/catalog.duckdb") mock_http = MagicMock() mock_resp = MagicMock() @@ -443,20 +319,26 @@ async def test_download_catalog_size_match_skips_download(self, tmp_path): mock_http.__aenter__.return_value = mock_http with patch( - "pysus.api.ducklake.client.httpx.AsyncClient", + "pysus.api.ducklake.catalog.adapters.httpx.AsyncClient", return_value=mock_http, ): - with patch.object(client, "_download") as mock_dl: - await client._download_catalog(local_path, remote_path) + with patch( + "pysus.api.ducklake.catalog.adapters.download_http", + new_callable=AsyncMock, + ) as mock_dl: + await client._catalog_adap._download_catalog( + local_path, "public/catalog.duckdb" + ) mock_dl.assert_not_called() @pytest.mark.asyncio async def test_download_catalog_size_mismatch_downloads(self, tmp_path): local_path = tmp_path / "catalog.duckdb" local_path.write_text("test") - remote_path = "public/catalog.duckdb" client = DuckLake() + client._catalog_adap.db_local = local_path + client._catalog_adap.db_remote = Path("public/catalog.duckdb") mock_http = MagicMock() mock_resp = MagicMock() @@ -466,19 +348,25 @@ async def test_download_catalog_size_mismatch_downloads(self, tmp_path): mock_http.__aenter__.return_value = mock_http with patch( - "pysus.api.ducklake.client.httpx.AsyncClient", + "pysus.api.ducklake.catalog.adapters.httpx.AsyncClient", return_value=mock_http, ): - with patch.object(client, "_download") as mock_dl: - await client._download_catalog(local_path, remote_path) - mock_dl.assert_awaited_once_with(remote_path, local_path) + with patch( + "pysus.api.ducklake.catalog.adapters.download_http", + new_callable=AsyncMock, + ) as mock_dl: + await client._catalog_adap._download_catalog( + local_path, "public/catalog.duckdb" + ) + mock_dl.assert_awaited_once() @pytest.mark.asyncio async def test_download_catalog_local_not_exists(self, tmp_path): local_path = tmp_path / "catalog.duckdb" - remote_path = "public/catalog.duckdb" client = DuckLake() + client._catalog_adap.db_local = local_path + client._catalog_adap.db_remote = Path("public/catalog.duckdb") mock_http = MagicMock() mock_resp = MagicMock() @@ -488,19 +376,25 @@ async def test_download_catalog_local_not_exists(self, tmp_path): mock_http.__aenter__.return_value = mock_http with patch( - "pysus.api.ducklake.client.httpx.AsyncClient", + "pysus.api.ducklake.catalog.adapters.httpx.AsyncClient", return_value=mock_http, ): - with patch.object(client, "_download") as mock_dl: - await client._download_catalog(local_path, remote_path) - mock_dl.assert_awaited_once_with(remote_path, local_path) + with patch( + "pysus.api.ducklake.catalog.adapters.download_http", + new_callable=AsyncMock, + ) as mock_dl: + await client._catalog_adap._download_catalog( + local_path, "public/catalog.duckdb" + ) + mock_dl.assert_awaited_once() @pytest.mark.asyncio async def test_download_catalog_head_fails(self, tmp_path): local_path = tmp_path / "catalog.duckdb" - remote_path = "public/catalog.duckdb" client = DuckLake() + client._catalog_adap.db_local = local_path + client._catalog_adap.db_remote = Path("public/catalog.duckdb") mock_http = MagicMock() mock_resp = MagicMock() @@ -509,20 +403,26 @@ async def test_download_catalog_head_fails(self, tmp_path): mock_http.__aenter__.return_value = mock_http with patch( - "pysus.api.ducklake.client.httpx.AsyncClient", + "pysus.api.ducklake.catalog.adapters.httpx.AsyncClient", return_value=mock_http, ): - with patch.object(client, "_download") as mock_dl: - await client._download_catalog(local_path, remote_path) - mock_dl.assert_awaited_once_with(remote_path, local_path) + with patch( + "pysus.api.ducklake.catalog.adapters.download_http", + new_callable=AsyncMock, + ) as mock_dl: + await client._catalog_adap._download_catalog( + local_path, "public/catalog.duckdb" + ) + mock_dl.assert_awaited_once() @pytest.mark.asyncio async def test_download_catalog_head_no_content_length(self, tmp_path): local_path = tmp_path / "catalog.duckdb" local_path.write_text("test") - remote_path = "public/catalog.duckdb" client = DuckLake() + client._catalog_adap.db_local = local_path + client._catalog_adap.db_remote = Path("public/catalog.duckdb") mock_http = MagicMock() mock_resp = MagicMock() @@ -532,46 +432,17 @@ async def test_download_catalog_head_no_content_length(self, tmp_path): mock_http.__aenter__.return_value = mock_http with patch( - "pysus.api.ducklake.client.httpx.AsyncClient", + "pysus.api.ducklake.catalog.adapters.httpx.AsyncClient", return_value=mock_http, ): - with patch.object(client, "_download") as mock_dl: - await client._download_catalog(local_path, remote_path) - mock_dl.assert_awaited_once_with(remote_path, local_path) - - @pytest.mark.asyncio - async def test_download_catalog_oserror_on_local_stat(self, tmp_path): - local_path = tmp_path / "catalog.duckdb" - local_path.write_text("test") - remote_path = "public/catalog.duckdb" - - client = DuckLake() - - mock_http = MagicMock() - mock_resp = MagicMock() - mock_resp.headers = {"content-length": "999"} - mock_resp.raise_for_status = MagicMock() - mock_http.head = AsyncMock(return_value=mock_resp) - mock_http.__aenter__.return_value = mock_http - - stat_call_count = 0 - original_stat = type(local_path).stat - - def broken_stat(self, *args, **kwargs): - nonlocal stat_call_count - stat_call_count += 1 - if stat_call_count == 2: - raise OSError(errno.EACCES, "permission denied") - return original_stat(self, *args, **kwargs) - - with patch.object(type(local_path), "stat", broken_stat): with patch( - "pysus.api.ducklake.client.httpx.AsyncClient", - return_value=mock_http, - ): - with patch.object(client, "_download") as mock_dl: - await client._download_catalog(local_path, remote_path) - mock_dl.assert_awaited_once_with(remote_path, local_path) + "pysus.api.ducklake.catalog.adapters.download_http", + new_callable=AsyncMock, + ) as mock_dl: + await client._catalog_adap._download_catalog( + local_path, "public/catalog.duckdb" + ) + mock_dl.assert_awaited_once() class TestDuckLakeDownloadFile: @@ -579,7 +450,8 @@ class TestDuckLakeDownloadFile: async def test_download_file_invalid_type_raises(self): client = DuckLake() with pytest.raises( - ValueError, match="FTP File was not properly instantiated" + ValueError, + match="DuckLake File was not properly instantiated", ): await client.download( "not-a-file", @@ -601,51 +473,40 @@ async def test_download_file_valid(self, tmp_path): ) dataset = MagicMock(spec=DuckDataset) + adapter = MagicMock(spec=DatasetAdapter) + dataset.border = adapter f = File(dataset=dataset, record=record) # type: ignore output = tmp_path / "output.csv" - with patch.object(client, "_download") as mock_dl: + with patch( + "pysus.api.ducklake.client.download_http", + new_callable=AsyncMock, + ) as mock_dl: result = await client.download(f, output) - mock_dl.assert_awaited_once_with(record.path, output, callback=None) + mock_dl.assert_awaited_once_with( + remote_path=record.path, + local_path=output, + callback=None, + ) assert result == output class TestDuckLakeUploadCatalog: @pytest.mark.asyncio - async def test_upload_catalog_with_datasets(self, tmp_path): - client = DuckLake( - credentials=DuckLakeCredentials(access_key="ak", secret_key="sk") - ) - client._s3_client = MagicMock() - - ds = AsyncMock(spec=DuckDataset) - local_db = tmp_path / "catalog_test.duckdb" - local_db.write_text("data") - ds._catalog_local = local_db - ds._catalog_name = "catalog_test.duckdb" - - with patch.object( - DuckLake, "datasets", new=AsyncMock(return_value=[ds]) - ): - await client._upload_catalog() - client._s3_client.upload_file.assert_called_once_with( - str(local_db), client.bucket, ds._catalog_name - ) + async def test_upload_catalog_no_credentials_raises(self, tmp_path): + with patch("pysus.api.ducklake.catalog.adapters.CACHEPATH", tmp_path): + with patch("pathlib.Path.mkdir"): + adapter = CatalogAdapter(credentials=None) + with pytest.raises(PermissionError, match="Admin credentials"): + await adapter._upload_catalog() @pytest.mark.asyncio - async def test_upload_catalog_skips_missing_local(self, tmp_path): - client = DuckLake( - credentials=DuckLakeCredentials(access_key="ak", secret_key="sk") - ) - client._s3_client = MagicMock() - - ds = AsyncMock(spec=DuckDataset) + async def test_upload_catalog_missing_file(self, tmp_path): nonexistent = tmp_path / "nonexistent.duckdb" - ds._catalog_local = nonexistent - ds._catalog_name = "catalog_test.duckdb" - - with patch.object( - DuckLake, "datasets", new=AsyncMock(return_value=[ds]) - ): - await client._upload_catalog() - client._s3_client.upload_file.assert_not_called() + creds = DuckLakeCredentials(access_key="ak", secret_key="sk") + with patch("pysus.api.ducklake.catalog.adapters.CACHEPATH", tmp_path): + with patch("pathlib.Path.mkdir"): + adapter = CatalogAdapter(credentials=creds) + adapter.db_local = nonexistent + with pytest.raises(FileNotFoundError, match="catalog file not found"): + await adapter._upload_catalog() diff --git a/pysus/tests/api/ducklake/test_models.py b/pysus/tests/api/ducklake/test_models.py index 7d171acb..25650af6 100644 --- a/pysus/tests/api/ducklake/test_models.py +++ b/pysus/tests/api/ducklake/test_models.py @@ -57,13 +57,25 @@ def mock_client(): mc = create_autospec(DuckLake, instance=True) mc._datasets = [] + mc.download = AsyncMock() return mc @pytest.fixture def mock_dataset(mock_client, record): + adapter = MagicMock() + adapter._engine = None + adapter._session_factory = None + adapter.dataset_id = 1 + adapter.db_local = Path("/tmp/test.db") + adapter.db_remote = Path("test.db") + adapter.credentials = None + adapter.update_on_close = False + adapter.__aenter__.return_value = adapter + adapter.__aexit__ = AsyncMock() + adapter.connect = AsyncMock() with patch("pathlib.Path.mkdir"): - ds = DuckDataset(record=record, client=mock_client) + ds = DuckDataset(record=record, client=mock_client, adapter=adapter) return ds @@ -140,9 +152,9 @@ async def test_download_with_explicit_output( f = File(dataset=mock_dataset, record=catalog_file_record) output = Path("/tmp/out.csv") cb = MagicMock() - mock_dataset.client._download_file.return_value = output + mock_dataset.client.download.return_value = output result = await f._download(output=output, callback=cb) - mock_dataset.client._download_file.assert_awaited_once_with( + mock_dataset.client.download.assert_awaited_once_with( f, output, callback=cb ) assert result == output @@ -155,9 +167,9 @@ async def test_download_without_output( f = File(dataset=mock_dataset, record=catalog_file_record) expected = CACHEPATH / f.name - mock_dataset.client._download_file.return_value = expected + mock_dataset.client.download.return_value = expected result = await f._download() - mock_dataset.client._download_file.assert_awaited_once_with( + mock_dataset.client.download.assert_awaited_once_with( f, expected, callback=None ) assert result == expected @@ -224,162 +236,140 @@ async def test_verify_mismatching_hash(self, mock_dataset, tmp_path): class TestDuckDataset: def test_init(self, mock_client, record): + adapter = MagicMock() with patch("pathlib.Path.mkdir"): - ds = DuckDataset(record=record, client=mock_client) + ds = DuckDataset(record=record, client=mock_client, adapter=adapter) assert ds.record is record assert ds.client is mock_client - assert ds._catalog_name == "catalog_sinan.duckdb" + assert ds.border is adapter def test_repr(self, mock_dataset): - assert repr(mock_dataset) == "SINAN" + assert str(mock_dataset) == "sinan" def test_name(self, mock_dataset): assert mock_dataset.name == "sinan" def test_long_name(self, mock_dataset): - assert mock_dataset.long_name == "" + assert mock_dataset.long_name == "SINAN" def test_description(self, mock_dataset): - assert mock_dataset.description == "" + assert mock_dataset.description == "SINAN dataset" def test_catalog_path(self, mock_dataset): - from pysus import CACHEPATH - - expected = Path(CACHEPATH) / "ducklake" / "catalog_sinan.duckdb" - assert mock_dataset.catalog_path == expected + assert mock_dataset.border.db_local is not None @pytest.mark.asyncio async def test_connect_already_connected(self, mock_dataset, mock_client): - mock_dataset._engine = MagicMock() - mock_dataset._Session = MagicMock() + mock_dataset.border._engine = MagicMock() + mock_dataset.border._session_factory = MagicMock() await mock_dataset.connect(force=False) - mock_client._download.assert_not_called() + assert mock_dataset in mock_client._datasets @pytest.mark.asyncio async def test_connect_force_reconnects(self, mock_dataset, mock_client): - mock_dataset._engine = MagicMock() - mock_dataset._Session = MagicMock() - - def run_sync(fn, *args, **kwargs): - return fn() - - with patch( - "pysus.api.ducklake.models.to_thread.run_sync", - side_effect=run_sync, - ): - with patch.object( - mock_client, "_setup_engine", return_value=MagicMock() - ): - await mock_dataset.connect(force=True) - - mock_client._download.assert_awaited_once() + mock_dataset.border._engine = MagicMock() + mock_dataset.border._session_factory = MagicMock() + await mock_dataset.connect(force=True) + mock_dataset.border.connect.assert_awaited_once_with(force=True) + assert mock_dataset in mock_client._datasets @pytest.mark.asyncio async def test_connect_creates_session_if_missing( self, mock_dataset, mock_client ): - mock_dataset._engine = MagicMock() - mock_dataset._Session = None + mock_dataset.border._engine = MagicMock() + mock_dataset.border._session_factory = None + + async def _connect(*args, **kwargs): + mock_dataset.border._session_factory = MagicMock() + + mock_dataset.border.connect = _connect await mock_dataset.connect(force=False) - assert mock_dataset._Session is not None - mock_client._download.assert_not_called() + assert mock_dataset.border._session_factory is not None + assert mock_dataset in mock_client._datasets @pytest.mark.asyncio async def test_connect_full_path(self, mock_dataset, mock_client): - mock_dataset._engine = None - - def run_sync(fn, *args, **kwargs): - return fn() - - with patch( - "pysus.api.ducklake.models.to_thread.run_sync", - side_effect=run_sync, - ): - with patch.object( - mock_client, "_setup_engine", return_value=MagicMock() - ): - await mock_dataset.connect() - - mock_client._download.assert_awaited_once() - assert mock_dataset._engine is not None - assert mock_dataset._Session is not None + mock_dataset.border._engine = None + mock_dataset.border._session_factory = None + mock_dataset.border.connect = AsyncMock() + await mock_dataset.connect() + mock_dataset.border.connect.assert_awaited_once() assert mock_dataset in mock_client._datasets @pytest.mark.asyncio async def test_close_disposes_engine(self, mock_dataset): engine = MagicMock() - mock_dataset._engine = engine - with patch( - "pysus.api.ducklake.models.to_thread.run_sync", - side_effect=lambda fn, *a, **kw: fn(), - ): - await mock_dataset.close() + + async def _close(*args, **kwargs): + engine.dispose() + mock_dataset.border._engine = None + + mock_dataset.border._engine = engine + mock_dataset.border.close = _close + await mock_dataset.close() engine.dispose.assert_called_once() - assert mock_dataset._engine is None - assert mock_dataset._Session is None + assert mock_dataset.border._engine is None @pytest.mark.asyncio async def test_close_noop_when_no_engine(self, mock_dataset): - mock_dataset._engine = None + mock_dataset.border._engine = None + mock_dataset.border.close = AsyncMock() await mock_dataset.close() @pytest.mark.asyncio async def test_close_with_update_catalog(self, mock_dataset, mock_client): engine = MagicMock() - mock_dataset._engine = engine - mock_client._is_authenticated = True - - with patch.object(mock_dataset, "_upload_catalog") as mock_upload: - with patch( - "pysus.api.ducklake.models.to_thread.run_sync", - side_effect=lambda fn, *a, **kw: fn(), - ): - await mock_dataset.close(update_catalog=True) - engine.dispose.assert_called_once() - mock_upload.assert_awaited_once() + + mock_upload = AsyncMock() + + async def _close(*args, **kwargs): + engine.dispose() + mock_dataset.border._engine = None + await mock_upload() + + mock_dataset.border._engine = engine + mock_dataset.border.close = _close + await mock_dataset.close(update_catalog=True) + engine.dispose.assert_called_once() + mock_upload.assert_awaited_once() @pytest.mark.asyncio async def test_upload_catalog_no_credentials_raises(self, mock_dataset): - mock_dataset.client.credentials = None + mock_dataset.border.credentials = None + mock_dataset.border.db_local = Path("/tmp/test.db") + mock_dataset.border.db_remote = Path("test.db") + with pytest.raises(PermissionError, match="Admin credentials required"): - await mock_dataset._upload_catalog() + from pysus.api.ducklake.catalog.adapters import BaseAdapter + + await BaseAdapter._upload_catalog(mock_dataset.border) @pytest.mark.asyncio async def test_upload_catalog_success( self, mock_dataset, mock_client, tmp_path ): - mock_client.credentials = MagicMock() - mock_client._s3_client = MagicMock() - mock_client.bucket = "pysus" + mock_dataset.border.credentials = MagicMock() local_db = tmp_path / "catalog_sinan.duckdb" local_db.write_text("data") - mock_dataset._catalog_local = local_db - - def run_sync(fn, *args, **kwargs): - return fn() + mock_dataset.border.db_local = local_db + mock_dataset.border.db_remote = Path("public/catalog_sinan.duckdb") with patch( - "pysus.api.ducklake.models.to_thread.run_sync", - side_effect=run_sync, - ): - await mock_dataset._upload_catalog() + "pysus.api.ducklake.catalog.adapters.upload_s3", + new_callable=AsyncMock, + ) as mock_upload: + from pysus.api.ducklake.catalog.adapters import BaseAdapter - mock_client._s3_client.upload_file.assert_called_once_with( - str(local_db), - mock_client.bucket, - f"catalog_{mock_dataset.record.name.lower()}.duckdb", - ) + await BaseAdapter._upload_catalog(mock_dataset.border) + mock_upload.assert_awaited_once() @pytest.mark.asyncio async def test_query_no_filters(self, mock_dataset): mock_session = MagicMock() mock_session.__enter__.return_value = mock_session - mock_query = MagicMock() - mock_session.query.return_value = mock_query - mock_query.options.return_value = mock_query - mock_query.all.return_value = [] - - mock_dataset._Session = MagicMock(return_value=mock_session) + mock_dataset.border.get_session.return_value = mock_session + mock_session.scalars.return_value.all.return_value = [] def run_sync(fn, *args, **kwargs): return fn() @@ -396,14 +386,8 @@ def run_sync(fn, *args, **kwargs): async def test_query_with_all_filters(self, mock_dataset): mock_session = MagicMock() mock_session.__enter__.return_value = mock_session - mock_query = MagicMock() - mock_session.query.return_value = mock_query - mock_query.options.return_value = mock_query - mock_query.join.return_value = mock_query - mock_query.filter.return_value = mock_query - mock_query.all.return_value = [] - - mock_dataset._Session = MagicMock(return_value=mock_session) + mock_dataset.border.get_session.return_value = mock_session + mock_session.scalars.return_value.all.return_value = [] def run_sync(fn, *args, **kwargs): return fn() @@ -423,27 +407,16 @@ def run_sync(fn, *args, **kwargs): @pytest.mark.asyncio async def test_query_connects_if_no_session(self, mock_dataset): - mock_dataset._Session = None - mock_session = MagicMock() mock_session.__enter__.return_value = mock_session - mock_query = MagicMock() - mock_session.query.return_value = mock_query - mock_query.options.return_value = mock_query - mock_query.all.return_value = [] + mock_session.scalars.return_value.all.return_value = [] + mock_dataset.border.get_session.return_value = mock_session - async def _connect(*args, **kwargs): - mock_dataset._Session = MagicMock(return_value=mock_session) - - with patch.object( - DuckDataset, "connect", new=AsyncMock(side_effect=_connect) - ) as mock_connect: - with patch( - "pysus.api.ducklake.models.to_thread.run_sync", - side_effect=lambda fn, *a, **kw: fn(), - ): - await mock_dataset.query() - mock_connect.assert_awaited_once() + with patch( + "pysus.api.ducklake.models.to_thread.run_sync", + side_effect=lambda fn, *a, **kw: fn(), + ): + await mock_dataset.query() @pytest.mark.asyncio async def test_fetch_content_with_groups_and_files( @@ -451,10 +424,7 @@ async def test_fetch_content_with_groups_and_files( ): mock_session = MagicMock() mock_session.__enter__.return_value = mock_session - mock_query = MagicMock() - mock_session.query.return_value = mock_query - mock_query.options.return_value = mock_query - mock_query.filter.return_value = mock_query + mock_dataset.border.get_session.return_value = mock_session group_rec = Group( name="dengue", @@ -473,17 +443,11 @@ async def test_fetch_content_with_groups_and_files( origin_path="remote/dengue/data.csv", ) - dataset_rec = Dataset( - name="sinan", - long_name="SINAN", - description="SINAN dataset", - ) - dataset_rec.groups = [group_rec] - dataset_rec.files = [file_rec] - - mock_query.first.return_value = dataset_rec - - mock_dataset._Session = MagicMock(return_value=mock_session) + mock_session.scalars.return_value.all.side_effect = [ + [group_rec], + [file_rec], + ] + mock_session.expunge_all = MagicMock() def run_sync(fn, *args, **kwargs): return fn() @@ -504,13 +468,9 @@ def run_sync(fn, *args, **kwargs): async def test_fetch_content_no_dataset(self, mock_dataset): mock_session = MagicMock() mock_session.__enter__.return_value = mock_session - mock_query = MagicMock() - mock_session.query.return_value = mock_query - mock_query.options.return_value = mock_query - mock_query.filter.return_value = mock_query - mock_query.first.return_value = None - - mock_dataset._Session = MagicMock(return_value=mock_session) + mock_dataset.border.get_session.return_value = mock_session + mock_session.scalars.return_value.all.side_effect = [[], []] + mock_session.expunge_all = MagicMock() def run_sync(fn, *args, **kwargs): return fn() @@ -527,10 +487,7 @@ def run_sync(fn, *args, **kwargs): async def test_fetch_content_only_groups(self, mock_dataset): mock_session = MagicMock() mock_session.__enter__.return_value = mock_session - mock_query = MagicMock() - mock_session.query.return_value = mock_query - mock_query.options.return_value = mock_query - mock_query.filter.return_value = mock_query + mock_dataset.border.get_session.return_value = mock_session group_rec = Group( name="dengue", @@ -538,17 +495,11 @@ async def test_fetch_content_only_groups(self, mock_dataset): description="Dengue data", ) - dataset_rec = Dataset( - name="sinan", - long_name="SINAN", - description="Test", - ) - dataset_rec.groups = [group_rec] - dataset_rec.files = [] - - mock_query.first.return_value = dataset_rec - - mock_dataset._Session = MagicMock(return_value=mock_session) + mock_session.scalars.return_value.all.side_effect = [ + [group_rec], + [], + ] + mock_session.expunge_all = MagicMock() def run_sync(fn, *args, **kwargs): return fn() @@ -564,35 +515,17 @@ def run_sync(fn, *args, **kwargs): @pytest.mark.asyncio async def test_fetch_content_connects_if_no_session(self, mock_dataset): - mock_dataset._Session = None - mock_session = MagicMock() mock_session.__enter__.return_value = mock_session - mock_query = MagicMock() - mock_session.query.return_value = mock_query - mock_query.options.return_value = mock_query - mock_query.filter.return_value = mock_query - ds = Dataset( - name="sinan", - long_name="SINAN", - description="Test", - ) - ds.groups = [] - ds.files = [] - mock_query.first.return_value = ds - - async def _connect(*args, **kwargs): - mock_dataset._Session = MagicMock(return_value=mock_session) + mock_session.scalars.return_value.all.side_effect = [[], []] + mock_session.expunge_all = MagicMock() + mock_dataset.border.get_session.return_value = mock_session - with patch.object( - DuckDataset, "connect", new=AsyncMock(side_effect=_connect) - ) as mock_connect: - with patch( - "pysus.api.ducklake.models.to_thread.run_sync", - side_effect=lambda fn, *a, **kw: fn(), - ): - await mock_dataset._fetch_content() - mock_connect.assert_awaited_once() + with patch( + "pysus.api.ducklake.models.to_thread.run_sync", + side_effect=lambda fn, *a, **kw: fn(), + ): + await mock_dataset._fetch_content() # --------------------------------------------------------------------------- @@ -609,7 +542,7 @@ def test_long_name(self, mock_group): def test_long_name_fallback(self, mock_group): mock_group.record.long_name = None - assert mock_group.long_name == "acidentes" + assert mock_group.long_name == "None" def test_description(self, mock_group): assert mock_group.description == "Acidentes de trĂ¢nsito" diff --git a/pysus/tests/api/ftp/test_client.py b/pysus/tests/api/ftp/test_client.py index 3fb2bb51..73112aca 100644 --- a/pysus/tests/api/ftp/test_client.py +++ b/pysus/tests/api/ftp/test_client.py @@ -146,7 +146,7 @@ async def test_download_file_reconnects_on_failure(ftp_client): patch("pysus.api.ftp.client.FTP.connect") as mock_connect, patch("builtins.open", MagicMock()), ): - await ftp_client._download_file(mock_file, pathlib.Path("test.dbc")) + await ftp_client.download(mock_file, pathlib.Path("test.dbc")) assert mock_connect.call_count >= 1 @@ -166,7 +166,7 @@ def simulate_retrbinary(cmd, cb): mock_ftp_internal.retrbinary.side_effect = simulate_retrbinary with patch("builtins.open", MagicMock()): - await ftp_client._download_file( + await ftp_client.download( mock_file, pathlib.Path("test.dbc"), callback=callback ) callback.assert_called_once() @@ -186,7 +186,7 @@ def simulate_retrbinary(cmd, cb): mock_ftp_internal.retrbinary.side_effect = simulate_retrbinary with patch("builtins.open", MagicMock()): - await ftp_client._download_file(mock_file, pathlib.Path("test.dbc")) + await ftp_client.download(mock_file, pathlib.Path("test.dbc")) @pytest.mark.asyncio diff --git a/pysus/tests/api/ftp/test_models.py b/pysus/tests/api/ftp/test_models.py index 5014e63c..a24cd39a 100644 --- a/pysus/tests/api/ftp/test_models.py +++ b/pysus/tests/api/ftp/test_models.py @@ -11,7 +11,7 @@ def mock_client(): client = MagicMock(spec=FTP) client._list_directory = AsyncMock() - client._download_file = AsyncMock() + client.download = AsyncMock() return client @@ -107,8 +107,8 @@ async def test_file_download_no_output(mock_client, mock_dataset, tmp_path): cache_dir.mkdir(parents=True, exist_ok=True) with patch("pysus.api.ftp.models.CACHEPATH", cache_dir): await file._download() - mock_client._download_file.assert_called_once() - args, _ = mock_client._download_file.call_args + mock_client.download.assert_called_once() + args, _ = mock_client.download.call_args assert args[1] == cache_dir / "test.dbc" @@ -124,7 +124,7 @@ async def test_file_download_calls_client(mock_client, mock_dataset, tmp_path): dest = Path(tmp_path / "test.dbc") await file._download(output=dest) - mock_client._download_file.assert_called_once_with(file, dest, None) + mock_client.download.assert_called_once_with(file, dest, None) @pytest.mark.asyncio diff --git a/pysus/tests/api/test_client.py b/pysus/tests/api/test_client.py index 32c48cc3..f91974d0 100644 --- a/pysus/tests/api/test_client.py +++ b/pysus/tests/api/test_client.py @@ -511,9 +511,9 @@ async def test_query_with_client_filter(self, test_db_path): ds.name = "sinan" mock_file1 = MagicMock() - mock_file1.record.path = "public/data/ftp/somefile" + mock_file1.path = "public/data/ftp/somefile" mock_file2 = MagicMock() - mock_file2.record.path = "public/data/dadosgov/otherfile" + mock_file2.path = "public/data/dadosgov/otherfile" ds.query = AsyncMock(return_value=[mock_file1, mock_file2]) mock_ducklake.datasets = AsyncMock(return_value=[ds]) @@ -597,7 +597,7 @@ async def test_download_re_fetches_when_size_differs(self, test_db_path): return_value=mock_local, ): mock_client = AsyncMock() - mock_client._download_file = AsyncMock() + mock_client.download = AsyncMock() client._ftp = mock_client await client.download(mock_file) @@ -642,7 +642,7 @@ async def _slow_download(*args, **kwargs): ), ): mock_client = AsyncMock() - mock_client._download_file = _slow_download + mock_client.download = _slow_download client._ftp = mock_client with pytest.raises( @@ -691,7 +691,7 @@ async def test_download_with_ducklake_client(self, test_db_path): ), ): mock_ducklake = AsyncMock() - mock_ducklake._download_file = AsyncMock() + mock_ducklake.download = AsyncMock() client._ducklake = mock_ducklake result = await client.download(mock_file) @@ -740,7 +740,7 @@ async def test_download_with_dadosgov_client(self, test_db_path): ), ): mock_dadosgov = AsyncMock() - mock_dadosgov._download_file = AsyncMock() + mock_dadosgov.download = AsyncMock() client._dadosgov = mock_dadosgov result = await client.download(mock_file, token="test_token") @@ -1128,15 +1128,8 @@ async def test_aenter(self, test_db_path): client = PySUS(db_path=test_db_path) - with ( - patch.object( - DuckLake, "_download_catalog", new_callable=AsyncMock - ) as mock_download, - patch.object(PySUS, "_attach_client_catalog") as mock_attach, - ): + with patch.object(DuckLake, "connect", new_callable=AsyncMock): await client.__aenter__() assert client._ducklake is not None - mock_download.assert_called_once() - mock_attach.assert_called_once() await client.__aexit__(None, None, None) diff --git a/pysus/tests/api/test_databases.py b/pysus/tests/api/test_databases.py index 8a398038..f78af6b1 100644 --- a/pysus/tests/api/test_databases.py +++ b/pysus/tests/api/test_databases.py @@ -203,7 +203,7 @@ def test_fetch_data_multiple_years(self): years = [2023, 2024] _fetch_data(dataset="sinan", year=years, show_progress=False) - assert mock_pysus.query.call_count == 2 + assert mock_pysus.query.call_count == 1 def test_fetch_data_with_group_filter(self): with patch("pysus.api._impl.databases.PySUS") as mock_pysus_class: @@ -253,6 +253,7 @@ def test_fetch_data_empty_result(self): dataset="sinan", year=2024, show_progress=False, + as_dataframe=True, ) assert isinstance(result, pd.DataFrame) @@ -299,6 +300,7 @@ def test_fetch_data_no_files(self): dataset="sinan", year=2024, show_progress=True, + as_dataframe=True, ) assert isinstance(result, pd.DataFrame) @@ -308,7 +310,11 @@ def test_fetch_data_no_files(self): def test_fetch_data_with_progress(self): with ( patch("pysus.api._impl.databases.PySUS") as mock_pysus_class, - patch("pysus.api._impl.databases.tqdm", new=lambda x, **kw: x), + patch( + "pysus.api._impl.databases.tqdm.gather", + new_callable=AsyncMock, + return_value=[MagicMock(), MagicMock()], + ) as mock_tqdm_gather, ): mock_pysus = MagicMock() enter_mock = AsyncMock(return_value=mock_pysus) @@ -326,7 +332,10 @@ def test_fetch_data_with_progress(self): _fetch_data(dataset="sinan", year=2024, show_progress=True) - assert mock_pysus.download.call_count == 2 + assert mock_tqdm_gather.called + + called_args = mock_tqdm_gather.call_args[0] + assert len(called_args) == 2 class TestFetchDataRunningLoop: diff --git a/pysus/tests/api/test_metadata.py b/pysus/tests/api/test_metadata.py index 75329381..7ea38ad1 100644 --- a/pysus/tests/api/test_metadata.py +++ b/pysus/tests/api/test_metadata.py @@ -1,14 +1,9 @@ -import builtins -from unittest.mock import patch - from pysus.api.metadata.models import ( Column, Dataset, DatasetGroup, File, FileMeta, - lookup_column_meta, - pick_description, ) from pysus.api.metadata.report import Columns, Footer, Header from pysus.api.types import VARCHAR @@ -28,48 +23,17 @@ def test_footer_instantiation(self): assert isinstance(f, Footer) -class TestLookupColumnMeta: - def test_found_returns_dict(self): - meta = lookup_column_meta("ABAND") - assert meta is not None - assert isinstance(meta, dict) - - def test_not_found_returns_none(self): - meta = lookup_column_meta("NONEXISTENT_COLUMN_XYZ") - assert meta is None - - def test_import_error_returns_none(self): - with patch.object( - builtins, "__import__", side_effect=ImportError("mock") - ): - result = lookup_column_meta("ABAND") - assert result is None - - -class TestPickDescription: - def test_none_meta_returns_empty(self): - assert pick_description(None) == "" - - def test_non_empty_value_returns_first_value(self): - meta = {"sinan": "Some description"} - assert pick_description(meta) == "Some description" - - def test_empty_dict_returns_empty(self): - assert pick_description({}) == "" - - def test_all_empty_values_returns_empty(self): - meta = {"sinan": "", "sih": ""} - assert pick_description(meta) == "" - - class TestColumnFromSchema: def test_from_schema_creates_column(self): - col = Column.from_schema("ABAND", VARCHAR) + col = Column.from_schema( + "ABAND", VARCHAR, description="Abandonment info" + ) assert isinstance(col, Column) assert col.name == "ABAND" assert col.dtype == VARCHAR + assert col.description == "Abandonment info" - def test_from_schema_unknown_column(self): + def test_from_schema_default_description(self): col = Column.from_schema("NONEXISTENT_COLUMN_XYZ", VARCHAR) assert col.name == "NONEXISTENT_COLUMN_XYZ" assert col.description == ""