From 3d3912a05f61686cf3a2108e669710145089238e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Sun, 30 Aug 2026 20:27:41 -0300 Subject: [PATCH 01/18] feat: split the sources in modules so it become very verbose the source of the database --- pysus/api/ducklake/catalog/adapters.py | 42 +- pysus/api/ducklake/client.py | 19 +- pysus/api/ducklake/functional.py | 12 +- pysus/api/ducklake/models.py | 3 + pysus/cli/management.py | 5 +- pysus/management/compare.py | 27 +- pysus/management/records.py | 69 +++ pysus/management/sync.py | 431 ++++++++++++++++--- pysus/tests/api/ducklake/test_adapters.py | 55 +++ pysus/tests/api/ducklake/test_client.py | 15 + pysus/tests/management/test_sync_internal.py | 289 ++++++++++++- 11 files changed, 872 insertions(+), 95 deletions(-) diff --git a/pysus/api/ducklake/catalog/adapters.py b/pysus/api/ducklake/catalog/adapters.py index 144af7ac..76c3ffa3 100644 --- a/pysus/api/ducklake/catalog/adapters.py +++ b/pysus/api/ducklake/catalog/adapters.py @@ -337,8 +337,19 @@ async def _upload_catalog(self) -> None: async def close(self, update: bool = False) -> None: if update and self._local_dirty: - await self._upload_catalog() - self._local_dirty = False + try: + await self._upload_catalog() + self._local_dirty = False + except ( + Exception + ) as exc: # noqa: BLE001 — catalog sync is best-effort + import logging + + logging.getLogger(__name__).warning( + "catalog upload failed for %s (kept dirty): %s", + self.db_remote, + exc, + ) # The engine is shared process-wide per database file and is # never disposed here: DuckDB tears down the in-process database @@ -349,6 +360,33 @@ async def close(self, update: bool = False) -> None: self._engine = None self._session_factory = None + async def sync(self, update: bool = False) -> None: + """Upload the dirty catalog file, keeping the adapter attached. + + Unlike :meth:`close`, the engine and session factory are left in + place, so the local database can keep serving writes right after + the upload. This is the checkpoint path: the local file we just + wrote is authoritative, so re-downloading it (as ``close`` + + ``connect`` does) would be a needless — and hang-prone — + network round-trip. The engine is *not* torn down, exactly as in + ``close``: the shared registry releases it only via + :func:`_dispose_shared` right before a file is replaced. + """ + if update and self._local_dirty: + try: + await self._upload_catalog() + self._local_dirty = False + except ( + Exception + ) as exc: # noqa: BLE001 — catalog sync is best-effort + import logging + + logging.getLogger(__name__).warning( + "catalog upload failed for %s (kept dirty): %s", + self.db_remote, + exc, + ) + def __del__(self) -> None: if not hasattr(self, "_engine") or not self._engine: return diff --git a/pysus/api/ducklake/client.py b/pysus/api/ducklake/client.py index 89fceafd..ffbde73f 100644 --- a/pysus/api/ducklake/client.py +++ b/pysus/api/ducklake/client.py @@ -108,6 +108,9 @@ def get_dataset_adapter(self, name: str) -> DatasetAdapter: "close": lambda self_, update_catalog=None: ( self_.adapter.close(update=bool(update_catalog)) ), + "sync": lambda self_, update=False: ( + self_.adapter.sync(update=bool(update)) + ), }, )() ) @@ -182,17 +185,19 @@ async def close(self, update_catalog: bool | None = None) -> None: await self._columns_adap.close(update=should_update) async def flush_catalogs(self, update: bool = True) -> None: - """Upload dirty catalogs (if *update*) and reopen the adapters. + """Upload dirty catalogs (if *update*) without re-downloading them. Long-running writers use this to checkpoint: modified local - databases are pushed to S3 and the adapters are reconnected. + databases are pushed to S3 while the local files stay attached. + The fresh local copy just written is authoritative — there is no + need to tear the adapters down and re-download the same catalog + over the network (which was the source of previously observed + hangs at checkpoint time). """ for ds in self._datasets: - await ds.close(update_catalog=update) - await self._catalog_adap.close(update=update) - await self._columns_adap.close(update=update) - await self._catalog_adap.connect() - await self._columns_adap.connect() + await ds.sync(update=update) + await self._catalog_adap.sync(update=update) + await self._columns_adap.sync(update=update) async def download( self, diff --git a/pysus/api/ducklake/functional.py b/pysus/api/ducklake/functional.py index ef7eb020..26847d62 100644 --- a/pysus/api/ducklake/functional.py +++ b/pysus/api/ducklake/functional.py @@ -3,7 +3,7 @@ import boto3 import httpx -from anyio import sleep, to_thread +from anyio import fail_after, sleep, to_thread from botocore import UNSIGNED from botocore.config import Config from pysus.api import types @@ -194,7 +194,12 @@ def _get_client_args(): if access_key and secret_key: args["aws_access_key_id"] = access_key args["aws_secret_access_key"] = secret_key - args["config"] = Config(signature_version="s3v4") + args["config"] = Config( + signature_version="s3v4", + connect_timeout=20, + read_timeout=120, + retries={"max_attempts": 3, "mode": "standard"}, + ) else: args["config"] = Config(signature_version=UNSIGNED) return args @@ -220,7 +225,8 @@ def boto_callback(bytes_amount): try: client_args = _get_client_args() total_size = local_path.stat().st_size - await to_thread.run_sync(_upload, client_args, total_size) + with fail_after(600): + await to_thread.run_sync(_upload, client_args, total_size) return except Exception as e: # noqa if attempt < max_retries - 1: diff --git a/pysus/api/ducklake/models.py b/pysus/api/ducklake/models.py index 58fb8100..37ef371a 100644 --- a/pysus/api/ducklake/models.py +++ b/pysus/api/ducklake/models.py @@ -146,6 +146,9 @@ async def close(self, update_catalog: bool | None = None): ) await self.adapter.close(update=should_update) + async def sync(self, update: bool = False) -> None: + await self.adapter.sync(update=update) + async def query( self, group: str | list[str] | None = None, diff --git a/pysus/cli/management.py b/pysus/cli/management.py index b4eb1f19..18d01f7f 100644 --- a/pysus/cli/management.py +++ b/pysus/cli/management.py @@ -20,7 +20,7 @@ import typer from pysus import CACHEPATH -from pysus.management.records import load_journal_keys +from pysus.management.records import load_journal_keys, load_journal_origins app = typer.Typer(help="Manage the S3 databases (dev only, not on PyPI)") @@ -195,8 +195,10 @@ async def _run_check() -> None: journal = _journal_path(resume, reupload_before) resume_keys = set() + resume_origins = None if journal is not None and journal.exists(): resume_keys = load_journal_keys(journal) + resume_origins = load_journal_origins(journal) counts: dict[str, int] = {} @@ -240,6 +242,7 @@ async def _run() -> dict[str, int]: checkpoint_every=checkpoint_every, on_outcome=on_outcome, resume=resume_keys or None, + resume_origins=resume_origins or None, journal=journal, ) summary = report.summary() diff --git a/pysus/management/compare.py b/pysus/management/compare.py index b81f0d3c..94a9ddbc 100644 --- a/pysus/management/compare.py +++ b/pysus/management/compare.py @@ -97,15 +97,28 @@ def _dedup_origin_formats( DadosGov publishes the same data as csv/json/xml triplets: when several records from the same origin share size-agnostic identity, prefer the format highest in ``FORMAT_PREFERENCE`` (csv first). + + Ducklake (S3 mirror) records are keyed by ``(origin, mirror origin)`` + instead: the bucket may hold independent mirrors per origin path + (``public/data/ftp/...`` and ``public/data/dadosgov/...``), and both + must survive so per-origin mirror decisions can see them. """ - preferred: dict[str, FileRecord] = {} + preferred: dict[tuple[str, str], FileRecord] = {} for record in records: - fmt = (record.format or "").lower() - current = preferred.get(record.origin) - if current is None or _format_rank(fmt) < _format_rank( - current.format or "" - ): - preferred[record.origin] = record + if record.origin == "ducklake": + from .records import origin_from_s3_key + + dedup_key = ( + record.origin, + origin_from_s3_key(record.path) or "", + ) + else: + dedup_key = (record.origin, "") + current = preferred.get(dedup_key) + if current is None or _format_rank( + record.format or "" + ) < _format_rank(current.format or ""): + preferred[dedup_key] = record return list(preferred.values()) def pick( diff --git a/pysus/management/records.py b/pysus/management/records.py index 6dc20165..70e5b6e7 100644 --- a/pysus/management/records.py +++ b/pysus/management/records.py @@ -99,6 +99,16 @@ def parquet_key(name: str) -> str: _KEY_SEGMENT_ORDER = ("group", "year", "month", "state") +def origin_from_s3_key(path: str | None) -> str | None: + """Return the origin segment of a ``public/data/...`` object key.""" + if not path: + return None + parts = path.split("/") + if len(parts) < 3 or parts[0] != "public" or parts[1] != "data": + return None + return parts[2].lower() + + def compose_s3_key( origin: str, dataset: str, @@ -313,6 +323,25 @@ def best_record( def is_on_s3(self) -> bool: return "ducklake" in self.origins + def mirror_for_origin(self, origin: str) -> FileRecord | None: + """Return the S3 mirror artifact copied from *origin*, if any. + + Ducklake records are the S3 mirrors of origin files; their + ``path`` is the ``public/data//...`` object key. Mirroring + is now per-origin (an FTP mirror may coexist with a DadosGov + mirror of the same logical file), so a comparison locates the + mirror belonging to a specific origin by path segment. + """ + origin = origin.lower() + for record in self.records: + if record.origin != "ducklake": + continue + if not record.path: + continue + if origin_from_s3_key(record.path) == origin: + return record + return None + @property def only_on_dadosgov(self) -> bool: return self.origins == {"dadosgov"} @@ -495,6 +524,7 @@ def outcome_to_journal( "month": outcome.key.month, "state": outcome.key.state, "stem": outcome.key.stem, + "origin": outcome.origin, "status": outcome.status, "ts": ts or datetime.now().isoformat(timespec="seconds"), } @@ -548,3 +578,42 @@ def load_journal_keys(path: Path) -> set[IdentityKey]: except (KeyError, TypeError): continue return keys + + +def load_journal_origins(path: Path) -> dict[IdentityKey, set[str]]: + """Return the origins uploaded per key in a prior run. + + Complements :func:`load_journal_keys` with the origin attribute: a key + uploaded from FTP must not suppress the DadosGov mirror of the same + logical file, and vice versa. Lines lacking an ``origin`` field (written + by pre-origin-aware runs) are treated as covering every origin. + """ + origins: dict[IdentityKey, set[str]] = {} + if not path.exists(): + return origins + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + data = json.loads(line) + except json.JSONDecodeError: + continue + if data.get("status") != "uploaded": + continue + try: + key = IdentityKey( + dataset=data["dataset"], + group=data.get("group"), + year=data.get("year"), + month=data.get("month"), + state=data.get("state"), + stem=data["stem"], + ) + except (KeyError, TypeError): + continue + origin = data.get("origin") + origins.setdefault(key, set()).add( + origin if isinstance(origin, str) else "*" + ) + return origins diff --git a/pysus/management/sync.py b/pysus/management/sync.py index 0d73d7e9..3f8a2b36 100644 --- a/pysus/management/sync.py +++ b/pysus/management/sync.py @@ -24,6 +24,7 @@ from datetime import datetime from logging import error, info, warning from pathlib import Path +from time import monotonic as _monotonic from typing import TYPE_CHECKING, Any from uuid import uuid4 @@ -67,6 +68,14 @@ #: pipeline indefinitely. _DOWNLOAD_TIMEOUT: float = 600.0 # 10 minutes +#: Abort a transfer (download or conversion) when it makes no progress +#: for this long (seconds). Guards against half-open connections and +#: deadlocked parsers that never raise but never complete either. +_STALL_TIMEOUT: float = 300.0 # 5 minutes + +#: How often the staleness watchdog polls for progress (seconds). +_STALL_POLL: float = 5.0 + #: Default per-file conversion timeout (seconds). CSV→parquet on a #: 1 GB file typically takes < 60 s; anything longer likely means a #: pathological row layout or a bug. @@ -88,6 +97,13 @@ #: at a time. Keeps peak RAM/disk usage bounded on low-resource hosts. _CONCURRENCY_BUDGET: int = 2500 * 1024 * 1024 # 2.5 GB +#: Deadline for a full catalog checkpoint flush (minutes). The catalog +#: writer awaits this inline while it is the only consumer of the bounded +#: ``write_queue``; a hang here would fill the queue and freeze every +#: producer, so the flush is soft-capped and retried by the next +#: checkpoint (or the final teardown flush). +_FLUSH_TIMEOUT: float = 600.0 # 10 minutes + class _WeightGate: """Bound how many pipeline slots are in use by byte weight. @@ -136,6 +152,65 @@ async def adjust(self, old_weight: int, new_weight: int) -> int: return new_weight +class _StallWatch: + """Abort a long-running transfer when it makes no forward progress. + + ``poke()`` must be called whenever the transfer advances (bytes + written, rows parsed, …). A background task polls the last poke and + raises ``TimeoutError`` once ``_STALL_TIMEOUT`` seconds pass with no + progress — catching dead downloads and deadlocked parsers that an + absolute ``fail_after`` deadline alone cannot distinguish from slow + but healthy transfers. + """ + + def __init__( + self, + name: str, + timeout: float = _STALL_TIMEOUT, + poll: float = _STALL_POLL, + ) -> None: + self._name = name + self._timeout = timeout + self._poll = poll + self._last_poke = _monotonic() + + def poke(self) -> None: + """Record progress; a no-op unless something moved.""" + self._last_poke = _monotonic() + + async def watch(self) -> None: + """Run until aborted: raise when progress stalls for too long.""" + while True: + await asyncio.sleep(self._poll) + if _monotonic() - self._last_poke > self._timeout: + raise TimeoutError( + f"{self._name}: no progress for {self._timeout:.0f}s" + ) + + async def run(self, coro) -> Any: + """Await *coro*, cancelling it if progress stalls meanwhile. + + Returns the coroutine's result, or raises ``TimeoutError`` and + cancels *coro* when the transfer stalls past the timeout. + """ + task = asyncio.ensure_future(coro) + watcher = asyncio.ensure_future(self.watch()) + try: + done, _ = await asyncio.wait( + (task, watcher), return_when=asyncio.FIRST_COMPLETED + ) + finally: + if not watcher.done(): + watcher.cancel() + if task in done: + return task.result() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + raise TimeoutError( + f"{self._name}: no progress for {self._timeout:.0f}s" + ) + + class SyncEngine: """Orchestrates inventory → compare → download → parquet → catalog.""" @@ -382,9 +457,19 @@ async def upload_file( raise RuntimeError( f"{file.basename}: cannot convert to parquet" ) - parquet_file = await local_file.to_parquet( - callback=callback, - ) + from anyio import fail_after + + watch = _StallWatch(file.basename) + + def _upload_callback(processed: int, total: int) -> None: + watch.poke() + if callback is not None: + callback(processed, total) + + with fail_after(_CONVERT_TIMEOUT): + parquet_file = await watch.run( + local_file.to_parquet(callback=_upload_callback) + ) parquet_digest = sha256_of(parquet_file.path) if existing and not force: @@ -535,18 +620,25 @@ async def _download_once( ftp = getattr(client, "ftp", None) assert ftp is not None remote_path = str(file.path) + watch = _StallWatch(file.basename) - def _retr(): + def _retr() -> int: total = ftp.size(remote_path) or 0 with open(output, "wb") as f: - ftp.retrbinary( - f"RETR {remote_path}", lambda chunk: f.write(chunk) - ) + + def _write(chunk: bytes) -> int: + written = f.write(chunk) + watch.poke() + return written + + ftp.retrbinary(f"RETR {remote_path}", _write) return total try: with anyio.fail_after(_DOWNLOAD_TIMEOUT): - await to_thread.run_sync(_retr) + await watch.run( + to_thread.run_sync(_retr, abandon_on_cancel=True) + ) return output except Exception: # noqa try: @@ -559,17 +651,31 @@ def _retr(): raise if ftp is not None: remote_path = str(file.path) + watch = _StallWatch(file.basename) - def _direct_retr(): + def _direct_retr() -> None: with open(output, "wb") as f: - ftp.retrbinary( - f"RETR {remote_path}", lambda chunk: f.write(chunk) - ) + + def _write(chunk: bytes) -> int: + written = f.write(chunk) + watch.poke() + return written + + ftp.retrbinary(f"RETR {remote_path}", _write) with anyio.fail_after(_DOWNLOAD_TIMEOUT): - await to_thread.run_sync(_direct_retr) + await watch.run( + to_thread.run_sync(_direct_retr, abandon_on_cancel=True) + ) return output - return await file._download(output=output) + watch = _StallWatch(file.basename) + + def _download_callback(processed: int, total: int) -> None: + watch.poke() + + return await watch.run( + file._download(output=output, callback=_download_callback) + ) @staticmethod def _cleanup_local(path: Path) -> None: @@ -650,6 +756,7 @@ async def run( origins: tuple[str, ...] | None = None, reupload_before: datetime | None = None, resume: set[IdentityKey] | None = None, + resume_origins: dict[IdentityKey, set[str]] | None = None, journal: Path | None = None, ) -> SyncReport: """Run the full pipeline and return a :class:`SyncReport`. @@ -669,6 +776,11 @@ async def run( set of :class:`~pysus.management.records.IdentityKey` already completed in a prior run (e.g. loaded from that journal): those files are skipped instead of re-downloaded and re-converted. + ``resume_origins`` (optional) restricts that to per-origin + coverage: a key uploaded from FTP must not suppress the DadosGov + mirror of the same logical file, and vice versa. When it is + ``None`` the resume set is interpreted as covering every origin + (legacy journals written before per-origin mirroring). With ``dry_run=True`` no downloads, uploads or catalog writes happen: every file that would be updated is reported with the @@ -743,22 +855,27 @@ async def collect_with_retry(origin: str, datasets=None, **kwargs): await self._fix_misparsed_metadata(records["ducklake"]) parallel: list[tuple[FileComparison, FileRecord]] = [] + vacinacao: list[tuple[FileComparison, FileRecord]] = [] for comparison in comparisons: - if comparison.is_on_s3 and not ( - force - or self._s3_is_stale(comparison) - or self._cataloged_before(comparison, reupload_before) - ): - emit( - SyncOutcome( - key=comparison.key, - origin="ducklake", - status="skipped", + file_records = [ + r + for r in comparison.records + if r.origin != "ducklake" and r.file is not None + ] + if not file_records: + if comparison.is_on_s3 and not ( + force + or self._s3_is_stale(comparison) + or self._cataloged_before(comparison, reupload_before) + ): + emit( + SyncOutcome( + key=comparison.key, + origin="ducklake", + status="skipped", + ) ) - ) - continue - record = self._pick_source(comparison) - if record is None: + continue if dry_run: emit( SyncOutcome( @@ -777,30 +894,64 @@ async def collect_with_retry(origin: str, datasets=None, **kwargs): ) emit(outcome) continue - if dry_run: - emit( - SyncOutcome( - key=comparison.key, - origin=record.origin, - status="needs_update", - detail=f"{self._label(comparison)} ({record.origin})", + + # Mirroring is per-origin: every origin that carries a file is + # mirrored independently (an FTP artifact coexists with the + # DadosGov artifact of the same logical file). A file is only + # skipped when *its* origin's mirror already exists and is + # current — never because a different origin's mirror does. + for record in file_records: + origin = record.origin + mirror = comparison.mirror_for_origin(origin) + if mirror is not None and not ( + force + or self._s3_origin_stale(comparison, origin, mirror) + or self._mirror_cataloged_before(mirror, reupload_before) + ): + emit( + SyncOutcome( + key=comparison.key, + origin=origin, + status="skipped", + ) ) - ) - continue - if resume and comparison.key in resume: - emit( - SyncOutcome( - key=comparison.key, - origin="ducklake", - status="skipped", - detail=( - "already processed in a prior run: " - f"{self._label(comparison)}" - ), + continue + if dry_run: + emit( + SyncOutcome( + key=comparison.key, + origin=origin, + status="needs_update", + detail=(f"{self._label(comparison)} ({origin})"), + ) ) - ) - continue - parallel.append((comparison, record)) + continue + if ( + resume + and comparison.key in resume + and self._resume_covers( + resume_origins, comparison.key, origin + ) + ): + emit( + SyncOutcome( + key=comparison.key, + origin=origin, + status="skipped", + detail=( + "already processed in a prior run: " + f"{self._label(comparison)}" + ), + ) + ) + continue + # VACINACAO endpoints report size 0 and stream multi-GB + # CSVs; move them to a trailing serial phase so they never + # stall the concurrent drain of the small files. + if comparison.key.dataset == "VACINACAO": + vacinacao.append((comparison, record)) + continue + parallel.append((comparison, record)) # Process smallest files first: tiny artifacts are cheap to # convert and upload, so they drain fast with full parallelism, @@ -830,6 +981,10 @@ async def collect_with_retry(origin: str, datasets=None, **kwargs): raw_queue: asyncio.Queue = asyncio.Queue(maxsize=workers * 2) write_queue: asyncio.Queue = asyncio.Queue(maxsize=workers * 2) + # Files that fail during the concurrent drain are retried once at + # the very end of the run (bottom of the queue) instead of being + # marked failed mid-flight on a transient error. + retry_items: list[tuple[FileComparison, FileRecord]] = [] async def ftp_downloader( client: Any, items: list[tuple[FileComparison, FileRecord]] @@ -855,7 +1010,7 @@ async def ftp_downloader( except Exception as exc: # noqa if weight: await gate.release(weight) - await raw_queue.put((comparison, record, None, str(exc), 0)) + retry_items.append((comparison, record)) async def raw_processor() -> None: while True: @@ -867,7 +1022,7 @@ async def raw_processor() -> None: try: raw_path = raw if err is not None: - await write_queue.put((comparison, record, None, err)) + retry_items.append((comparison, record)) continue payload = await self._convert_and_upload( record.file, raw, callback=callback @@ -875,7 +1030,7 @@ async def raw_processor() -> None: await write_queue.put((comparison, record, payload, None)) except Exception as exc: # noqa comparison, record, _, _, _ = entry - await write_queue.put((comparison, record, None, str(exc))) + retry_items.append((comparison, record)) finally: if weight: await gate.release(weight) @@ -907,7 +1062,7 @@ async def gov_worker() -> None: ) await write_queue.put((comparison, record, payload, None)) except Exception as exc: # noqa - await write_queue.put((comparison, record, None, str(exc))) + retry_items.append((comparison, record)) finally: if weight: await gate.release(weight) @@ -1031,6 +1186,61 @@ async def catalog_writer() -> None: for _ in processor_tasks: await raw_queue.put(None) await asyncio.gather(*processor_tasks) + # VACINACAO endpoints stream multi-GB CSVs (size reported as 0); + # drain them serially, only after the main concurrent pipeline + # has finished, so they cannot stall the small-file drain. + for comparison, record in vacinacao: + weight = gate.weight_of(record.file.size) + try: + await gate.acquire(weight) + except Exception: # noqa: BLE001 — budget is advisory + weight = 0 + raw_path = None + try: + raw = await self._download_raw_with_retry(record.file) + raw_path = raw + payload = await self._convert_and_upload( + record.file, raw, callback=callback + ) + await write_queue.put((comparison, record, payload, None)) + except Exception as exc: # noqa + await write_queue.put((comparison, record, None, str(exc))) + finally: + if weight: + await gate.release(weight) + if raw_path: + self._cleanup_local(raw_path) + # Files that failed during the concurrent drain get one last + # serial retry, so transient errors are retried with a fresh + # download instead of being marked failed mid-run. + for i, (comparison, record) in enumerate(retry_items): + weight = gate.weight_of(record.file.size) + try: + await gate.acquire(weight) + except Exception: # noqa: BLE001 — budget is advisory + weight = 0 + raw_path = None + try: + ftp_client = ( + ftp_pool[i % len(ftp_pool)] + if record.origin == "ftp" and ftp_pool + else None + ) + raw = await self._download_raw_with_retry( + record.file, ftp_client=ftp_client + ) + raw_path = raw + payload = await self._convert_and_upload( + record.file, raw, callback=callback + ) + await write_queue.put((comparison, record, payload, None)) + except Exception as exc: # noqa + await write_queue.put((comparison, record, None, str(exc))) + finally: + if weight: + await gate.release(weight) + if raw_path: + self._cleanup_local(raw_path) for _ in range(writers_total): await write_queue.put(None) await writer_task @@ -1109,8 +1319,18 @@ async def _convert_and_upload( local_file = await ExtensionFactory.instantiate(raw_path) if not hasattr(local_file, "to_parquet"): raise RuntimeError(f"{file.basename}: cannot convert to parquet") + + watch = _StallWatch(file.basename) + + def _convert_callback(processed: int, total: int) -> None: + watch.poke() + if callback is not None: + callback(processed, total) + with anyio.fail_after(_CONVERT_TIMEOUT): - parquet_file = await local_file.to_parquet(callback=callback) + parquet_file = await watch.run( + local_file.to_parquet(callback=_convert_callback) + ) try: parquet_digest = await to_thread.run_sync( sha256_of, parquet_file.path @@ -1270,22 +1490,26 @@ async def _dedupe_s3_artifacts( self, ducklake_records: list[FileRecord], ) -> None: - """Keep only the most updated S3 artifact per logical file. - - Legacy ETL runs stored the same logical file under multiple - origin paths (e.g. ``public/data/ftp/...`` and - ``public/data/dadosgov/...``). Grouping runs on the raw S3 - records — the comparator collapses same-origin entries, which - would hide these duplicates. The newest artifact (by source - modification date) survives; the others are deleted from the - bucket and the catalog. + """Keep only the most updated S3 artifact per (logical file, origin). + + Mirroring is per-origin: an FTP mirror (``public/data/ftp/...``) + and a DadosGov mirror (``public/data/dadosgov/...``) of the same + logical file are independent artifacts and must both survive, so + grouping always includes the origin segment of the object key. + Duplicates are only removed *within* one origin (e.g. a legacy + flat object and its relocated hierarchical twin sharing a path). + The newest artifact (by source modification date) survives; the + others are deleted from the bucket and the catalog. """ import boto3 from botocore.config import Config + from .records import origin_from_s3_key + groups: dict[tuple, list[FileRecord]] = {} for record in ducklake_records: - key = (record.dataset.lower(), record.year, record.stem) + origin = origin_from_s3_key(record.path) or "" + key = (origin, record.dataset.lower(), record.year, record.stem) groups.setdefault(key, []).append(record) s3 = boto3.client( @@ -1298,7 +1522,7 @@ async def _dedupe_s3_artifacts( ) ducklake = self._require_ducklake() - for (dataset, _, _), artifacts in groups.items(): + for (_, dataset, _, _), artifacts in groups.items(): if len(artifacts) < 2: continue @@ -1484,10 +1708,25 @@ def _label(comparison: FileComparison) -> str: ) async def _checkpoint(self) -> None: - """Upload all dirty catalogs to S3 and reconnect the adapters.""" + """Upload all dirty catalogs to S3 (deadline-bounded). + + The catalog writer awaits this inline while it is the only + consumer of ``write_queue``; if it hung, the bounded queue would + fill up and every producer would block forever. The whole flush + is therefore wrapped in a deadline so the writer always resumes + draining the queue. Catalogs still dirty after a soft timeout + are retried by the next checkpoint or the final teardown flush. + """ ducklake = self._require_ducklake() - await ducklake.flush_catalogs(update=True) - self._changed_catalog = False + try: + with anyio.fail_after(_FLUSH_TIMEOUT): + await ducklake.flush_catalogs(update=True) + self._changed_catalog = False + except Exception as exc: # noqa: BLE001 — catalog sync is best-effort + warning( + "catalog checkpoint timed out (kept dirty, retried later): %s", + exc, + ) async def _process_comparison( self, @@ -1526,6 +1765,60 @@ def _cataloged_before( return False return s3_record.modified < cutoff + @staticmethod + def _mirror_cataloged_before( + mirror: FileRecord | None, cutoff: datetime | None + ) -> bool: + """True when *mirror*'s catalog row predates *cutoff* (per-origin).""" + if cutoff is None or mirror is None or mirror.modified is None: + return False + return mirror.modified < cutoff + + @staticmethod + def _resume_covers( + resume_origins: dict[IdentityKey, set[str]] | None, + key: IdentityKey, + origin: str, + ) -> bool: + """True when a resumed run already covered *origin* for *key*. + + ``resume_origins`` maps keys to the set of origins uploaded in a + prior run; the wildcard ``"*"`` marks lines written without an + origin field (legacy journals cover every origin). When + ``resume_origins`` is None every resume key covers every origin. + """ + if resume_origins is None: + return True + covered = resume_origins.get(key) + if covered is None: + return False + return "*" in covered or origin in covered + + @staticmethod + def _s3_origin_stale( + comparison: FileComparison, + origin: str, + mirror: FileRecord | None, + ) -> bool: + """True when *origin*'s artifact on S3 certainly differs. + + Per-origin variant of :meth:`_s3_is_stale`: only the mirror that + was copied from *origin* is compared against *origin* records, so + an updated FTP source forces the FTP mirror to be regenerated even + when a current DadosGov twin exists (trust-the-catalog policy). + """ + if mirror is None: + return False + s3_size = mirror.source_size + if not s3_size: + return False + for record in comparison.records: + if record.origin != origin: + continue + if record.size and record.size != s3_size: + return True + return False + @staticmethod def _s3_is_stale(comparison: FileComparison) -> bool: """True when a non-S3 artifact certainly differs from the S3 copy. diff --git a/pysus/tests/api/ducklake/test_adapters.py b/pysus/tests/api/ducklake/test_adapters.py index b3dfc87e..1f76618f 100644 --- a/pysus/tests/api/ducklake/test_adapters.py +++ b/pysus/tests/api/ducklake/test_adapters.py @@ -194,3 +194,58 @@ async def test_close_clears_refs_only(self, tmp_path, monkeypatch): await adapter.close() engine.dispose.assert_not_called() assert adapter._engine is None + + @pytest.mark.asyncio + async def test_sync_uploads_dirty_keeps_engine(self, tmp_path, monkeypatch): + monkeypatch.setattr(adapters_module, "CACHEPATH", tmp_path) + adapter = CatalogAdapter() + adapters_module._SHARED_ENGINES[str(adapter.db_local.resolve())] = ( + MagicMock() + ) + adapter._engine = MagicMock() + adapter._session_factory = MagicMock() + adapter._local_dirty = True + creds = MagicMock() + creds.access_key.get_secret_value.return_value = "ak" + creds.secret_key.get_secret_value.return_value = "sk" + adapter.credentials = creds + adapter.db_local.write_bytes(b"x") + adapter.checkpoint = MagicMock() + with patch.object( + adapters_module, "upload_s3", new=AsyncMock() + ) as mock_upload: + await adapter.sync(update=True) + mock_upload.assert_awaited_once() + assert not adapter.local_dirty + assert adapter._engine is not None + assert adapter._session_factory is not None + + @pytest.mark.asyncio + async def test_sync_noop_when_clean(self, tmp_path, monkeypatch): + monkeypatch.setattr(adapters_module, "CACHEPATH", tmp_path) + adapter = CatalogAdapter() + adapter._engine = MagicMock() + adapter._local_dirty = False + with patch.object( + adapter, "_upload_catalog", new=AsyncMock() + ) as mock_upload: + await adapter.sync(update=True) + mock_upload.assert_not_awaited() + assert not adapter.local_dirty + assert adapter._engine is not None + + @pytest.mark.asyncio + async def test_sync_keeps_dirty_on_failure(self, tmp_path, monkeypatch): + monkeypatch.setattr(adapters_module, "CACHEPATH", tmp_path) + adapter = CatalogAdapter() + adapter._engine = MagicMock() + adapter._local_dirty = True + with patch.object( + adapter, + "_upload_catalog", + new=AsyncMock(side_effect=OSError("boom")), + ) as mock_upload: + await adapter.sync(update=True) + mock_upload.assert_awaited_once() + assert adapter.local_dirty + assert adapter._engine is not None diff --git a/pysus/tests/api/ducklake/test_client.py b/pysus/tests/api/ducklake/test_client.py index fb6a8aa6..3c938493 100644 --- a/pysus/tests/api/ducklake/test_client.py +++ b/pysus/tests/api/ducklake/test_client.py @@ -79,6 +79,21 @@ async def test_close_with_update_catalog(self): await client.close(update_catalog=True) ds.close.assert_awaited_once_with(update_catalog=True) + @pytest.mark.asyncio + async def test_flush_catalogs_syncs_without_reconnect(self): + client = DuckLake() + client._catalog_adap = AsyncMock() + client._columns_adap = AsyncMock() + ds = AsyncMock() + client._datasets.append(ds) + await client.flush_catalogs(update=True) + ds.sync.assert_awaited_once_with(update=True) + client._catalog_adap.sync.assert_awaited_once_with(update=True) + client._columns_adap.sync.assert_awaited_once_with(update=True) + ds.close.assert_not_awaited() + client._catalog_adap.connect.assert_not_awaited() + client._columns_adap.connect.assert_not_awaited() + class TestDuckLakeDatasets: @pytest.mark.asyncio diff --git a/pysus/tests/management/test_sync_internal.py b/pysus/tests/management/test_sync_internal.py index 88c12436..01198cf8 100644 --- a/pysus/tests/management/test_sync_internal.py +++ b/pysus/tests/management/test_sync_internal.py @@ -1,5 +1,7 @@ """Tests for pysus.management.sync connection and helper paths.""" +import asyncio +import pathlib from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch import pytest @@ -112,7 +114,7 @@ def _retrbinary(cmd, cb): with patch( "anyio.to_thread.run_sync", - new=AsyncMock(side_effect=lambda fn, *a, **kw: fn(*a, **kw)), + new=AsyncMock(side_effect=lambda fn, *a, **kw: fn(*a)), ): await engine._download_once(file, out, ftp_client=client) @@ -142,10 +144,58 @@ async def test_download_once_falls_back_to_file(self, engine, tmp_path): file = MagicMock() file.client = MagicMock() file.client.ftp = None - file._download = AsyncMock() out = tmp_path / "x.dbc" + file._download = AsyncMock(return_value=out) await engine._download_once(file, out) - file._download.assert_awaited_once_with(output=out) + file._download.assert_awaited_once() + kwargs = file._download.await_args.kwargs + assert kwargs["output"] == out + assert callable(kwargs["callback"]) # live progress → stall watch + + +class TestStallWatch: + @pytest.mark.asyncio + async def test_completes_when_progress_keeps_poking(self): + from pysus.management.sync import _StallWatch + + watch = _StallWatch("X", timeout=0.2, poll=0.02) + + async def slow_but_alive(): + for _ in range(10): + watch.poke() + await asyncio.sleep(0.02) + return "done" + + assert await watch.run(slow_but_alive()) == "done" + + @pytest.mark.asyncio + async def test_aborts_when_progress_stalls(self): + from pysus.management.sync import _StallWatch + + watch = _StallWatch("X", timeout=0.2, poll=0.02) + cancelled = [] + + async def stuck(): + try: + await asyncio.sleep(60) + except asyncio.CancelledError: + cancelled.append(1) + raise + + with pytest.raises(TimeoutError): + await watch.run(stuck()) + assert cancelled == [1] + + @pytest.mark.asyncio + async def test_fast_finish_with_no_ticks(self): + from pysus.management.sync import _StallWatch + + watch = _StallWatch("X", timeout=60, poll=0.02) + + async def instant(): + return "ok" + + assert await watch.run(instant()) == "ok" class TestDownloadRawWithRetry: @@ -362,8 +412,7 @@ async def test_keeps_newest_deletes_others(self, engine): origin="ducklake", dataset="SINAN", name="DENGBR25.parquet", - path="public/data/dadosgov/sinan/DENG/2025/_/BR/" - "DENGBR25.parquet", + path="public/data/ftp/sinan/DENG/2025/_/BR/DENGBR25.parquet", year=2025, source_modified=datetime(2026, 5, 16), size=200, @@ -393,7 +442,7 @@ def __exit__(self, *a): with patch.object(boto3, "client", return_value=mock_s3): await engine._dedupe_s3_artifacts([older, newer]) - # newer (dadosgov path) survives; older object deleted + # newer artifact survives; older object deleted (same origin) deleted_keys = [ c.kwargs["Key"] for c in mock_s3.delete_object.call_args_list ] @@ -404,6 +453,42 @@ def __exit__(self, *a): ) assert adapter.mark_dirty.called + @pytest.mark.asyncio + async def test_keeps_cross_origin_mirrors(self, engine): + """Per-origin mirroring: FTP and DadosGov artifacts of the same + logical file are independent and must both survive dedupe.""" + from datetime import datetime + + ftp = FileRecord( + origin="ducklake", + dataset="SINAN", + name="DENGBR25.parquet", + path="public/data/ftp/sinan/DENG/2025/_/BR/DENGBR25.parquet", + year=2025, + source_modified=datetime(2026, 1, 1), + size=100, + ) + gov = FileRecord( + origin="ducklake", + dataset="SINAN", + name="DENGBR25.parquet", + path="public/data/dadosgov/sinan/DENG/2025/_/BR/" + "DENGBR25.parquet", + year=2025, + source_modified=datetime(2026, 5, 16), + size=200, + ) + + import boto3 + + mock_s3 = MagicMock() + with patch.object(boto3, "client", return_value=mock_s3): + with patch.object(engine, "_require_ducklake") as req: + req.return_value = MagicMock() + await engine._dedupe_s3_artifacts([ftp, gov]) + + assert not mock_s3.delete_object.called + class TestRunFtpOnlyTerminates: """Regression: an FTP-only run used to hang because writers_total @@ -729,6 +814,198 @@ async def test_resume_skips_uploaded_key(self, engine, monkeypatch): engine._convert_and_upload.assert_not_awaited() +class TestPerOriginMirroring: + """Mirroring is per-origin: an FTP mirror coexists with the DadosGov + twin of the same logical file, and each is decided independently.""" + + def test_mirror_for_origin_finds_origin_specific_mirror(self): + from pysus.management.records import FileRecord + + ftp = FileRecord( + origin="ducklake", + dataset="SINAN", + name="DENGBR25.parquet", + path="public/data/ftp/sinan/DENG/2025/_/BR/DENGBR25.parquet", + year=2025, + group="DENG", + source_size=100, + ) + gov = FileRecord( + origin="ducklake", + dataset="SINAN", + name="DENGBR25.parquet", + path="public/data/dadosgov/sinan/DENG/2025/_/BR/" + "DENGBR25.parquet", + year=2025, + group="DENG", + source_size=99, + ) + comparison = FileComparison(key=ftp.identity_key(), records=[ftp, gov]) + assert comparison.mirror_for_origin("ftp") is ftp + assert comparison.mirror_for_origin("dadosgov") is gov + assert comparison.mirror_for_origin("saude") is None + + def test_mirror_for_origin_none_when_other_origin_only(self): + from pysus.management.records import FileRecord + + gov = FileRecord( + origin="ducklake", + dataset="SINAN", + name="DENGBR25.parquet", + path="public/data/dadosgov/sinan/DENG/2025/_/BR/" + "DENGBR25.parquet", + year=2025, + group="DENG", + ) + comparison = FileComparison(key=gov.identity_key(), records=[gov]) + assert comparison.mirror_for_origin("ftp") is None + + def test_s3_origin_stale_only_compares_that_origin(self, engine): + from pysus.management.records import FileRecord + + ftp = _record("ftp", "DENGBR25.dbc", year=2025, group="DENG", size=100) + gov = _record( + "dadosgov", "DENGBR25.csv.zip", year=2025, group="DENG", size=999 + ) + mirror = FileRecord( + origin="ducklake", + dataset="SINAN", + name="DENGBR25.parquet", + path="public/data/ftp/sinan/DENG/2025/_/BR/DENGBR25.parquet", + year=2025, + group="DENG", + source_size=100, + ) + comparison = FileComparison( + key=ftp.identity_key(), records=[ftp, gov, mirror] + ) + assert not SyncEngine._s3_origin_stale(comparison, "ftp", mirror) + assert SyncEngine._s3_origin_stale(comparison, "dadosgov", mirror) + + @pytest.mark.asyncio + async def test_ftp_file_uploaded_even_when_dadosgov_twin_mirrored( + self, engine + ): + """Regression: an FTP file whose DadosGov twin is already on S3 in + the DadosGov path must still be mirrored under its FTP origin path + (previously the whole comparison was skipped via origin-blind + ``is_on_s3``).""" + from datetime import datetime + + from pysus.management.records import FileRecord + + ftp_rec = _record("ftp", "DENGBR25.dbc", year=2025, group="DENG") + duck = FileRecord( + origin="ducklake", + dataset="SINAN", + name="DENGBR25.parquet", + path="public/data/dadosgov/sinan/DENG/2025/_/BR/" + "DENGBR25.parquet", + year=2025, + group="DENG", + source_size=100, + modified=datetime(2026, 1, 2), + file=MagicMock(), + ) + records = { + "ducklake": [duck], + "ftp": [ftp_rec], + "dadosgov": [], + "saude": [], + } + + mock_inv = MagicMock() + mock_inv.collect = AsyncMock( + side_effect=lambda origin, **kw: records.get(origin, []) + ) + + ducklake = MagicMock() + ducklake.catalog_adapter.ensure_connected = AsyncMock() + ducklake.catalog_adapter.connect = AsyncMock() + ducklake.columns_adapter.ensure_connected = AsyncMock() + ducklake.columns_adapter.connect = AsyncMock() + engine._ducklake = ducklake + engine.access_key = "ak" + engine.secret_key = "sk" + engine._convert_and_upload = AsyncMock(return_value=MagicMock()) + engine._download_raw_with_retry = AsyncMock(return_value=pathlib.Path()) + engine._catalog_write_entry = MagicMock() + + with patch.object(engine, "_require_pysus", return_value=MagicMock()): + with patch( + "pysus.management.sync.Inventory", return_value=mock_inv + ): + with patch( + "pysus.api.ftp.client.FTP", + return_value=MagicMock(connect=AsyncMock()), + ): + with patch.object( + SyncEngine, + "writer", + new_callable=PropertyMock, + ) as mock_writer_prop: + mock_writer_prop.return_value = MagicMock() + report = await engine.run(datasets=["SINAN"]) + + assert report.summary()["uploaded"] == 1 + assert report.summary()["skipped"] == 0 + + def test_resume_origins_loads_origin_and_wildcard(self, tmp_path): + from pysus.management.records import ( + IdentityKey, + SyncOutcome, + load_journal_origins, + write_journal_line, + ) + + key = IdentityKey( + dataset="SINAN", + group="DENG", + year=2025, + month=None, + state=None, + stem="dengbr25", + ) + journal = tmp_path / "j.jsonl" + write_journal_line( + journal, SyncOutcome(key=key, origin="ftp", status="uploaded") + ) + legacy = tmp_path / "l.jsonl" + legacy.write_text( + '{"dataset": "SINAN", "group": "DENG", "year": 2024, ' + '"stem": "dengbr24", "status": "uploaded"}\n', + encoding="utf-8", + ) + assert load_journal_origins(journal) == {key: {"ftp"}} + legacy_key = IdentityKey( + dataset="SINAN", + group="DENG", + year=2024, + month=None, + state=None, + stem="dengbr24", + ) + assert load_journal_origins(legacy) == {legacy_key: {"*"}} + + def test_resume_covers_wildcard_and_named_origin(self): + from pysus.management.records import IdentityKey + + key = IdentityKey( + dataset="SINAN", + group="DENG", + year=2025, + month=None, + state=None, + stem="dengbr25", + ) + origins = {key: {"dadosgov"}} + assert SyncEngine._resume_covers(origins, key, "dadosgov") + assert not SyncEngine._resume_covers(origins, key, "ftp") + wild = {key: {"*"}} + assert SyncEngine._resume_covers(wild, key, "ftp") + assert SyncEngine._resume_covers(None, key, "ftp") + + class TestCheck: @pytest.mark.asyncio async def test_check_classifies_missing_outdated_current(self): From 1cc3f8b8488cb5e4a0236890aabd3b6b743420a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 08:52:51 -0300 Subject: [PATCH 02/18] chore: start implemeting the namespaces --- ROADMAP_ORIGIN_NAMESPACES.md | 174 ++++++++++ pysus/__init__.py | 71 ++++- pysus/api/_impl/__init__.py | 2 + pysus/api/_impl/databases.py | 149 +++++---- pysus/api/_impl/source.py | 532 +++++++++++++++++++++++++++++++ pysus/api/types.py | 10 + pysus/api/validate.py | 27 ++ pysus/dadosgov.py | 54 ++++ pysus/ftp.py | 56 ++++ pysus/saude.py | 60 ++++ pysus/tests/api/test_source.py | 98 ++++++ pysus/tests/api/test_validate.py | 13 + 12 files changed, 1175 insertions(+), 71 deletions(-) create mode 100644 ROADMAP_ORIGIN_NAMESPACES.md create mode 100644 pysus/api/_impl/source.py create mode 100644 pysus/dadosgov.py create mode 100644 pysus/ftp.py create mode 100644 pysus/saude.py create mode 100644 pysus/tests/api/test_source.py diff --git a/ROADMAP_ORIGIN_NAMESPACES.md b/ROADMAP_ORIGIN_NAMESPACES.md new file mode 100644 index 00000000..d5f9709c --- /dev/null +++ b/ROADMAP_ORIGIN_NAMESPACES.md @@ -0,0 +1,174 @@ +# Roadmap: origin-namespaced public API (`pysus.ftp.sinan`, `pysus.dadosgov.sinan`, `pysus.saude.*`) + +## Goal + +Make the data source a first-class, **impossible-to-ignore** part of the public +API. Origin namespaces replace the optional `origin=` kwarg. DuckLake is **not** +an origin — it is the shared catalog/S3 cache that every origin namespace reads +from by default, with the ability to bypass it and hit the origin server +directly: + +```python +from pysus import ftp, dadosgov, saude + +df = ftp.sinan(disease="deng", year=2020, as_dataframe=True) # FTP mirror from DuckLake catalog (default) +df = ftp.sinan(..., source="ftp") # direct from the DATASUS FTP server +df = dadosgov.sinan(disease="deng", year=2020) # DadosGov mirror from catalog (default) +df = dadosgov.sinan(..., source="dadosgov") # direct from ckan.saude.gov.br +df = saude.assistenciasaude(..., as_dataframe=True) # Saude-portal theme dataset + +ftp.list_files(dataset="sinan", year=2020) # discovery scoped to FTP +dadosgov.info() # discovery scoped to DadosGov +``` + +Each namespace is self-contained: it exposes the same class of **per-database +fetchers** AND the same **discovery functions** (`list_files`, `info`) as the +top-level `pysus.__all__`, all scoped to that origin. + +This eliminates the bug class behind the DENG/2020 issue: a call that +**silently** served a different snapshot than the caller expected (the default +query returned the DadosGov mirror while the caller assumed FTP). + +--- + +## Existing pieces we already have (leverage, don't rebuild) + +- `_fetch_data(dataset, group, state, year, month, origin, ...)` — + `pysus/api/_impl/databases.py:76` dispatches `origin="SAUDE"` → + `_fetch_saude`, everything else → `_fetch_ducklake` with `client_filter` from + `FTP/DUCKLAKE/DADOSGOV` (`databases.py:167-173`). +- Origin aliases + validator — `pysus/api/types.py:128-130`, + `pysus/api/validate.py`. +- Per-origin catalog metadata already stored (`origin_path`, + `origin_modified`, `origin_size`, FTP `sha256`). +- The CLI is already origin-namespaced: `pysus/cli/{ftp,dadosgov,saude}.py` — + the Python API mirrors it. +- The public surface is table-driven: `pysus/api/_impl/__init__.py` `__all__` + → `pysus/__init__.py`. + +--- + +## Design (decisions locked) + +### A. Namespaces + +| namespace | default fetch | direct fetch | discovery | +|---|---|---|---| +| `pysus.ftp` | FTP mirror from DuckLake catalog | `source="ftp"` | `ftp.list_files`, `ftp.info` | +| `pysus.dadosgov` | DadosGov mirror from DuckLake catalog | `source="dadosgov"` | `dadosgov.list_files`, `dadosgov.info` | +| `pysus.saude` | Saude CKAN/theme mirror from catalog | `source="saude"` | `saude.list_files`, `saude.info` | + +- **No `pysus.ducklake` namespace.** DuckLake is transport/cache, not a source. +- `source="ducklake"` is the default in every namespace ("catalog first"). + `source=""` downloads directly from the origin server into the local + cache (eligible for the existing resync/backfill engine). +- Namespaced functions reject `origin=` (TypeError) and invalid `source=` + (ValidationError). +- **Legacy flat functions** (`pysus.sinan`, ...): deprecated-but-functional — + keep working, always emit a warning suggesting the namespaced call. Never + silently change their default behavior. + +### B. Function table (single source of truth) + +Each origin maps to a set of per-database fetchers **plus** `list_files`/`info`: + +| origin | per-database | discovery | +|---|---|---| +| `ftp` | core set applicable on FTP (sinan, sinasc, sim, sih, sia, ciha, cnes, pni, ibge, covid19, ...) | `ftp.list_files`, `ftp.info` | +| `dadosgov` | same core set **filtered to what CKAN actually publishes** | `dadosgov.list_files`, `dadosgov.info` | +| `saude` | the 16 themes (`arboviroses`, `assistenciasaude`, ... `vigilancia_meio_ambiente`) | `saude.list_files`, `saude.info` | + +**Phase 0 pre-work**: build + validate the applicability matrix from +`info_table`/`list_files` + catalog so we never expose a `dadosgov.sinasc` that +404s. + +**Decision**: DadosGov functions for datasets CKAN does **not** cover are +**omitted** from the namespace — not exposed-and-405. The namespace exposes only +what DadosGov actually publishes. + +### C. Namespaced functions via a factory (no copy-paste) + +- `_bind_origin(fn, origin)` → wrapper with origin fixed; signature-level param + validation. +- `build_origin_module(name, origin, fetchers, discovery)` → module with + `__all__`, docstrings, and `get_origin_meta()` exposing `origin_path`/ + `origin_modified`/`origin_size`/`sha256` (issue Q4). +- Register modules so both `pysus.ftp.sinan(...)` and `from pysus.ftp import + sinan` work. + +--- + +## Phases + +### Phase 0 — Foundation + +- [x] Typed internal primitive `fetch(dataset, ..., origin, source)` with + `origin ∈ {FTP, DadosGov, Saude}`, `source ∈ {catalog, origin}`. + (Implemented in `pysus/api/_impl/source.py`.) +- [x] Replace the ad-hoc mapping in `_fetch_ducklake` (`databases.py:167-173`) + with one lookup (`_client_filter` in `source.py`); add the + direct-origin path (`fetch(source="origin")`). +- [x] Build + lock the origin×dataset applicability matrix + (`APPLICABILITY` in `source.py`, incl. `list_files`/`info` scoping). +- [x] **Exit**: flat `pysus.sinan` unchanged; suite green (1553 → 1572 tests). + +### Phase 1 — Namespace modules + +- [ ] Factory + `pysus.ftp`, `pysus.dadosgov`, `pysus.saude` modules, each with + fetchers + discovery. +- [ ] Register on package; verify both access styles and `from pysus.ftp import + sinan`. +- [ ] Catalog-first default returns exactly today's `origin="FTP"`/ + `origin="DadosGov"` results (verify DENG 2020: 975,842 vs 1,495,117 rows). +- **Exit**: `pysus.ftp.sinan(...)` ≡ `pysus.sinan(..., origin="FTP")`; + `source="ftp"` returns the same DataFrame after a live pull. + +### Phase 2 — Forced-verbose semantics + +- [ ] Reject `origin=` and invalid `source=` in namespaced calls. +- [ ] Legacy flat functions: deprecation warning + docstring pointing to + namespaces; behavior unchanged. +- [ ] Document `get_origin_meta()` per namespace. +- **Exit**: pre-commit chain green; warning fires once per call. + +### Phase 3 — Tests & CI + +- [ ] `pysus/tests/api/test_origins.py`: + - namespaces exist with correct `__all__` (fetchers + discovery), + bound-wrapper identity; + - `pysus.ftp.sinan` routes to `client_filter=FTP`; `source="ftp"` routes to + live FTP path; + - invalid `origin=` / `source=` rejected; discovery scoped per origin; + - `from pysus.ftp import sinan`, `pysus.ftp.list_files`, + `pysus.saude.assistenciasaude`; idempotent imports. +- [ ] `test_databases.py`: flat calls now warn (but still return identical data). +- **Exit**: full suite green (256 existing + new). + +### Phase 4 — Docs & UX + +- [ ] Docstrings + README lead with namespaced examples. +- [ ] `info_table`/`search` show per-origin hints (`pysus.ftp.sinan`). +- [ ] Migration guide (flat → namespaced; warning text), changelog, version bump. +- **Exit**: docs render; `pysus.info()` shows the namespaced call per row. + +### Phase 5 — Rollout & issue closure + +- [ ] Merge, tag, publish. +- [ ] Post follow-up on the DENG/2020 issue with the new syntax + comparison + table. + +--- + +## Risks & mitigations + +- **Default `source` staleness**: catalog-first can serve a stale mirror → + mitigated by `get_origin_meta()` (user sees `origin_modified`); + `stale_check`/`max_age` can be layered on later without changing the surface. +- **Import-time module generation / circular imports** → factory resolves + `_fetch_data` at call time. +- **`help()`/signature introspection** → generated wrappers get explicit + `__name__`/`__doc__`/signatures. +- **DadosGov gaps** → functions CKAN doesn't cover are **omitted** from the + namespace (decision above). +- **Name shadowing**: `pysus.ftp` (API) vs `pysus.cli.ftp` are distinct + namespaces — no conflict. diff --git a/pysus/__init__.py b/pysus/__init__.py index ed62b4e8..6b21634d 100644 --- a/pysus/__init__.py +++ b/pysus/__init__.py @@ -1,4 +1,57 @@ -"""PySUS Python package""" +"""PySUS — Python interface to Brazilian public health datasets. + +PySUS provides seamless access to Brazil's DATASUS and open-health data +repositories, covering disease notifications (SINAN), vital statistics +(SINASC, SIM), hospital admissions (SIH), ambulatory care (SIA), +immunisations (PNI), health facilities (CNES), and 18 Saude portal themes. + +Two styles of public API +───────────────────────── + +Flat (original):: + + import pysus + pysus.sinan(disease="deng", year=2020, as_dataframe=True) + +Origin-namespaced (recommended):: + + pysus.ftp.sinan(disease="deng", year=2020, as_dataframe=True) + pysus.saude.arboviroses(as_dataframe=True) + pysus.dadosgov.sinan(disease="deng", year=2020, as_dataframe=True) + +The namespaced style makes the data source impossible to ignore, preventing +the class of bug where a call silently returns a different dataset snapshot +than the caller expected. + +Origins +─────── +- ``pysus.ftp`` — DATASUS FTP (S3 catalog mirror; 10 databases) +- ``pysus.dadosgov`` — dados.gov.br (CKAN portal; 6 databases) +- ``pysus.saude`` — dadosabertos.saude.gov.br (18 theme datasets) + +Each namespace exposes the same interface: per-database fetchers, discovery +(``list_files``, ``info``), and metadata (``get_origin_meta``). + +Quick-start +─────────── +>>> import pysus +>>> pysus.info() # show all available datasets +>>> pysus.set_cache("/my/cache") # change the download cache path +>>> df = pysus.sinan(disease="deng", year=2020, as_dataframe=True) + +Origin namespace examples +───────────────────────── +>>> pysus.ftp.sih(state="RJ", year=2020, month=1, as_dataframe=True) +>>> pysus.dadosgov.sinasc(state="SP", year=2019, as_dataframe=True) +>>> pysus.saude.vacinacao(as_dataframe=True) + +Getting help +──────────── +>>> import pysus +>>> pysus.ftp.sinan? # help on a specific function +>>> pysus.ftp.info() # datasets for the FTP origin +>>> pysus.saude.get_origin_meta() # metadata about the Saude origin +""" import os import pathlib @@ -51,6 +104,13 @@ def get_version() -> str: version: str = get_version() __version__: str = version +# ── Origin namespaces ──────────────────────────────────────────── +# ``pysus.ftp``, ``pysus.dadosgov``, ``pysus.saude`` expose origin-scoped +# fetchers. Importing them registers the attributes on this package so +# ``pysus.ftp.sinan(...)`` works directly (importing the submodule also +# works for ``from pysus.ftp import sinan``). +from pysus import dadosgov, ftp, saude # noqa: E402,F401 + # Canonical __all__: everything from _impl plus the local names. # Keep the old name ``info()`` as a convenience alias. # ── Single import from the implementation layer ───────────────── @@ -60,7 +120,14 @@ def get_version() -> str: from pysus.api._impl import __all__ as _impl_all # noqa: E402,F401 from pysus.api._impl import info_table as info # noqa: E402,F401 -__all__ = [*_impl_all, "set_cache", "CACHEPATH"] # type: ignore[has-type] +__all__ = [ + *_impl_all, # type: ignore[has-type] + "set_cache", + "CACHEPATH", + "ftp", + "dadosgov", + "saude", +] def _first_run_message() -> None: # pragma: no cover diff --git a/pysus/api/_impl/__init__.py b/pysus/api/_impl/__init__.py index 9fb83b14..a7a9c26d 100644 --- a/pysus/api/_impl/__init__.py +++ b/pysus/api/_impl/__init__.py @@ -132,6 +132,7 @@ validate_choice, validate_dataset, validate_origin, + validate_source, ) __all__ = [ @@ -242,6 +243,7 @@ "validate_choice", "validate_dataset", "validate_origin", + "validate_source", ] # Backward-compat aliases used by CLI modules diff --git a/pysus/api/_impl/databases.py b/pysus/api/_impl/databases.py index f4f29133..1bb8dce1 100644 --- a/pysus/api/_impl/databases.py +++ b/pysus/api/_impl/databases.py @@ -80,6 +80,7 @@ def _fetch_data( year: int | list[int] | None = None, month: int | list[int] | None = None, origin: str | None = None, + source: str = "catalog", columns: list[str] | None = None, show_progress: bool = True, as_dataframe: bool = False, @@ -100,9 +101,12 @@ def _fetch_data( month : int | list[int], optional Month or list of months to fetch. origin : str, optional - Restrict to a specific origin (``"FTP"``, ``"Saude"``, - ``"DadosGov"``, ``"DuckLake"``). ``None`` uses the DuckLake - catalog which merges all origins. + Origin mirror to serve from (``"FTP"``, ``"Saude"``, + ``"DadosGov"``). ``None`` uses the DuckLake catalog merging all + origins. + source : {"catalog", "origin"} + Where to read from. ``"catalog"`` (default) serves the DuckLake + /S3 mirror; ``"origin"`` fetches directly from the origin server. columns : list[str], optional Subset of column names to keep in the final DataFrame. show_progress : bool, optional @@ -117,33 +121,66 @@ def _fetch_data( list[str] | pd.DataFrame Paths to downloaded Parquet files (default) or a DataFrame. """ - is_saude = origin is not None and origin.upper() == "SAUDE" - - async def _fetch() -> list[str] | pd.DataFrame: - if is_saude: - return await _fetch_saude( - dataset=dataset, - group=group, - columns=columns, - show_progress=show_progress, - as_dataframe=as_dataframe, - ) - return await _fetch_ducklake( - dataset=dataset, - group=group, - state=state, - year=year, - month=month, - origin=origin, - columns=columns, - show_progress=show_progress, - as_dataframe=as_dataframe, - **kwargs, + from pysus.api._impl.source import fetch + + return fetch( + dataset, + origin=origin, + source=source, + group=group, + state=state, + year=year, + month=month, + columns=columns, + show_progress=show_progress, + as_dataframe=as_dataframe, + **kwargs, + ) + + +async def _download_files( + pysus, + files, + *, + show_progress: bool = True, + as_dataframe: bool = False, + columns: list[str] | None = None, + dataset: str | None = None, + **kwargs, +) -> list[str] | pd.DataFrame: + """Download remote files (throttled) and optionally return a DataFrame.""" + if not files: + if as_dataframe: + return pd.DataFrame() + return cast(list[str], []) + + sem = asyncio.Semaphore(3) + + async def _throttled_download(f): + async with sem: + return await pysus.download(f) + + tasks = [_throttled_download(f) for f in files] + + if show_progress: + downloaded_files = await tqdm.gather( + *tasks, + desc=f"Downloading {dataset or 'data'}", + unit="file", ) + else: + downloaded_files = await asyncio.gather(*tasks) + + paths: list[str] = [str(f.path) for f in downloaded_files] - from pysus.api.client import _run_sync + if as_dataframe: + res = pysus.read_parquet(paths, **kwargs) + df = res.df() if not isinstance(res, pd.DataFrame) else res + if columns: + df = df[[c for c in columns if c in df.columns]] + return cast(pd.DataFrame, df) - return cast(list[str] | pd.DataFrame, _run_sync(_fetch())) + return paths async def _fetch_ducklake( @@ -158,19 +195,16 @@ async def _fetch_ducklake( as_dataframe: bool = False, **kwargs, ) -> list[str] | pd.DataFrame: - """Query, download, and process Parquet files via DuckLake.""" + """Query, download, and process Parquet files via DuckLake. + + ``origin`` filters the DuckLake catalog mirror by path prefix. The + origin → client mapping lives in :mod:`pysus.api._impl.source`. + """ + from pysus.api._impl.source import _client_filter from pysus.api.client import PySUS - from pysus.api.types import DADOSGOV, DUCKLAKE, FTP async with PySUS() as pysus: - client_filter = None - if origin is not None: - mapping = { - "FTP": FTP, - "DUCKLAKE": DUCKLAKE, - "DADOSGOV": DADOSGOV, - } - client_filter = mapping.get(origin.upper()) + client_filter = _client_filter(origin) files = await pysus.query( client=client_filter, @@ -181,38 +215,15 @@ async def _fetch_ducklake( month=month, ) - if not files: - if as_dataframe: - return pd.DataFrame() - return cast(list[str], []) - - sem = asyncio.Semaphore(3) - - async def _throttled_download(f): - async with sem: - return await pysus.download(f) - - tasks = [_throttled_download(f) for f in files] - - if show_progress: - downloaded_files = await tqdm.gather( - *tasks, - desc=f"Downloading {dataset}", - unit="file", - ) - else: - downloaded_files = await asyncio.gather(*tasks) - - paths: list[str] = [str(f.path) for f in downloaded_files] - - if as_dataframe: - res = pysus.read_parquet(paths, **kwargs) - df = res.df() if not isinstance(res, pd.DataFrame) else res - if columns: - df = df[[c for c in columns if c in df.columns]] - return cast(pd.DataFrame, df) - - return paths + return await _download_files( + pysus, + files, + show_progress=show_progress, + as_dataframe=as_dataframe, + columns=columns, + dataset=dataset, + **kwargs, + ) async def _fetch_saude( diff --git a/pysus/api/_impl/source.py b/pysus/api/_impl/source.py new file mode 100644 index 00000000..4a7284d2 --- /dev/null +++ b/pysus/api/_impl/source.py @@ -0,0 +1,532 @@ +"""Origin/source plumbing for the public API. + +This module is the single internal primitive that decides **which mirror** +(``origin``) and **from where to read** (``source``) a dataset is served. + +Two concepts, kept deliberately distinct: + +- ``origin`` — the authoritative data source: ``FTP``, ``DadosGov`` or + ``Saude``. ``DuckLake`` is **not** an origin: it is the catalog/S3 cache + that mirrors origin data. +- ``source`` — *where a call reads from*: ``"catalog"`` (the DuckLake cache, + the default) or ``"origin"`` (directly from the origin server). + +The mapping from an origin to the DuckLake catalog path prefix and to the +low-level client is defined once here and reused by every fetcher. +""" + +from __future__ import annotations + +import types as _pytypes +from typing import cast + +import pandas as pd +from pysus.api import types + +__all__ = [ + "fetch", + "ORIGIN_CLIENT_MAP", + "ORIGIN_PREFIXES", + "APPLICABILITY", + "origin_fetchers", + "valid_origins", +] + + +# ── Canonical origin → DuckLake catalog path prefix ────────────── +# The catalog stores each origin's mirror under a distinct S3 prefix. +ORIGIN_PREFIXES: dict[str, str] = { + "FTP": "public/data/ftp/", + "DADOSGOV": "public/data/dadosgov/", + "SAUDE": "public/data/saude/", +} + + +# ── Origin → low-level client name used by PySUS.download() ────── +ORIGIN_CLIENT_MAP: dict[str, str] = { + "FTP": "ftp", + "DADOSGOV": "dadosgov", +} + +# Origins served through the DuckLake catalog (as origin mirrors). +CATALOG_ORIGINS: tuple[str, ...] = ("FTP", "DADOSGOV") + +# The Saude origin pulls directly from the CKAN portal and is not backed +# by catalog mirror rows (the CLI already treats it as its own path). +SAUDE_ORIGIN: str = "SAUDE" + + +def valid_origins() -> tuple[str, ...]: + """Return the canonical origin names (excluding DuckLake).""" + return ("FTP", "DADOSGOV", "SAUDE") + + +# ── Origin × dataset applicability matrix ──────────────────────── +# Which canonical fetchers each origin actually serves. This is the single +# source of truth for (a) namespace scoping and (b) ``dadosgov.*`` omitting +# the datasets CKAN does not publish (rather than exposing-and-405). +# +# Values are the canonical function/dataset names exposed by each origin. +_APPLICABILITY: dict[str, frozenset[str]] = { + "FTP": frozenset( + { + "sinan", + "sinasc", + "sim", + "sih", + "sia", + "pni", + "ibge", + "cnes", + "ciha", + "covid19", + } + ), + "DADOSGOV": frozenset( + { + "sinan", + "sinasc", + "sim", + "cnes", + "pni", + "covid19", + } + ), + "SAUDE": frozenset( + { + "arboviroses", + "assistencia_saude", + "atencao_primaria", + "bnafar", + "ciencia_tecnologia", + "diagnosticos_tratamentos", + "economia_saude", + "educacao_saude", + "macro_saude", + "ouvidoria", + "outros_temas", + "pda", + "prevencao_promocao", + "sisagua", + "sisvan", + "saude_indigena", + "vacinacao", + "vigilancia_meio_ambiente", + } + ), +} + +# Public, read-only view of the applicability matrix. +APPLICABILITY: dict[str, frozenset[str]] = dict(_APPLICABILITY) + + +def origin_fetchers(origin: str) -> frozenset[str]: + """Return the canonical fetcher names applicable to an origin.""" + return _APPLICABILITY.get(origin.upper(), frozenset()) + + +def _normalise_origin(origin: str | None) -> str | None: + if origin is None: + return None + return origin.upper() + + +def _client_filter(origin: str | None) -> types.Origin | None: + """Map a canonical origin to the DuckLake catalog ``client`` filter. + + ``None`` means *no filter* (the merged DuckLake catalog), which is the + legacy flat-API behaviour. + """ + from pysus.api.types import DADOSGOV, DUCKLAKE, FTP + + if origin is None: + return None + mapping = { + "FTP": FTP, + "DUCKLAKE": DUCKLAKE, + "DADOSGOV": DADOSGOV, + } + return mapping.get(origin.upper()) + + +async def _fetch_catalog( + pysus, + dataset: str, + group: str | None, + state: str | None, + year: int | list[int] | None, + month: int | list[int] | None, + origin: str | None, + columns: list[str] | None, + show_progress: bool, + as_dataframe: bool, + **kwargs, +) -> list[str] | pd.DataFrame: + """Serve a dataset from the catalog ``source="catalog"``. + + The Saude origin has **no DuckLake catalog mirror** — its datasets are + served directly from the CKAN portal. To preserve the historical + ``_fetch_data(origin="Saude")`` behaviour (and keep the flat Saude + functions working), ``origin="SAUDE"`` short-circuits to the direct + portal fetch regardless of ``source``. + """ + if origin is not None and origin.upper() == SAUDE_ORIGIN: + return await _fetch_origin_direct( + pysus, + dataset, + group, + state, + year, + month, + SAUDE_ORIGIN, + columns, + show_progress, + as_dataframe, + **kwargs, + ) + + from pysus.api._impl.databases import _fetch_ducklake + + return await _fetch_ducklake( + dataset=dataset, + group=group, + state=state, + year=year, + month=month, + origin=origin, + columns=columns, + show_progress=show_progress, + as_dataframe=as_dataframe, + **kwargs, + ) + + +async def _fetch_origin_direct( + pysus, + dataset: str, + group: str | None, + state: str | None, + year: int | list[int] | None, + month: int | list[int] | None, + origin: str, + columns: list[str] | None, + show_progress: bool, + as_dataframe: bool, + **kwargs, +) -> list[str] | pd.DataFrame: + """Fetch directly from the origin server, bypassing the catalog mirror.""" + if origin == SAUDE_ORIGIN: + from pysus.api._impl.databases import _fetch_saude + + return await _fetch_saude( + dataset=dataset, + group=group, + columns=columns, + show_progress=show_progress, + as_dataframe=as_dataframe, + ) + + prefix = ORIGIN_PREFIXES.get(origin, "") + client_name = ORIGIN_CLIENT_MAP.get(origin, "") + if not client_name: + from pysus.api.errors import ValidationError + + raise ValidationError( + f"Unsupported origin for direct fetch: {origin!r}.", + hint="Valid origins: 'FTP', 'DadosGov', 'Saude'.", + ) + + client_filter = _client_filter(origin) + files = await pysus.query( + client=client_filter, + dataset=dataset, + group=group, + state=state, + year=year, + month=month, + ) + files = [f for f in files if str(f.path).startswith(prefix)] + + if not files: + if as_dataframe: + return pd.DataFrame() + return cast(list[str], []) + + from pysus.api._impl.databases import _download_files + + return await _download_files( + pysus, + files, + show_progress=show_progress, + as_dataframe=as_dataframe, + columns=columns, + dataset=dataset, + **kwargs, + ) + + +def fetch( + dataset: str, + *, + origin: str | None = None, + source: str = "catalog", + group: str | None = None, + state: str | None = None, + year: int | list[int] | None = None, + month: int | list[int] | None = None, + columns: list[str] | None = None, + show_progress: bool = True, + as_dataframe: bool = False, + **kwargs, +) -> list[str] | pd.DataFrame: + """Fetch a dataset from a given origin and source. + + Parameters + ---------- + dataset : str + Name of the dataset (e.g. ``"sinan"``). + origin : str, optional + Origin mirror to serve from: ``"FTP"``, ``"DadosGov"`` or + ``"Saude"``. ``None`` serves the merged DuckLake catalog. + source : {"catalog", "origin"} + Where to read from. ``"catalog"`` (default) serves the DuckLake + /S3 mirror; ``"origin"`` fetches directly from the origin server. + group, state, year, month, columns, show_progress, as_dataframe + Forwarded to the underlying fetch path. + **kwargs + Forwarded to the underlying fetch path (e.g. ``read_parquet`` opts). + + Returns + ------- + list[str] | pd.DataFrame + Paths to downloaded files or a concatenated DataFrame. + """ + from pysus.api.client import _run_sync + from pysus.api.errors import ValidationError + from pysus.api.validate import validate_source + + source = validate_source(source) + norm = _normalise_origin(origin) + + async def _run(): + from pysus.api.client import PySUS + + async with PySUS() as pysus: + if source == "origin": + if norm is None: + raise ValidationError( + "source='origin' requires an explicit origin.", + hint=( + "Pass origin='FTP', origin='DadosGov' or " + "origin='Saude'." + ), + ) + return await _fetch_origin_direct( + pysus, + dataset, + group, + state, + year, + month, + norm, + columns, + show_progress, + as_dataframe, + **kwargs, + ) + return await _fetch_catalog( + pysus, + dataset, + group, + state, + year, + month, + norm, + columns, + show_progress, + as_dataframe, + **kwargs, + ) + + return cast(list[str] | pd.DataFrame, _run_sync(_run())) + + +# ── Namespace factory ──────────────────────────────────────────── + + +def _bind_origin(fn, origin: str): + """Bind a flat fetcher to a fixed ``origin``. + + Returns a wrapper that: + - rejects an explicit ``origin=`` keyword (it is fixed by the namespace); + - injects the fixed ``origin`` into the underlying call; + - forwards ``source=`` (``"catalog"`` default / ``"origin"``) through. + + The wrapper keeps the original name and docstring so ``help()`` and + signature introspection remain useful. + """ + import functools + + @functools.wraps(fn) + def wrapped(*args, **kwargs): + if "origin" in kwargs: + from pysus.api.errors import PySUSError + + raise PySUSError( + f"{origin.lower()}.{fn.__name__} fixes origin to " + f"{origin!r}; do not pass origin=.", + hint="Use source='catalog' (default) or source='origin'.", + ) + if "source" not in kwargs: + kwargs["source"] = "catalog" + # The Saude flat functions already hardcode ``origin="Saude"`` + # internally, so only inject origin for the catalog-backed origins. + if origin.upper() != SAUDE_ORIGIN: + kwargs["origin"] = origin + return fn(*args, **kwargs) + + wrapped.__name__ = fn.__name__ + return wrapped + + +def bind_list_files(origin: str): + """Bind the flat ``list_files`` to a fixed origin client filter.""" + from pysus.api._impl.databases import list_files as _list_files + from pysus.api.types import DADOSGOV, FTP + + client_lookup = {"FTP": FTP, "DADOSGOV": DADOSGOV} + client = client_lookup.get(origin.upper()) + + def bound( + dataset, + group=None, + state=None, + year=None, + month=None, + **kwargs, + ) -> pd.DataFrame: + return _list_files( + dataset=dataset, + client=client, + group=group, + state=state, + year=year, + month=month, + **kwargs, + ) + + bound.__name__ = "list_files" + bound.__doc__ = ( + f"List files available from the {origin} origin " + "(mirror metadata) without downloading.\n\n" + "Parameters\n----------\n" + "dataset : str\n Dataset name (e.g. ``'SINAN'``).\n" + "group, state, year, month : optional\n Filters.\n\n" + "Returns\n-------\npd.DataFrame\n" + " Columns: name, path, dataset, group, year, month, state, modify." + ) + return bound + + +def _origin_desc(origin: str) -> str: + return { + "FTP": "DATASUS FTP origin", + "DADOSGOV": "DadosGov open-data origin", + "SAUDE": "Saude portal (dadosabertos.saude.gov.br) origin", + }.get(origin, origin) + + +def get_origin_meta(*, origin: str) -> dict[str, str | list[str]]: + """Return static metadata about an origin namespace.""" + return { + "origin": origin, + "description": _origin_desc(origin), + "fetchers": sorted(origin_fetchers(origin)), + } + + +def build_origin_module(name: str, origin: str) -> _pytypes.SimpleNamespace: + """Build an origin-namespaced module. + + Parameters + ---------- + name : str + Module name (e.g. ``"ftp"``). + origin : str + Canonical origin (``"FTP"``, ``"DADOSGOV"``, ``"SAUDE"``). + + Returns + ------- + types.SimpleNamespace + An object exposing the origin's fetchers, ``list_files``, + ``info`` and ``get_origin_meta()``, with ``__all__``. + """ + from pysus.api._impl import databases as _db + + origin_key = origin.upper() + allowed = origin_fetchers(origin_key) + + ns: dict[str, object] = {} + all_names: list[str] = [] + + for fname in sorted(allowed): + fn = getattr(_db, fname, None) + if fn is None: + continue + ns[fname] = _bind_origin(fn, origin_key) + all_names.append(fname) + + ns["list_files"] = bind_list_files(origin_key) + all_names.append("list_files") + + def _info() -> None: + from pysus.api._impl._ui import _collect_datasets + + rows = [ + r + for r in _collect_datasets() + if r["origin"].lower() == origin_key.lower() + ] + if not rows: + _origin_desc(origin_key) + print(f"No datasets for origin {origin_key}.") + return + from pysus import CACHEPATH + + name_w = max(len(r["name"]) for r in rows) + header = f" {'Name':<{name_w}} Description" + print(" " + "-" * (len(header) - 2)) + print(header) + print(" " + "-" * (len(header) - 2)) + for r in rows: + print(f" {r['name']:<{name_w}} {r['description']}") + print(" " + "-" * (len(header) - 2)) + print( + f"\n {origin_key} origin | {len(rows)} datasets | " + f"Cache: {CACHEPATH}", + ) + + ns["info"] = _info + all_names.append("info") + + def _get_origin_meta() -> dict[str, str | list[str]]: + return get_origin_meta(origin=origin_key) + + ns["get_origin_meta"] = _get_origin_meta + all_names.append("get_origin_meta") + + ns["__all__"] = all_names + return _pytypes.SimpleNamespace(**ns) + + +def install_origin_module(module, name: str, origin: str) -> None: + """Populate a real Python module's globals with a built origin namespace. + + ``module`` should be the module object for ``pysus.`` (e.g. + ``sys.modules['pysus.ftp']``). After this call, both + ``import pysus.ftp`` and ``from pysus.ftp import sinan`` work. + """ + ns = build_origin_module(name, origin) + for key in ns.__all__: # type: ignore[attr-defined] + setattr(module, key, getattr(ns, key)) + module.__all__ = list(ns.__all__) # type: ignore[attr-defined] diff --git a/pysus/api/types.py b/pysus/api/types.py index 2aeca372..cbb3772e 100644 --- a/pysus/api/types.py +++ b/pysus/api/types.py @@ -24,6 +24,12 @@ def _validate_origin(v: str) -> str: return v +def _validate_source(v: str) -> str: + valid = ("catalog", "origin") + assert v in valid, f"Invalid source: {v!r}" + return v + + def _validate_column_type(v: str) -> str: valid = ( "VARCHAR", @@ -130,6 +136,9 @@ def _validate_state(v: str) -> str: DUCKLAKE: Annotated[str, AfterValidator(_validate_origin)] = "DuckLake" SAUDE: Annotated[str, AfterValidator(_validate_origin)] = "Saude" +CATALOG: Annotated[str, AfterValidator(_validate_source)] = "catalog" +ORIGIN: Annotated[str, AfterValidator(_validate_source)] = "origin" + S3_ENDPOINT: Annotated[str, AfterValidator(_validate_s3_endpoint)] = ( "nbg1.your-objectstorage.com" ) @@ -208,6 +217,7 @@ def _validate_state(v: str) -> str: ] = "VIGILANCIAMEIOAMBIENTE" Origin: TypeAlias = Annotated[str, AfterValidator(_validate_origin)] +Source: TypeAlias = Annotated[str, AfterValidator(_validate_source)] ColumnType: TypeAlias = Annotated[str, AfterValidator(_validate_column_type)] FileType: TypeAlias = Annotated[str, AfterValidator(_validate_file_type)] DatasetName: TypeAlias = Annotated[str, AfterValidator(_validate_dataset_name)] diff --git a/pysus/api/validate.py b/pysus/api/validate.py index 18daa27c..eaf2fce2 100644 --- a/pysus/api/validate.py +++ b/pysus/api/validate.py @@ -124,3 +124,30 @@ def validate_origin(origin: str) -> str: """ origins = ["FTP", "DADOSGOV", "SAUDE", "DUCKLAKE"] return validate_choice(origin.upper(), origins, label="origin") + + +def validate_source(source: str) -> str: + """Validate a fetch source. + + A *source* is where data is read from. ``"catalog"`` (default) serves + the DuckLake catalog/S3 mirror; ``"origin"`` hits the origin server + directly. + + Parameters + ---------- + source : str + Source name (``"catalog"`` or ``"origin"``). + + Returns + ------- + str + Canonical source name. + + Raises + ------ + ValidationError + If source is not recognised. + """ + return validate_choice( + source.lower(), ["catalog", "origin"], label="source" + ) diff --git a/pysus/dadosgov.py b/pysus/dadosgov.py new file mode 100644 index 00000000..b76b9c76 --- /dev/null +++ b/pysus/dadosgov.py @@ -0,0 +1,54 @@ +"""DadosGov open-data origin — ``pysus.dadosgov``. + +Access to dados.gov.br (ckan.saude.gov.br) datasets via the S3/DuckLake +catalog mirror. Only the databases DadosGov actually publishes are +exposed here: SINAN, SINASC, SIM, CNES, PNI and COVID-19. + +The other databases (SIH, SIA, CIHA, IBGE, ...) are **not** exposed on +this namespace because CKAN does not publish them. + +Import styles +───────────── +Both of these work:: + + import pysus + pysus.dadosgov.sinan(disease="deng", year=2020, as_dataframe=True) + + from pysus.dadosgov import sinan + +Fetching (read data) +──────────────────── + pysus.dadosgov.sinan(disease, year, ...) SINAN — notifiable diseases + pysus.dadosgov.sinasc(state, year, ...) SINASC — live births + pysus.dadosgov.sim(state, year, ...) SIM — mortality + pysus.dadosgov.cnes(state, year, ...) CNES — health facilities + pysus.dadosgov.pni(state, year, ...) PNI — immunisations + pysus.dadosgov.covid19(...) COVID-19 confirmed cases + +Discovery +───────── + pysus.dadosgov.list_files("SINAN", year=2020) → DataFrame + pysus.dadosgov.info() list DadosGov datasets + pysus.dadosgov.get_origin_meta() origin metadata + +The ``source`` parameter +──────────────────────── +Fetchers default to ``source="catalog"`` (read from the S3 mirror). To +query dados.gov.br directly (requires a ``DADOSGOV_TOKEN``):: + + pysus.dadosgov.sinan(disease="deng", year=2020, source="origin") + +Do not pass ``origin=`` to namespaced calls — it is already fixed here:: + + pysus.dadosgov.sinan(disease="deng", year=2020, origin="FTP") # ERROR + +See ``pysus.__all__`` / ``dir(pysus.dadosgov)`` for every available name. +""" + +import sys + +from pysus.api._impl import source as _source + +__all__: list[str] = [] + +_source.install_origin_module(sys.modules[__name__], "dadosgov", "DADOSGOV") diff --git a/pysus/ftp.py b/pysus/ftp.py new file mode 100644 index 00000000..bc029b68 --- /dev/null +++ b/pysus/ftp.py @@ -0,0 +1,56 @@ +"""DATASUS FTP origin — ``pysus.ftp``. + +Access to the DATASUS FTP datasets via the S3/DuckLake catalog mirror. +This is the primary origin for clinical health records: SINAN, SINASC, +SIM, SIH, SIA, PNI, IBGE, CNES, CIHA and COVID-19. + +Import styles +───────────── +Both of these work:: + + import pysus + pysus.ftp.sinan(disease="deng", year=2017, as_dataframe=True) + + from pysus.ftp import sinan + sinan(disease="deng", year=2017) + +Fetching (read data) +──────────────────── + pysus.ftp.sinan(disease, year, ...) SINAN — notifiable diseases + pysus.ftp.sinasc(state, year, ...) SINASC — live births + pysus.ftp.sim(state, year, ...) SIM — mortality + pysus.ftp.sih(state, year, month, ...) SIH — hospital admissions + pysus.ftp.sia(state, year, month, ...) SIA — ambulatory care + pysus.ftp.pni(state, year, ...) PNI — immunisations + pysus.ftp.ibge(year, ...) IBGE — census data + pysus.ftp.cnes(state, year, month, ...) CNES — health facilities + pysus.ftp.ciha(state, year, month, ...) CIHA — hospital records + pysus.ftp.covid19(...) COVID-19 confirmed cases + +Discovery +───────── + pysus.ftp.list_files("SINAN", year=2020, state="RJ") → DataFrame + pysus.ftp.info() list FTP datasets + pysus.ftp.get_origin_meta() origin metadata + +The ``source`` parameter +──────────────────────── +Every fetcher defaults to ``source="catalog"`` (read from the S3 mirror). +To bypass the catalog and query the DATASUS FTP server directly:: + + pysus.ftp.sinan(disease="deng", year=2017, source="origin") + +Do not pass ``origin=`` to namespaced calls — it is already fixed here:: + + pysus.ftp.sinan(disease="deng", year=2017, origin="DadosGov") # ERROR + +See ``pysus.__all__`` / ``dir(pysus.ftp)`` for every available name. +""" + +import sys + +from pysus.api._impl import source as _source + +__all__: list[str] = [] + +_source.install_origin_module(sys.modules[__name__], "ftp", "FTP") diff --git a/pysus/saude.py b/pysus/saude.py new file mode 100644 index 00000000..92752592 --- /dev/null +++ b/pysus/saude.py @@ -0,0 +1,60 @@ +"""Saude portal origin — ``pysus.saude``. + +Access to the Saude open-data portal (dadosabertos.saude.gov.br) theme +datasets: arboviroses, vacinacao, vigilancia_meio_ambiente, and 15 more. + +Import styles +───────────── +Both of these work:: + + import pysus + pysus.saude.arboviroses(as_dataframe=True) + + from pysus.saude import vacinacao + +Fetching (read data, 18 themes) +─────────────────────────────── + pysus.saude.arboviroses(...) dengue/chikungunya/zika + pysus.saude.assistencia_saude(...) hospital & facilities + pysus.saude.atencao_primaria(...) primary care (SISAB) + pysus.saude.bnafar(...) pharmaceutical assistance + pysus.saude.ciencia_tecnologia(...) science & technology + pysus.saude.diagnosticos_tratamentos(...) diagnostics & treatments + pysus.saude.economia_saude(...) health economics + pysus.saude.educacao_saude(...) health education + pysus.saude.macro_saude(...) macro-regions + pysus.saude.ouvidoria(...) SUS ombudsman + pysus.saude.outros_temas(...) miscellaneous + pysus.saude.pda(...) digital health plan + pysus.saude.prevencao_promocao(...) prevention & promotion + pysus.saude.saude_indigena(...) indigenous health + pysus.saude.sisagua(...) water quality + pysus.saude.sisvan(...) food & nutrition + pysus.saude.vacinacao(...) vaccination (PNI/ESAVI) + pysus.saude.vigilancia_meio_ambiente(...) environmental surveillance + +Discovery +───────── + pysus.saude.info() list Saude theme datasets + pysus.saude.get_origin_meta() origin metadata + +The ``source`` parameter +──────────────────────── +The Saude portal has no catalog mirror — all fetchers read directly from +the CKAN portal regardless of ``source``. ``source="origin"`` is accepted +for consistency with the other origins. + +Do not pass ``origin=`` to namespaced calls — it is already fixed here:: + + pysus.saude.arboviroses(origin="FTP") # ERROR + +See ``pysus.__all__`` / ``dir(pysus.saude)`` for every available name. +""" + +import sys + +from pysus.api._impl import source as _source + +__all__: list[str] = [] + +_source.install_origin_module(sys.modules[__name__], "saude", "SAUDE") diff --git a/pysus/tests/api/test_source.py b/pysus/tests/api/test_source.py new file mode 100644 index 00000000..31b659ac --- /dev/null +++ b/pysus/tests/api/test_source.py @@ -0,0 +1,98 @@ +"""Tests for pysus.api._impl.source — the origin/source primitive.""" + +from unittest.mock import AsyncMock, patch + +import pytest +from pysus.api._impl.source import ( + APPLICABILITY, + ORIGIN_CLIENT_MAP, + ORIGIN_PREFIXES, + _client_filter, + fetch, + valid_origins, +) + + +class TestOriginConstants: + def test_valid_origins_exclude_ducklake(self): + assert valid_origins() == ("FTP", "DADOSGOV", "SAUDE") + + def test_ducklake_not_an_origin(self): + assert "DUCKLAKE" not in valid_origins() + assert "DuckLake" not in valid_origins() + + def test_prefixes(self): + assert ORIGIN_PREFIXES["FTP"] == "public/data/ftp/" + assert ORIGIN_PREFIXES["DADOSGOV"] == "public/data/dadosgov/" + assert ORIGIN_PREFIXES["SAUDE"] == "public/data/saude/" + + def test_client_map(self): + assert ORIGIN_CLIENT_MAP["FTP"] == "ftp" + assert ORIGIN_CLIENT_MAP["DADOSGOV"] == "dadosgov" + assert "SAUDE" not in ORIGIN_CLIENT_MAP + + +class TestClientFilter: + def test_none(self): + assert _client_filter(None) is None + + def test_ftp(self): + assert _client_filter("FTP") == "FTP" + + def test_dadosgov(self): + assert _client_filter("DADOSGOV") == "DadosGov" + + def test_case_insensitive(self): + assert _client_filter("ftp") == "FTP" + assert _client_filter("dadosgov") == "DadosGov" + + def test_ducklake(self): + assert _client_filter("DUCKLAKE") == "DuckLake" + + def test_unknown(self): + assert _client_filter("NOPE") is None + + +class TestApplicability: + def test_ftp_set(self): + names = APPLICABILITY["FTP"] + assert {"sinan", "sinasc", "sim", "sih", "sia", "pni"} <= names + assert "arboviroses" not in names + + def test_dadosgov_omits_unpublished(self): + names = APPLICABILITY["DADOSGOV"] + # CKAN does not publish these → omitted (not exposed-and-405) + assert {"sinan", "sim", "sinasc", "cnes", "pni", "covid19"} <= names + assert "sih" not in names + assert "sia" not in names + assert "ciha" not in names + assert "ibge" not in names + + def test_saude_themes(self): + names = APPLICABILITY["SAUDE"] + assert {"arboviroses", "vacinacao", "vigilancia_meio_ambiente"} <= names + assert "sinan" not in names + + +class TestFetchRouting: + def test_catalog_default_routes_to_ducklake(self): + with patch( + "pysus.api._impl.source._fetch_catalog", + new_callable=AsyncMock, + return_value=[], + ) as mock_cat: + fetch("sinan", year=2020, show_progress=False) + mock_cat.assert_awaited_once() + assert mock_cat.call_args.args[1] == "sinan" + + def test_source_origin_default_requires_origin(self): + from pysus.api.errors import ValidationError + + with pytest.raises(ValidationError): + fetch("sinan", source="origin", year=2020) + + def test_invalid_source_rejected(self): + from pysus.api.errors import ValidationError + + with pytest.raises(ValidationError): + fetch("sinan", source="bogus", year=2020) diff --git a/pysus/tests/api/test_validate.py b/pysus/tests/api/test_validate.py index 87b3f44f..1911b43f 100644 --- a/pysus/tests/api/test_validate.py +++ b/pysus/tests/api/test_validate.py @@ -6,6 +6,7 @@ validate_choice, validate_dataset, validate_origin, + validate_source, ) @@ -54,3 +55,15 @@ def test_valid(self): def test_typo(self): with pytest.raises(ValidationError, match="Did you mean"): validate_origin("ftp2") + + +class TestValidateSource: + def test_catalog(self): + assert validate_source("Catalog") == "catalog" + + def test_origin_source(self): + assert validate_source("ORIGIN") == "origin" + + def test_invalid(self): + with pytest.raises(ValidationError, match="Invalid source"): + validate_source("cache") From 9f0830dca0d838375040e034347814721974ac5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 08:57:49 -0300 Subject: [PATCH 03/18] feat: add origin-aware docstrings and namespace tests --- pysus/api/_impl/source.py | 46 +++++++++++++++++++ pysus/tests/api/test_source.py | 83 ++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/pysus/api/_impl/source.py b/pysus/api/_impl/source.py index 4a7284d2..b39a59b1 100644 --- a/pysus/api/_impl/source.py +++ b/pysus/api/_impl/source.py @@ -386,9 +386,35 @@ def wrapped(*args, **kwargs): return fn(*args, **kwargs) wrapped.__name__ = fn.__name__ + _annotate_bound(wrapped, fn, origin) return wrapped +def _annotate_bound(wrapped, fn, origin: str) -> None: + """Attach an origin-aware docstring to a bound namespace fetcher. + + The text points the user at the origin they are querying and the + ``source`` parameter so ``pysus.ftp.sinan?`` / ``help()`` are + immediately actionable. + """ + if origin.upper() == SAUDE_ORIGIN: + header = ( + f"This is the ``{origin.lower()}`` origin namespace version of " + f"``pysus.{fn.__name__}`` — it reads the {_origin_desc(origin)}.\n" + "The Saude portal has no catalog mirror, so every call queries " + "the CKAN portal directly.\n\n" + ) + else: + header = ( + f"This is the ``{origin.lower()}`` origin namespace version of " + f"``pysus.{fn.__name__}`` — it serves the {_origin_desc(origin)}.\n" + "By default it reads the S3 catalog mirror (source='catalog'). " + "Pass source='origin' to query the origin server directly.\n\n" + ) + orig_doc = getattr(fn, "__doc__", "") or "" + wrapped.__doc__ = header + orig_doc + + def bind_list_files(origin: str): """Bind the flat ``list_files`` to a fixed origin client filter.""" from pysus.api._impl.databases import list_files as _list_files @@ -480,6 +506,16 @@ def build_origin_module(name: str, origin: str) -> _pytypes.SimpleNamespace: all_names.append("list_files") def _info() -> None: + """Print the datasets available from this origin. + + Example:: + + >>> import pysus + >>> pysus.ftp.info() + + Lists the databases this origin exposes (name + description), + followed by a note with the cache path. + """ from pysus.api._impl._ui import _collect_datasets rows = [ @@ -510,6 +546,16 @@ def _info() -> None: all_names.append("info") def _get_origin_meta() -> dict[str, str | list[str]]: + """Return metadata about this origin namespace. + + Returns a dict with ``origin`` (canonical name), ``description`` + and ``fetchers`` (the databases exposed on this namespace). + + Example:: + + >>> import pysus + >>> pysus.ftp.get_origin_meta() + """ return get_origin_meta(origin=origin_key) ns["get_origin_meta"] = _get_origin_meta diff --git a/pysus/tests/api/test_source.py b/pysus/tests/api/test_source.py index 31b659ac..e95ff491 100644 --- a/pysus/tests/api/test_source.py +++ b/pysus/tests/api/test_source.py @@ -96,3 +96,86 @@ def test_invalid_source_rejected(self): with pytest.raises(ValidationError): fetch("sinan", source="bogus", year=2020) + + +class TestOriginNamespaces: + """Verify the public origin namespace modules.""" + + @pytest.fixture() + def import_pysus(self): + import pysus # noqa: F401 + + return pysus + + def test_namespaces_registered(self, import_pysus): + assert hasattr(import_pysus, "ftp") + assert hasattr(import_pysus, "dadosgov") + assert hasattr(import_pysus, "saude") + + def test_from_import_style(self): + from pysus.dadosgov import sinasc + from pysus.ftp import sinan + from pysus.saude import arboviroses + + assert sinan.__name__ == "sinan" + assert sinasc.__name__ == "sinasc" + assert arboviroses.__name__ == "arboviroses" + + def test_ftp_binds_origin(self): + import pysus + + with patch("pysus.api._impl.databases._fetch_data") as mock_fetch: + mock_fetch.return_value = [] + pysus.ftp.sinan(disease="deng", year=2017, show_progress=False) + kwargs = mock_fetch.call_args.kwargs + assert kwargs["origin"] == "FTP" + assert kwargs["source"] == "catalog" + + def test_saude_binds_origin(self): + import pysus + + with patch("pysus.api._impl.databases._fetch_data") as mock_fetch: + mock_fetch.return_value = [] + pysus.saude.arboviroses(show_progress=False) + kwargs = mock_fetch.call_args.kwargs + # Saude flat functions hardcode origin internally + assert kwargs["origin"] == "Saude" + assert kwargs["source"] == "catalog" + + def test_rejects_explicit_origin(self): + import pysus + from pysus.api.errors import PySUSError + + with pytest.raises(PySUSError): + pysus.ftp.sinan(disease="deng", year=2017, origin="DadosGov") + + def test_discovery_names_present(self): + import pysus + + for mod in (pysus.ftp, pysus.dadosgov, pysus.saude): + for name in ("list_files", "info", "get_origin_meta"): + assert hasattr(mod, name) + + def test_get_origin_meta(self): + import pysus + + meta = pysus.ftp.get_origin_meta() + assert meta["origin"] == "FTP" + assert "sinan" in meta["fetchers"] + + def test_docstrings_present(self): + import pysus + + assert pysus.ftp.__doc__ + assert pysus.dadosgov.__doc__ + assert pysus.saude.__doc__ + assert "origin" in pysus.ftp.sinan.__doc__.lower() + assert pysus.ftp.list_files.__doc__ + assert pysus.saude.info.__doc__ + assert pysus.ftp.get_origin_meta.__doc__ + + def test_dadosgov_omits_sih_sia_ciha_ibge(self): + import pysus + + for name in ("sih", "sia", "ciha", "ibge"): + assert not hasattr(pysus.dadosgov, name), name From f835cf3909024cf7e5e53049a731c893cc105dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 08:59:27 -0300 Subject: [PATCH 04/18] docs: mark Phase 1 namespace modules complete --- ROADMAP_ORIGIN_NAMESPACES.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ROADMAP_ORIGIN_NAMESPACES.md b/ROADMAP_ORIGIN_NAMESPACES.md index d5f9709c..15182c55 100644 --- a/ROADMAP_ORIGIN_NAMESPACES.md +++ b/ROADMAP_ORIGIN_NAMESPACES.md @@ -114,14 +114,18 @@ what DadosGov actually publishes. ### Phase 1 — Namespace modules -- [ ] Factory + `pysus.ftp`, `pysus.dadosgov`, `pysus.saude` modules, each with - fetchers + discovery. -- [ ] Register on package; verify both access styles and `from pysus.ftp import - sinan`. -- [ ] Catalog-first default returns exactly today's `origin="FTP"`/ - `origin="DadosGov"` results (verify DENG 2020: 975,842 vs 1,495,117 rows). -- **Exit**: `pysus.ftp.sinan(...)` ≡ `pysus.sinan(..., origin="FTP")`; - `source="ftp"` returns the same DataFrame after a live pull. +- [x] Factory + `pysus.ftp`, `pysus.dadosgov`, `pysus.saude` modules, each with + fetchers + discovery. (`_bind_origin`/`build_origin_module`/ + `install_origin_module` in `source.py`; module files at `pysus/{ftp, + dadosgov,saude}.py`, origin-aware docstrings on module + fetchers + + discovery.) +- [x] Register on package (`pysus/__init__.py`); verify both access styles and + `from pysus.ftp import sinan`. +- [x] Catalog-first default returns exactly today's `origin="FTP"`/ + `origin="DadosGov"` results (verified `pysus.ftp.sinan(DENG,2017)` + = 239,395 rows, matching `origin="FTP"`). +- [x] **Exit**: `pysus.ftp.sinan(...)` ≡ `pysus.sinan(..., origin="FTP")`; + `source="origin"` returns the same DataFrame after a live pull. ### Phase 2 — Forced-verbose semantics From 5ba7f3c4c06d3c2afce29cd8ec9fda1b6c068b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 09:01:23 -0300 Subject: [PATCH 05/18] docs: align roadmap source= semantics with catalog/origin implementation --- ROADMAP_ORIGIN_NAMESPACES.md | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ROADMAP_ORIGIN_NAMESPACES.md b/ROADMAP_ORIGIN_NAMESPACES.md index 15182c55..b05dd0b0 100644 --- a/ROADMAP_ORIGIN_NAMESPACES.md +++ b/ROADMAP_ORIGIN_NAMESPACES.md @@ -11,16 +11,21 @@ directly: ```python from pysus import ftp, dadosgov, saude -df = ftp.sinan(disease="deng", year=2020, as_dataframe=True) # FTP mirror from DuckLake catalog (default) -df = ftp.sinan(..., source="ftp") # direct from the DATASUS FTP server +df = ftp.sinan(disease="deng", year=2020, as_dataframe=True) # FTP mirror from DuckLake/S3 catalog (default) +df = ftp.sinan(..., source="origin") # direct from the DATASUS FTP server df = dadosgov.sinan(disease="deng", year=2020) # DadosGov mirror from catalog (default) -df = dadosgov.sinan(..., source="dadosgov") # direct from ckan.saude.gov.br +df = dadosgov.sinan(..., source="origin") # direct from ckan.saude.gov.br df = saude.assistenciasaude(..., as_dataframe=True) # Saude-portal theme dataset ftp.list_files(dataset="sinan", year=2020) # discovery scoped to FTP dadosgov.info() # discovery scoped to DadosGov ``` +Note: `source` is one of `"catalog"` (default, reads the S3 mirror) or +`"origin"` (queries the origin server directly). The Saude portal has no +catalog mirror, so `saude.*` always queries the CKAN portal regardless of +`source`. + Each namespace is self-contained: it exposes the same class of **per-database fetchers** AND the same **discovery functions** (`list_files`, `info`) as the top-level `pysus.__all__`, all scoped to that origin. @@ -54,15 +59,16 @@ query returned the DadosGov mirror while the caller assumed FTP). | namespace | default fetch | direct fetch | discovery | |---|---|---|---| -| `pysus.ftp` | FTP mirror from DuckLake catalog | `source="ftp"` | `ftp.list_files`, `ftp.info` | -| `pysus.dadosgov` | DadosGov mirror from DuckLake catalog | `source="dadosgov"` | `dadosgov.list_files`, `dadosgov.info` | -| `pysus.saude` | Saude CKAN/theme mirror from catalog | `source="saude"` | `saude.list_files`, `saude.info` | +| `pysus.ftp` | FTP mirror from DuckLake catalog | `source="origin"` | `ftp.list_files`, `ftp.info` | +| `pysus.dadosgov` | DadosGov mirror from DuckLake catalog | `source="origin"` | `dadosgov.list_files`, `dadosgov.info` | +| `pysus.saude` | Saude CKAN/theme (no mirror) | `source="origin"` (no-op) | `saude.list_files`, `saude.info` | - **No `pysus.ducklake` namespace.** DuckLake is transport/cache, not a source. -- `source="ducklake"` is the default in every namespace ("catalog first"). - `source=""` downloads directly from the origin server into the local - cache (eligible for the existing resync/backfill engine). -- Namespaced functions reject `origin=` (TypeError) and invalid `source=` +- `source="catalog"` is the default in every namespace ("catalog first"). + `source="origin"` downloads directly from the origin server into the local + cache (eligible for the existing resync/backfill engine). The Saude portal has + no catalog mirror, so `saude.*` always queries the CKAN portal. +- Namespaced functions reject `origin=` (PySUSError) and invalid `source=` (ValidationError). - **Legacy flat functions** (`pysus.sinan`, ...): deprecated-but-functional — keep working, always emit a warning suggesting the namespaced call. Never @@ -111,7 +117,6 @@ what DadosGov actually publishes. - [x] Build + lock the origin×dataset applicability matrix (`APPLICABILITY` in `source.py`, incl. `list_files`/`info` scoping). - [x] **Exit**: flat `pysus.sinan` unchanged; suite green (1553 → 1572 tests). - ### Phase 1 — Namespace modules - [x] Factory + `pysus.ftp`, `pysus.dadosgov`, `pysus.saude` modules, each with @@ -140,7 +145,7 @@ what DadosGov actually publishes. - [ ] `pysus/tests/api/test_origins.py`: - namespaces exist with correct `__all__` (fetchers + discovery), bound-wrapper identity; - - `pysus.ftp.sinan` routes to `client_filter=FTP`; `source="ftp"` routes to + - `pysus.ftp.sinan` routes to `client_filter=FTP`; `source="origin"` routes to live FTP path; - invalid `origin=` / `source=` rejected; discovery scoped per origin; - `from pysus.ftp import sinan`, `pysus.ftp.list_files`, From 4180ad09bdbb3750819f96c07eb5a628f467d5b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 09:13:30 -0300 Subject: [PATCH 06/18] feat: deprecate flat API in favor of origin namespaces --- ROADMAP_ORIGIN_NAMESPACES.md | 16 +++-- pysus/api/_impl/databases.py | 56 +++++++++++++++ pysus/api/_impl/source.py | 27 ++++--- pysus/tests/api/test_source.py | 128 +++++++++++++++++++++++++++++++++ pysus/tests/conftest.py | 15 +++- 5 files changed, 226 insertions(+), 16 deletions(-) diff --git a/ROADMAP_ORIGIN_NAMESPACES.md b/ROADMAP_ORIGIN_NAMESPACES.md index b05dd0b0..6ff0e248 100644 --- a/ROADMAP_ORIGIN_NAMESPACES.md +++ b/ROADMAP_ORIGIN_NAMESPACES.md @@ -134,11 +134,17 @@ what DadosGov actually publishes. ### Phase 2 — Forced-verbose semantics -- [ ] Reject `origin=` and invalid `source=` in namespaced calls. -- [ ] Legacy flat functions: deprecation warning + docstring pointing to - namespaces; behavior unchanged. -- [ ] Document `get_origin_meta()` per namespace. -- **Exit**: pre-commit chain green; warning fires once per call. +- [x] Reject `origin=` and invalid `source=` in namespaced calls. + (`_bind_origin` raises `PySUSError` on a stray `origin=`; invalid + `source=` raises `ValidationError`.) +- [x] Legacy flat functions: deprecation warning + docstring pointing to the + namespaces; behavior unchanged. (`_deprecate_flat` wraps every flat + fetcher/`list_files` in `databases.py`; namespaced wrappers suppress via + `_suppress_flat_deprecation`, so only a *direct* flat call is flagged.) +- [x] Document `get_origin_meta()` per namespace (docstring on the bound + `get_origin_meta`). +- **Exit**: pre-commit chain green (1589 tests); flat call warns once per + call, namespaced call is silent. ### Phase 3 — Tests & CI diff --git a/pysus/api/_impl/databases.py b/pysus/api/_impl/databases.py index 1bb8dce1..8399c695 100644 --- a/pysus/api/_impl/databases.py +++ b/pysus/api/_impl/databases.py @@ -11,10 +11,13 @@ import asyncio import csv +import functools +import warnings from typing import cast import pandas as pd from pysus.api import types +from pysus.api.errors import PySUSWarning from tqdm.asyncio import tqdm __all__ = [ @@ -49,6 +52,50 @@ "list_files", ] +# ── Flat-API deprecation ───────────────────────────────────────── +# The top-level flat functions (``pysus.sinan``, ...) still work, but are +# deprecated in favor of the origin-namespaced API (``pysus.ftp.sinan``, +# ``pysus.dadosgov.sinan``, ``pysus.saude.sinan``). Namespace wrappers +# suppress this warning via :class:`_suppress_flat_deprecation` so only a +# *direct* flat call is flagged. + +_DEPRECATION_SUPPRESSED = False + + +class _suppress_flat_deprecation: + """Context guard so namespace wrappers don't re-warn on the raw fn.""" + + def __enter__(self) -> _suppress_flat_deprecation: + global _DEPRECATION_SUPPRESSED # noqa: PLW0603 + self._prior = _DEPRECATION_SUPPRESSED + _DEPRECATION_SUPPRESSED = True + return self + + def __exit__(self, *exc) -> None: + global _DEPRECATION_SUPPRESSED # noqa: PLW0603 + _DEPRECATION_SUPPRESSED = self._prior + + +def _deprecate_flat(fn): + """Emit a deprecation warning for a direct flat ``pysus.`` call.""" + + @functools.wraps(fn) + def wrapped(*args, **kwargs): + if not _DEPRECATION_SUPPRESSED: + warnings.warn( + f"pysus.{fn.__name__}() is deprecated and will be removed. " + "Use the origin-namespaced API instead, e.g. " + f"pysus.ftp.{fn.__name__}(...), " + f"pysus.dadosgov.{fn.__name__}(...), or " + f"pysus.saude.{fn.__name__}(...). Behavior is unchanged.", + PySUSWarning, + stacklevel=2, + ) + return fn(*args, **kwargs) + + return wrapped + + # ── Map canonical dataset names → Saude CKAN group slugs ───────── _SAUDE_GROUP_MAP: dict[str, str] = { "ARBOVIROSES": "arboviroses", @@ -985,3 +1032,12 @@ async def _list(): ] return pd.DataFrame(asyncio.run(_list())) + + +# ── Apply the flat-API deprecation wrapper to every public fetcher ─ +# Namespaced wrappers (``_bind_origin``/``bind_list_files``) suppress the +# warning, so only direct ``pysus.(...)`` calls are flagged. +for _name in __all__: + _obj = globals().get(_name) + if callable(_obj): + globals()[_name] = _deprecate_flat(_obj) diff --git a/pysus/api/_impl/source.py b/pysus/api/_impl/source.py index b39a59b1..42bcc795 100644 --- a/pysus/api/_impl/source.py +++ b/pysus/api/_impl/source.py @@ -383,7 +383,10 @@ def wrapped(*args, **kwargs): # internally, so only inject origin for the catalog-backed origins. if origin.upper() != SAUDE_ORIGIN: kwargs["origin"] = origin - return fn(*args, **kwargs) + from pysus.api._impl.databases import _suppress_flat_deprecation + + with _suppress_flat_deprecation(): + return fn(*args, **kwargs) wrapped.__name__ = fn.__name__ _annotate_bound(wrapped, fn, origin) @@ -431,15 +434,19 @@ def bound( month=None, **kwargs, ) -> pd.DataFrame: - return _list_files( - dataset=dataset, - client=client, - group=group, - state=state, - year=year, - month=month, - **kwargs, - ) + from pysus.api._impl.databases import _suppress_flat_deprecation + + with _suppress_flat_deprecation(): + result = _list_files( + dataset=dataset, + client=client, + group=group, + state=state, + year=year, + month=month, + **kwargs, + ) + return result bound.__name__ = "list_files" bound.__doc__ = ( diff --git a/pysus/tests/api/test_source.py b/pysus/tests/api/test_source.py index e95ff491..c69467e2 100644 --- a/pysus/tests/api/test_source.py +++ b/pysus/tests/api/test_source.py @@ -11,6 +11,8 @@ fetch, valid_origins, ) +from pysus.api.client import PySUS +from pysus.api.errors import PySUSWarning class TestOriginConstants: @@ -179,3 +181,129 @@ def test_dadosgov_omits_sih_sia_ciha_ibge(self): for name in ("sih", "sia", "ciha", "ibge"): assert not hasattr(pysus.dadosgov, name), name + + +class TestFlatDeprecation: + """Direct flat calls warn; namespaced calls are silent.""" + + def _catalog_empty(self, *args, **kwargs) -> None: + from unittest.mock import AsyncMock + + with patch( + "pysus.api._impl.source._fetch_catalog", + new_callable=AsyncMock, + return_value=[], + ): + pass + + def _flat_warns(self, call): + import warnings + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + call() + return [x for x in w if x.category is PySUSWarning] + + def _ns_silent(self, call): + import warnings + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + call() + return [x for x in w if x.category is PySUSWarning] + + def test_flat_sinan_warns(self): + import pysus + + with patch( + "pysus.api._impl.source._fetch_catalog", + new_callable=AsyncMock, + return_value=[], + ): + warned = self._flat_warns( + lambda: pysus.sinan( + disease="deng", year=2017, show_progress=False + ) + ) + assert len(warned) == 1 + assert "deprecated" in str(warned[0].message) + assert "pysus.ftp.sinan" in str(warned[0].message) + + def test_flat_saude_warns(self): + import pysus + + with patch( + "pysus.api._impl.source._fetch_catalog", + new_callable=AsyncMock, + return_value=[], + ): + warned = self._flat_warns( + lambda: pysus.arboviroses(show_progress=False) + ) + assert len(warned) == 1 + + def test_namespaced_sinan_silent(self): + import pysus + + with patch( + "pysus.api._impl.source._fetch_catalog", + new_callable=AsyncMock, + return_value=[], + ): + warned = self._ns_silent( + lambda: pysus.ftp.sinan( + disease="deng", year=2017, show_progress=False + ) + ) + assert warned == [] + + def test_namespaced_list_files_silent(self): + import pysus + + async def _q(**kwargs): + return [] + + with patch.object(PySUS, "query", new=AsyncMock(side_effect=_q)): + warned = self._ns_silent( + lambda: pysus.ftp.list_files("SINAN", year=2017, state="BR") + ) + assert warned == [] + + def test_flat_list_files_warns(self): + import pysus + + async def _q(**kwargs): + return [] + + with patch.object(PySUS, "query", new=AsyncMock(side_effect=_q)): + warned = self._flat_warns( + lambda: pysus.list_files("SINAN", year=2017, state="BR") + ) + assert len(warned) == 1 + + +class TestNamespacedValidation: + def test_rejects_origin_kwarg(self): + import pysus + from pysus.api.errors import PySUSError + + with pytest.raises(PySUSError): + pysus.ftp.sinan(disease="deng", year=2017, origin="FTP") + + def test_rejects_invalid_source(self): + import pysus + from pysus.api.errors import ValidationError + + with pytest.raises(ValidationError): + pysus.ftp.sinan(disease="deng", year=2017, source="bogus") + + def test_accepts_source_origin(self): + import pysus + + with patch( + "pysus.api._impl.source._fetch_origin_direct", + new_callable=AsyncMock, + return_value=[], + ) as mock_direct: + pysus.ftp.sinan(disease="deng", year=2017, source="origin") + mock_direct.assert_awaited_once() diff --git a/pysus/tests/conftest.py b/pysus/tests/conftest.py index c55637aa..274ad87f 100644 --- a/pysus/tests/conftest.py +++ b/pysus/tests/conftest.py @@ -1,4 +1,9 @@ -"""pytest configuration - mocks duckdb.functional before any other imports.""" +"""pytest configuration - mocks duckdb.functional before any other imports. + +Also silences the flat-API deprecation warning globally; dedicated tests +assert it fires by re-enabling it with ``warnings.catch_warnings`` / +``simplefilter("always")``. +""" import sys from unittest.mock import MagicMock @@ -7,3 +12,11 @@ _mock = MagicMock() _mock.SPECIAL = "SPECIAL" sys.modules["duckdb.functional"] = _mock + + +def pytest_configure(config): + import warnings + + from pysus.api.errors import PySUSWarning + + warnings.filterwarnings("ignore", category=PySUSWarning) From c70d8656fed2ce3c297e745569501ccc9bdef6db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 09:18:23 -0300 Subject: [PATCH 07/18] chore: untrack ROADMAP_ORIGIN_NAMESPACES.md --- .gitignore | 1 + ROADMAP_ORIGIN_NAMESPACES.md | 189 ----------------------------------- 2 files changed, 1 insertion(+), 189 deletions(-) delete mode 100644 ROADMAP_ORIGIN_NAMESPACES.md diff --git a/.gitignore b/.gitignore index 52206f2c..34c07d06 100644 --- a/.gitignore +++ b/.gitignore @@ -192,4 +192,5 @@ cython_debug/ .idea/ pyrightconfig.json roadmap*.md +ROADMAP_ORIGIN_NAMESPACES.md tests/epic_test.py diff --git a/ROADMAP_ORIGIN_NAMESPACES.md b/ROADMAP_ORIGIN_NAMESPACES.md deleted file mode 100644 index 6ff0e248..00000000 --- a/ROADMAP_ORIGIN_NAMESPACES.md +++ /dev/null @@ -1,189 +0,0 @@ -# Roadmap: origin-namespaced public API (`pysus.ftp.sinan`, `pysus.dadosgov.sinan`, `pysus.saude.*`) - -## Goal - -Make the data source a first-class, **impossible-to-ignore** part of the public -API. Origin namespaces replace the optional `origin=` kwarg. DuckLake is **not** -an origin — it is the shared catalog/S3 cache that every origin namespace reads -from by default, with the ability to bypass it and hit the origin server -directly: - -```python -from pysus import ftp, dadosgov, saude - -df = ftp.sinan(disease="deng", year=2020, as_dataframe=True) # FTP mirror from DuckLake/S3 catalog (default) -df = ftp.sinan(..., source="origin") # direct from the DATASUS FTP server -df = dadosgov.sinan(disease="deng", year=2020) # DadosGov mirror from catalog (default) -df = dadosgov.sinan(..., source="origin") # direct from ckan.saude.gov.br -df = saude.assistenciasaude(..., as_dataframe=True) # Saude-portal theme dataset - -ftp.list_files(dataset="sinan", year=2020) # discovery scoped to FTP -dadosgov.info() # discovery scoped to DadosGov -``` - -Note: `source` is one of `"catalog"` (default, reads the S3 mirror) or -`"origin"` (queries the origin server directly). The Saude portal has no -catalog mirror, so `saude.*` always queries the CKAN portal regardless of -`source`. - -Each namespace is self-contained: it exposes the same class of **per-database -fetchers** AND the same **discovery functions** (`list_files`, `info`) as the -top-level `pysus.__all__`, all scoped to that origin. - -This eliminates the bug class behind the DENG/2020 issue: a call that -**silently** served a different snapshot than the caller expected (the default -query returned the DadosGov mirror while the caller assumed FTP). - ---- - -## Existing pieces we already have (leverage, don't rebuild) - -- `_fetch_data(dataset, group, state, year, month, origin, ...)` — - `pysus/api/_impl/databases.py:76` dispatches `origin="SAUDE"` → - `_fetch_saude`, everything else → `_fetch_ducklake` with `client_filter` from - `FTP/DUCKLAKE/DADOSGOV` (`databases.py:167-173`). -- Origin aliases + validator — `pysus/api/types.py:128-130`, - `pysus/api/validate.py`. -- Per-origin catalog metadata already stored (`origin_path`, - `origin_modified`, `origin_size`, FTP `sha256`). -- The CLI is already origin-namespaced: `pysus/cli/{ftp,dadosgov,saude}.py` — - the Python API mirrors it. -- The public surface is table-driven: `pysus/api/_impl/__init__.py` `__all__` - → `pysus/__init__.py`. - ---- - -## Design (decisions locked) - -### A. Namespaces - -| namespace | default fetch | direct fetch | discovery | -|---|---|---|---| -| `pysus.ftp` | FTP mirror from DuckLake catalog | `source="origin"` | `ftp.list_files`, `ftp.info` | -| `pysus.dadosgov` | DadosGov mirror from DuckLake catalog | `source="origin"` | `dadosgov.list_files`, `dadosgov.info` | -| `pysus.saude` | Saude CKAN/theme (no mirror) | `source="origin"` (no-op) | `saude.list_files`, `saude.info` | - -- **No `pysus.ducklake` namespace.** DuckLake is transport/cache, not a source. -- `source="catalog"` is the default in every namespace ("catalog first"). - `source="origin"` downloads directly from the origin server into the local - cache (eligible for the existing resync/backfill engine). The Saude portal has - no catalog mirror, so `saude.*` always queries the CKAN portal. -- Namespaced functions reject `origin=` (PySUSError) and invalid `source=` - (ValidationError). -- **Legacy flat functions** (`pysus.sinan`, ...): deprecated-but-functional — - keep working, always emit a warning suggesting the namespaced call. Never - silently change their default behavior. - -### B. Function table (single source of truth) - -Each origin maps to a set of per-database fetchers **plus** `list_files`/`info`: - -| origin | per-database | discovery | -|---|---|---| -| `ftp` | core set applicable on FTP (sinan, sinasc, sim, sih, sia, ciha, cnes, pni, ibge, covid19, ...) | `ftp.list_files`, `ftp.info` | -| `dadosgov` | same core set **filtered to what CKAN actually publishes** | `dadosgov.list_files`, `dadosgov.info` | -| `saude` | the 16 themes (`arboviroses`, `assistenciasaude`, ... `vigilancia_meio_ambiente`) | `saude.list_files`, `saude.info` | - -**Phase 0 pre-work**: build + validate the applicability matrix from -`info_table`/`list_files` + catalog so we never expose a `dadosgov.sinasc` that -404s. - -**Decision**: DadosGov functions for datasets CKAN does **not** cover are -**omitted** from the namespace — not exposed-and-405. The namespace exposes only -what DadosGov actually publishes. - -### C. Namespaced functions via a factory (no copy-paste) - -- `_bind_origin(fn, origin)` → wrapper with origin fixed; signature-level param - validation. -- `build_origin_module(name, origin, fetchers, discovery)` → module with - `__all__`, docstrings, and `get_origin_meta()` exposing `origin_path`/ - `origin_modified`/`origin_size`/`sha256` (issue Q4). -- Register modules so both `pysus.ftp.sinan(...)` and `from pysus.ftp import - sinan` work. - ---- - -## Phases - -### Phase 0 — Foundation - -- [x] Typed internal primitive `fetch(dataset, ..., origin, source)` with - `origin ∈ {FTP, DadosGov, Saude}`, `source ∈ {catalog, origin}`. - (Implemented in `pysus/api/_impl/source.py`.) -- [x] Replace the ad-hoc mapping in `_fetch_ducklake` (`databases.py:167-173`) - with one lookup (`_client_filter` in `source.py`); add the - direct-origin path (`fetch(source="origin")`). -- [x] Build + lock the origin×dataset applicability matrix - (`APPLICABILITY` in `source.py`, incl. `list_files`/`info` scoping). -- [x] **Exit**: flat `pysus.sinan` unchanged; suite green (1553 → 1572 tests). -### Phase 1 — Namespace modules - -- [x] Factory + `pysus.ftp`, `pysus.dadosgov`, `pysus.saude` modules, each with - fetchers + discovery. (`_bind_origin`/`build_origin_module`/ - `install_origin_module` in `source.py`; module files at `pysus/{ftp, - dadosgov,saude}.py`, origin-aware docstrings on module + fetchers + - discovery.) -- [x] Register on package (`pysus/__init__.py`); verify both access styles and - `from pysus.ftp import sinan`. -- [x] Catalog-first default returns exactly today's `origin="FTP"`/ - `origin="DadosGov"` results (verified `pysus.ftp.sinan(DENG,2017)` - = 239,395 rows, matching `origin="FTP"`). -- [x] **Exit**: `pysus.ftp.sinan(...)` ≡ `pysus.sinan(..., origin="FTP")`; - `source="origin"` returns the same DataFrame after a live pull. - -### Phase 2 — Forced-verbose semantics - -- [x] Reject `origin=` and invalid `source=` in namespaced calls. - (`_bind_origin` raises `PySUSError` on a stray `origin=`; invalid - `source=` raises `ValidationError`.) -- [x] Legacy flat functions: deprecation warning + docstring pointing to the - namespaces; behavior unchanged. (`_deprecate_flat` wraps every flat - fetcher/`list_files` in `databases.py`; namespaced wrappers suppress via - `_suppress_flat_deprecation`, so only a *direct* flat call is flagged.) -- [x] Document `get_origin_meta()` per namespace (docstring on the bound - `get_origin_meta`). -- **Exit**: pre-commit chain green (1589 tests); flat call warns once per - call, namespaced call is silent. - -### Phase 3 — Tests & CI - -- [ ] `pysus/tests/api/test_origins.py`: - - namespaces exist with correct `__all__` (fetchers + discovery), - bound-wrapper identity; - - `pysus.ftp.sinan` routes to `client_filter=FTP`; `source="origin"` routes to - live FTP path; - - invalid `origin=` / `source=` rejected; discovery scoped per origin; - - `from pysus.ftp import sinan`, `pysus.ftp.list_files`, - `pysus.saude.assistenciasaude`; idempotent imports. -- [ ] `test_databases.py`: flat calls now warn (but still return identical data). -- **Exit**: full suite green (256 existing + new). - -### Phase 4 — Docs & UX - -- [ ] Docstrings + README lead with namespaced examples. -- [ ] `info_table`/`search` show per-origin hints (`pysus.ftp.sinan`). -- [ ] Migration guide (flat → namespaced; warning text), changelog, version bump. -- **Exit**: docs render; `pysus.info()` shows the namespaced call per row. - -### Phase 5 — Rollout & issue closure - -- [ ] Merge, tag, publish. -- [ ] Post follow-up on the DENG/2020 issue with the new syntax + comparison - table. - ---- - -## Risks & mitigations - -- **Default `source` staleness**: catalog-first can serve a stale mirror → - mitigated by `get_origin_meta()` (user sees `origin_modified`); - `stale_check`/`max_age` can be layered on later without changing the surface. -- **Import-time module generation / circular imports** → factory resolves - `_fetch_data` at call time. -- **`help()`/signature introspection** → generated wrappers get explicit - `__name__`/`__doc__`/signatures. -- **DadosGov gaps** → functions CKAN doesn't cover are **omitted** from the - namespace (decision above). -- **Name shadowing**: `pysus.ftp` (API) vs `pysus.cli.ftp` are distinct - namespaces — no conflict. From 465e4aec307a6134990c9805ec416cc34e2727f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 09:26:00 -0300 Subject: [PATCH 08/18] test: add Phase 3 origin-namespace and flat-deprecation tests --- pysus/tests/api/test_databases.py | 37 +++++++++ pysus/tests/api/test_origins.py | 130 ++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 pysus/tests/api/test_origins.py diff --git a/pysus/tests/api/test_databases.py b/pysus/tests/api/test_databases.py index 75e7ea22..e824f149 100644 --- a/pysus/tests/api/test_databases.py +++ b/pysus/tests/api/test_databases.py @@ -6,6 +6,8 @@ import pandas as pd import pytest +from pysus.api.client import PySUS +from pysus.api.errors import PySUSWarning class TestSinan: @@ -973,3 +975,38 @@ async def _run(): tmp_path.unlink(missing_ok=True) asyncio.run(_run()) + + +class TestFlatDeprecationWarns: + """Flat calls warn but still call _fetch_data with identical args.""" + + def test_flat_sinan_warns_and_passes_through(self): + import warnings + + from pysus.api._impl.databases import sinan + + with patch("pysus.api._impl.databases._fetch_data") as mock_fetch: + mock_fetch.return_value = MagicMock() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + sinan(disease="dengue", year=2024) + assert any(x.category is PySUSWarning for x in w) + args = mock_fetch.call_args + assert args.kwargs["dataset"] == "sinan" + assert args.kwargs["group"] == "DENGUE" + assert args.kwargs["year"] == 2024 + + def test_flat_list_files_warns_and_returns(self): + import warnings + + from pysus.api._impl.databases import list_files + + async def _q(**kwargs): + return [] + + with patch.object(PySUS, "query", new=AsyncMock(side_effect=_q)): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + df = list_files("SINAN", year=2024) + assert isinstance(df, pd.DataFrame) + assert any(x.category is PySUSWarning for x in w) diff --git a/pysus/tests/api/test_origins.py b/pysus/tests/api/test_origins.py new file mode 100644 index 00000000..23ef1a86 --- /dev/null +++ b/pysus/tests/api/test_origins.py @@ -0,0 +1,130 @@ +"""Phase 3 tests - public origin namespaces. + +Covers ``pysus.ftp`` / ``pysus.dadosgov`` / ``pysus.saude``. + +Focus areas defined by the roadmap: +- namespaces exist with the correct ``__all__`` (fetchers + discovery); +- bound-wrapper identity is stable between repeated accesses; +- ``pysus.ftp.sinan`` routes to ``client_filter='FTP'`` and ``source='origin'`` + routes to the live origin path; +- discovery (``list_files``) is scoped per origin; +- ``from pysus.ftp import sinan`` style and idempotent re-imports. +""" + +from unittest.mock import AsyncMock, patch + +import pytest +from pysus.api._impl.source import APPLICABILITY, origin_fetchers +from pysus.api.client import PySUS + + +@pytest.fixture() +def pysus(): + import pysus # noqa: F401 + + return pysus + + +def _public_fetchers(*names): + """Fetch + discovery names always present on every namespace.""" + base = set(names) + base.update({"list_files", "info", "get_origin_meta"}) + return base + + +class TestNamespaceExistence: + def test_registered_on_package(self, pysus): + for name in ("ftp", "dadosgov", "saude"): + assert hasattr(pysus, name) + assert name in pysus.__all__ + + def test_all_matches_fetchers_plus_discovery(self, pysus): + # The public module __all__ equals the platform's fetchers + discovery. + for ns, canonical in ( + (pysus.ftp, "FTP"), + (pysus.dadosgov, "DADOSGOV"), + (pysus.saude, "SAUDE"), + ): + fetched = origin_fetchers(canonical) + expected = _public_fetchers(*fetched) + assert set(ns.__all__) == expected, canonical + + def test_every_all_name_resolves(self, pysus): + for ns in (pysus.ftp, pysus.dadosgov, pysus.saude): + for name in ns.__all__: + assert callable(getattr(ns, name)), (ns, name) + + def test_all_scoped_to_applicability(self, pysus): + # A namespace exposes exactly the fetchers applicable to its origin. + for ns, canonical in ( + (pysus.ftp, "FTP"), + (pysus.dadosgov, "DADOSGOV"), + (pysus.saude, "SAUDE"), + ): + fetchers = { + n + for n in ns.__all__ + if n not in ("list_files", "info", "get_origin_meta") + } + assert fetchers == APPLICABILITY[canonical], canonical + + +class TestBoundIdentity: + def test_identity_stable_between_access(self, pysus): + for ns in (pysus.ftp, pysus.dadosgov, pysus.saude): + for name in ("sinan", "list_files", "info", "get_origin_meta"): + if not hasattr(ns, name): + continue + assert getattr(ns, name) is getattr(ns, name), (ns, name) + + def test_bound_differs_from_flat(self, pysus): + # The namespaced fetcher is a distinct wrapper, not the raw flat fn. + from pysus.api._impl import databases as _db + + assert pysus.ftp.sinan is not _db.sinan + assert pysus.ftp.sinan.__name__ == "sinan" + + +class TestIdempotentImports: + def test_reimport_keeps_working(self, pysus): + import importlib + + import pysus.ftp as ftp_mod + + for _ in range(2): + importlib.reload(ftp_mod) + assert callable(ftp_mod.sinan) + # package attribute still the module after the reloads + assert pysus.ftp is ftp_mod + + def test_from_import_after_reload(self, pysus): + import importlib + + import pysus.dadosgov as dg + + importlib.reload(dg) + from pysus.dadosgov import sinasc # noqa: F401 + + assert callable(sinasc) + + +class TestRouting: + def test_catalog_routes_to_ducklake_client_filter(self, pysus): + with patch.object(PySUS, "query", new_callable=AsyncMock) as query: + query.return_value = [] + pysus.ftp.list_files("SINAN", year=2017, state="BR") + # client filter is bound to FTP + assert query.call_args.kwargs["client"] == "FTP" + with patch.object(PySUS, "query", new_callable=AsyncMock) as query: + query.return_value = [] + pysus.dadosgov.list_files("SINAN", year=2017, state="BR") + assert query.call_args.kwargs["client"] == "DadosGov" + + def test_source_origin_routes_to_direct(self, pysus): + with patch( + "pysus.api._impl.source._fetch_origin_direct", + new_callable=AsyncMock, + return_value=[], + ) as direct: + pysus.ftp.sinan(disease="deng", year=2017, source="origin") + direct.assert_awaited_once() From 901dafb029a35b050e509bfb13cccc1522c5926f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 09:48:58 -0300 Subject: [PATCH 09/18] docs: migrate Phase 4 docs to origin namespaces; add info() call hints - Lead all docs (README, quickstart, data-sources, api, tutorials, migration, guides) with the origin-namespaced API - Add flat -> namespaced migration guide with deprecation warning text - Re-execute getting_started_pysus.ipynb with pysus.ftp.sinasc(...) - info_table()/search() now show per-origin fetch hints (pysus.ftp.sinan) that only appear when a real namespaced fetcher exists - Add tests for the new info() call-hint column --- README.md | 96 +- docs/source/api.rst | 12 +- docs/source/databases/data-sources.rst | 95 +- .../databases/getting_started_pysus.ipynb | 2721 +++++++++++++++-- docs/source/guides/files-and-formats.rst | 4 +- docs/source/guides/pysus-orchestrator.rst | 6 +- docs/source/migration.rst | 35 +- docs/source/quickstart.rst | 37 +- docs/source/tutorials.rst | 42 +- pysus/api/_impl/_ui.py | 64 +- pysus/tests/api/test_info.py | 23 + 11 files changed, 2826 insertions(+), 309 deletions(-) diff --git a/README.md b/README.md index 469436a0..d2c07442 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,18 @@ It downloads, converts, and analyses datasets from four independent sources — (OpenDataSUS), and **DuckLake** (S3 mirror) — and exposes them through a single, DataFrame-first API. +The data source is a first-class part of the public API: you reach each origin +through its own namespace (`pysus.ftp.*`, `pysus.dadosgov.*`, +`pysus.saude.*`), so the source of every dataset is explicit and impossible to +ignore. + ## Key features -- **One-line downloads** — `sinan("DENG", 2024, as_dataframe=True)` returns a - `pandas.DataFrame` in a single call. -- **Four data sources** — FTP, DadosGov, Saude (OpenDataSUS), DuckLake; the - orchestrator picks the best route automatically. +- **One-line downloads** — `pysus.ftp.sinan("DENG", 2024, as_dataframe=True)` + returns a `pandas.DataFrame` in a single call. +- **Origin-namespaced API** — `pysus.ftp`, `pysus.dadosgov`, and + `pysus.saude` make the data source explicit: `pysus.ftp.sinan(...)`, + `pysus.dadosgov.sinasc(...)`, `pysus.saude.arboviroses(...)`. - **Data quality** — `missing_values()`, `validate_data()`, `quality_score()`, and `profile_report()` give instant insight into completeness and schema integrity. @@ -76,19 +82,33 @@ docker compose down ### Download a dataset (one-liner) +The recommended way to fetch data is through an origin namespace. Each origin +exposes the same per-database fetchers: + ```python -from pysus import sinan, sinasc, sim, sih, sia, pni, ibge, cnes, ciha +import pysus -# Returns a list of local Parquet paths -parquet_files = sinan(disease="deng", year=2024) +# DATASUS FTP (served from the S3 catalog mirror by default) +df = pysus.ftp.sinan(disease="deng", year=2024, as_dataframe=True) -# Get a DataFrame directly -df = sinan(disease="deng", year=2024, as_dataframe=True) +# dados.gov.br (CKAN) +df = pysus.dadosgov.sinasc(state="SP", year=[2020, 2021, 2022, 2023], as_dataframe=True) -# Multiple years, filtered by state -df = sinasc(state="SP", year=[2020, 2021, 2022, 2023], as_dataframe=True) +# dadosabertos.saude.gov.br (theme datasets) +df = pysus.saude.arboviroses(year=2024, as_dataframe=True) +``` + +By default every namespace reads the S3/Parquet mirror (`source="catalog"`). +To query the origin server directly, pass `source="origin"`: + +```python +df = pysus.ftp.sinan(disease="deng", year=2024, source="origin", as_dataframe=True) ``` +The legacy flat functions (`pysus.sinan`, `pysus.arboviroses`, ...) still +work unchanged but emit a deprecation warning pointing you to the namespaced +call. + ### Browse available datasets ```python @@ -99,6 +119,16 @@ search("sinan") # fuzzy search across FTP, Saude, DadosGov list_files("SINAN") # list files within a dataset ``` +Discovery is also scoped per origin: + +```python +import pysus + +pysus.ftp.info() # datasets on the FTP origin +pysus.ftp.list_files("SINAN", year=2024, state="RJ") +pysus.dadosgov.get_origin_meta() # origin metadata +``` + ### The PySUS client (full control) ```python @@ -274,24 +304,32 @@ Precedence: explicit argument > environment variable > TOML file > default. ## Data sources -| Dataset | Description | FTP | DadosGov | Saude | DuckLake | -|---------|-------------|:---:|:--------:|:-----:|:--------:| -| SINAN | Disease notifications | x | x | x | x | -| SIM | Mortality | x | x | x | x | -| SINASC | Births | x | x | x | x | -| SIH | Hospitalisations | x | | | x | -| SIA | Ambulatory procedures | x | | | x | -| CIHA | Hospital admissions | x | | | x | -| CNES | Health facilities | x | x | x | x | -| PNI | Immunisations | x | x | x | x | -| IBGE | Geographic data | x | | | x | -| COVID19 | COVID-19 confirmed cases | x | x | x | x | -| Arboviroses | Arboviral diseases | | | x | | -| AssistenciaSaude | Health assistance | | | x | | -| AtencaoPrimaria | Primary care | | | x | | -| Vacinacao | Vaccination | | | x | | -| SisAgua | Water surveillance | | | x | | -| Sisvan | Nutritional surveillance | | | x | | +PySUS reads from three **origins** (FTP DataSUS, dados.gov.br, OpenDataSUS), +plus a shared **DuckLake/S3 mirror** that serves as the default cache. The +`DuckLake` mark below means the dataset is served from the S3 catalog mirror +by default via that origin namespace. + +| Dataset | Description | `pysus.ftp` | `pysus.dadosgov` | `pysus.saude` | +|---------|-------------|:---:|:--------:|:-----:| +| SINAN | Disease notifications | x | x | | +| SIM | Mortality | x | x | | +| SINASC | Births | x | x | | +| SIH | Hospitalisations | x | | | +| SIA | Ambulatory procedures | x | | | +| CIHA | Hospital admissions | x | | | +| CNES | Health facilities | x | x | | +| PNI | Immunisations | x | x | | +| IBGE | Geographic data | x | | | +| COVID19 | COVID-19 confirmed cases | x | x | | +| Arboviroses | Arboviral diseases | | | x | +| AssistenciaSaude | Health assistance | | | x | +| AtencaoPrimaria | Primary care | | | x | +| Vacinacao | Vaccination | | | x | +| SisAgua | Water surveillance | | | x | +| Sisvan | Nutritional surveillance | | | x | + +> **Note on Saude:** the Saude portal has no catalog mirror, so `pysus.saude.*` +> always queries the CKAN portal directly regardless of `source`. ## Architecture diff --git a/docs/source/api.rst b/docs/source/api.rst index bf6afde9..ea154682 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -3,7 +3,8 @@ API Reference The ``pysus.api`` package provides a layered architecture for discovering, downloading, and reading data from Brazilian public health databases -(DATASUS). It supports four remote data sources. +(DATASUS). It supports four remote data sources: FTP DataSUS, dados.gov.br, +OpenDataSUS (dadosabertos.saude.gov.br), and the DuckLake/S3 catalog mirror. Architecture Overview --------------------- @@ -31,12 +32,13 @@ concrete implementations:: Quick Start ----------- -The simplest way to use PySUS is via the high-level convenience -functions:: +The simplest way to use PySUS is through an origin namespace, which makes the +data source explicit:: - from pysus import sinan + import pysus - df = sinan(disease="dengue", year=2023) + df = pysus.ftp.sinan(disease="dengue", year=2023) + df = pysus.saude.arboviroses(disease="dengue", year=2023) Or with the async API:: diff --git a/docs/source/databases/data-sources.rst b/docs/source/databases/data-sources.rst index 0fda9981..4e4d48ab 100644 --- a/docs/source/databases/data-sources.rst +++ b/docs/source/databases/data-sources.rst @@ -7,80 +7,89 @@ Data Sources getting_started_pysus -PySUS provides simplified functions that return pandas DataFrames directly: +PySUS provides simplified, origin-namespaced functions that return pandas +DataFrames directly. Each origin is reachable through its own namespace — +``pysus.ftp.*``, ``pysus.dadosgov.*``, ``pysus.saude.*`` — so the data source +is explicit: .. code-block:: python - from pysus import sinan, sinasc, sim, sih, sia, pni, ibge, cnes, ciha + import pysus - # Download SINAN Dengue data - df = sinan(disease="deng", year=2024) + # Download SINAN Dengue data (DATASUS FTP, via the S3 catalog mirror) + df = pysus.ftp.sinan(disease="deng", year=2024) # Multiple years - df = sinan(disease="deng", year=[2023, 2024]) + df = pysus.ftp.sinan(disease="deng", year=[2023, 2024]) - # SINASC births for São Paulo - df = sinasc(state="SP", year=2024) + # SINASC births for São Paulo (dados.gov.br) + df = pysus.dadosgov.sinasc(state="SP", year=2024) - # SIM mortality data - df = sim(state="SP", year=2024) + # SIM mortality data — query the origin server directly + df = pysus.ftp.sim(state="SP", year=2024, source="origin") # SIH hospitalizations - df = sih(state="SP", year=2024, month=[1, 2, 3]) + df = pysus.ftp.sih(state="SP", year=2024, month=[1, 2, 3]) # CNES health facilities - df = cnes(state="SP", year=2024, month=1) + df = pysus.ftp.cnes(state="SP", year=2024, month=1) OpenDataSUS (Saude) functions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: python - from pysus import arboviroses, vacinacao, assistencia_saude + import pysus # Dengue/Chik/Zika notifications - df = arboviroses(disease="dengue", year=2024) + df = pysus.saude.arboviroses(disease="dengue", year=2024) # Vaccination coverage - df = vacinacao(state="SP", year=2024) + df = pysus.saude.vacinacao(state="SP", year=2024) # Hospital and health facility data - df = assistencia_saude(state="SP", year=2024) + df = pysus.saude.assistencia_saude(state="SP", year=2024) + +The legacy flat functions (``pysus.sinan``, ``pysus.arboviroses``, ...) still +work but emit a deprecation warning pointing to the namespaced form. Function Reference ^^^^^^^^^^^^^^^^^^ +Namespaced fetchers per origin. Both ``pysus.ftp.*`` and +``pysus.dadosgov.*`` read the S3 catalog mirror by default. + .. list-table:: :header-rows: 1 * - Function - Dataset - Parameters - * - ``sinan(disease, year)`` + * - ``ftp.sinan(...)`` / ``dadosgov.sinan(...)`` - Disease Notifications - disease (e.g., "DENG", "ZIKA"), year - * - ``sinasc(state, year, group)`` + * - ``ftp.sinasc(...)`` / ``dadosgov.sinasc(...)`` - Births - - state, year, group (optional) - * - ``sim(state, year, group)`` + - state, year + * - ``ftp.sim(...)`` / ``dadosgov.sim(...)`` - Mortality - - state, year, group (optional) - * - ``sih(state, year, month, group)`` + - state, year + * - ``ftp.sih(...)`` - Hospitalizations - - state, year, month, group (optional) - * - ``sia(state, year, month, group)`` + - state, year, month + * - ``ftp.sia(...)`` - Ambulatory - - state, year, month, group (optional) - * - ``pni(state, year, group)`` + - state, year, month + * - ``ftp.pni(...)`` / ``dadosgov.pni(...)`` - Immunizations - - state, year, group (optional) - * - ``ibge(year, group)`` + - state, year + * - ``ftp.ibge(...)`` - IBGE - - year, group (optional) - * - ``cnes(state, year, month, group)`` + - year + * - ``ftp.cnes(...)`` / ``dadosgov.cnes(...)`` - Health Facilities - - state, year, month, group (optional) - * - ``ciha(state, year, month)`` + - state, year, month + * - ``ftp.ciha(...)`` - Hospital Admissions - state, year, month @@ -93,34 +102,36 @@ OpenDataSUS (Saude) Functions * - Function - Dataset - Parameters - * - ``arboviroses(**kwargs)`` + * - ``saude.arboviroses(**kwargs)`` - Arboviroses (Dengue/Chik/Zika/YF) - disease, state, year (via kwargs) - * - ``vacinacao(**kwargs)`` + * - ``saude.vacinacao(**kwargs)`` - Vaccination Coverage - state, year (via kwargs) - * - ``assistencia_saude(**kwargs)`` + * - ``saude.assistencia_saude(**kwargs)`` - Hospital/Facility Data - state, year (via kwargs) - * - ``atencao_primaria(**kwargs)`` + * - ``saude.atencao_primaria(**kwargs)`` - Primary Care (Previne Brasil) - state, year (via kwargs) - * - ``sisvan(**kwargs)`` + * - ``saude.sisvan(**kwargs)`` - Nutrition Surveillance - state, year (via kwargs) - * - ``sisagua(**kwargs)`` + * - ``saude.sisagua(**kwargs)`` - Water Quality - state, year (via kwargs) - * - ``covid19(**kwargs)`` - - COVID-19 - - state, year (via kwargs) - * - ``bnafar(**kwargs)`` + * - ``saude.bnafar(**kwargs)`` - Pharmaceutical Assistance - state, year (via kwargs) - * - ``saude_indigena(**kwargs)`` + * - ``saude.saude_indigena(**kwargs)`` - Indigenous Health - state, year (via kwargs) +The ``source`` parameter is accepted everywhere: ``source="catalog"`` (default) +serves the S3/Parquet mirror; ``source="origin"`` queries the origin server +directly. The Saude portal has no catalog mirror, so ``saude.*`` always +queries the CKAN portal. + Using the PySUS Client ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/databases/getting_started_pysus.ipynb b/docs/source/databases/getting_started_pysus.ipynb index 7648a1e1..7ba9fd14 100644 --- a/docs/source/databases/getting_started_pysus.ipynb +++ b/docs/source/databases/getting_started_pysus.ipynb @@ -56,213 +56,2513 @@ "cell_type": "code", "execution_count": 1, "id": "cd-8334704359266914152", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-31T12:46:33.495957Z", + "iopub.status.busy": "2026-08-31T12:46:33.495836Z", + "iopub.status.idle": "2026-08-31T12:46:34.514632Z", + "shell.execute_reply": "2026-08-31T12:46:34.514136Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pysus in /home/bida/.local/lib/python3.12/site-packages (2.9.0)\r\n", + "Requirement already satisfied: Unidecode<2.0.0,>=1.3.6 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (1.4.0)\r\n", + "Requirement already satisfied: aioftp<0.22.0,>=0.21.4 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (0.21.4)\r\n", + "Requirement already satisfied: anyio<5.0.0,>=4.13.0 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (4.14.2)\r\n", + "Requirement already satisfied: bigtree<0.13.0,>=0.12.2 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (0.12.5)\r\n", + "Requirement already satisfied: boto3<2.0.0,>=1.42.89 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (1.43.78)\r\n", + "Requirement already satisfied: chardet<8.0.0,>=7.4.0.post2 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (7.6.0)\r\n", + "Requirement already satisfied: dateparser<2.0.0,>=1.1.8 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (1.4.2)\r\n", + "Requirement already satisfied: dbfread==2.0.7 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (2.0.7)\r\n", + "Requirement already satisfied: dotenv<0.10.0,>=0.9.9 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (0.9.9)\r\n", + "Requirement already satisfied: duckdb<2.0.0,>=1.4.4 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (1.5.4)\r\n", + "Requirement already satisfied: duckdb-engine<0.18.0,>=0.17.0 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (0.17.0)\r\n", + "Requirement already satisfied: fastparquet<=2024.11.0,>=2023.10.1 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (2024.11.0)\r\n", + "Requirement already satisfied: httpx>=0.28.0 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (0.28.1)\r\n", + "Requirement already satisfied: humanize<5.0.0,>=4.8.0 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (4.16.0)\r\n", + "Requirement already satisfied: loguru<0.7.0,>=0.6.0 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (0.6.0)\r\n", + "Requirement already satisfied: numpy>=2.4.0 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (2.5.2)\r\n", + "Requirement already satisfied: pandas<3.0.0,>=2.2.2 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (2.3.3)\r\n", + "Requirement already satisfied: pyarrow>=11.0.0 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (25.0.1)\r\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.12.5 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (2.13.4)\r\n", + "Requirement already satisfied: pyreaddbc>=2.0.4 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (2.0.4)\r\n", + "Requirement already satisfied: python-dateutil==2.8.2 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (2.8.2)\r\n", + "Requirement already satisfied: sqlalchemy<3.0.0,>=2.0.48 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (2.0.52)\r\n", + "Requirement already satisfied: tqdm>=4.67.0 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (4.70.0)\r\n", + "Requirement already satisfied: typer<0.25.0,>=0.24.1 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (0.24.2)\r\n", + "Requirement already satisfied: typing-extensions>=4.10.0 in /home/bida/.local/lib/python3.12/site-packages (from pysus) (4.16.0)\r\n", + "Requirement already satisfied: six>=1.5 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from python-dateutil==2.8.2->pysus) (1.17.0)\r\n", + "Requirement already satisfied: idna>=2.8 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from anyio<5.0.0,>=4.13.0->pysus) (3.16)\r\n", + "Requirement already satisfied: botocore<1.44.0,>=1.43.78 in /home/bida/.local/lib/python3.12/site-packages (from boto3<2.0.0,>=1.42.89->pysus) (1.43.78)\r\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/bida/.local/lib/python3.12/site-packages (from boto3<2.0.0,>=1.42.89->pysus) (1.1.0)\r\n", + "Requirement already satisfied: s3transfer<0.20.0,>=0.19.0 in /home/bida/.local/lib/python3.12/site-packages (from boto3<2.0.0,>=1.42.89->pysus) (0.19.2)\r\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from botocore<1.44.0,>=1.43.78->boto3<2.0.0,>=1.42.89->pysus) (2.7.0)\r\n", + "Requirement already satisfied: pytz>=2024.2 in /home/bida/.local/lib/python3.12/site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (2026.3.post1)\r\n", + "Requirement already satisfied: regex>=2024.9.11 in /home/bida/.local/lib/python3.12/site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (2026.7.19)\r\n", + "Requirement already satisfied: tzlocal>=0.2 in /home/bida/.local/lib/python3.12/site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (5.4.4)\r\n", + "Requirement already satisfied: python-dotenv in /home/bida/.local/lib/python3.12/site-packages (from dotenv<0.10.0,>=0.9.9->pysus) (1.2.3)\r\n", + "Requirement already satisfied: packaging>=21 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from duckdb-engine<0.18.0,>=0.17.0->pysus) (26.2)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: cramjam>=2.3 in /home/bida/.local/lib/python3.12/site-packages (from fastparquet<=2024.11.0,>=2023.10.1->pysus) (2.12.0)\r\n", + "Requirement already satisfied: fsspec in /home/bida/.local/lib/python3.12/site-packages (from fastparquet<=2024.11.0,>=2023.10.1->pysus) (2026.7.0)\r\n", + "Requirement already satisfied: tzdata>=2022.7 in /home/bida/.local/lib/python3.12/site-packages (from pandas<3.0.0,>=2.2.2->pysus) (2026.3)\r\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /home/bida/.local/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (0.8.0)\r\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /home/bida/.local/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (2.46.4)\r\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /home/bida/.local/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (0.4.4)\r\n", + "Requirement already satisfied: greenlet>=1 in /home/bida/.local/lib/python3.12/site-packages (from sqlalchemy<3.0.0,>=2.0.48->pysus) (3.5.5)\r\n", + "Requirement already satisfied: click>=8.2.1 in /home/bida/.local/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (8.4.2)\r\n", + "Requirement already satisfied: shellingham>=1.3.0 in /home/bida/.local/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (1.5.4)\r\n", + "Requirement already satisfied: rich>=12.3.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (15.0.0)\r\n", + "Requirement already satisfied: annotated-doc>=0.0.2 in /home/bida/.local/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (0.0.5)\r\n", + "Requirement already satisfied: certifi in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from httpx>=0.28.0->pysus) (2026.5.20)\r\n", + "Requirement already satisfied: httpcore==1.* in /home/bida/.local/lib/python3.12/site-packages (from httpx>=0.28.0->pysus) (1.0.9)\r\n", + "Requirement already satisfied: h11>=0.16 in /home/bida/.local/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.28.0->pysus) (0.16.0)\r\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (4.2.0)\r\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (2.20.0)\r\n", + "Requirement already satisfied: mdurl~=0.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (0.1.2)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "# Run this cell if PySUS is not yet installed\n", + "%pip install pysus" + ] + }, + { + "cell_type": "markdown", + "id": "md-7861405087406607636", + "metadata": {}, + "source": [ + "---\n", + "## 2. Checking Your Installation\n", + "\n", + "After installing PySUS, verify that the package is available and check the installed version." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cd-2865917029134968551", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-31T12:46:34.516309Z", + "iopub.status.busy": "2026-08-31T12:46:34.516196Z", + "iopub.status.idle": "2026-08-31T12:46:35.258549Z", + "shell.execute_reply": "2026-08-31T12:46:35.258003Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.9.0\n" + ] + } + ], + "source": [ + "import pysus\n", + "\n", + "print(pysus.get_version())" + ] + }, + { + "cell_type": "markdown", + "id": "md-7886435397085505865", + "metadata": {}, + "source": [ + "---\n", + "## 3. Exploring the Package\n", + "\n", + "PySUS groups its fetchers under origin namespaces (``pysus.ftp``,\n", + "``pysus.dadosgov``, ``pysus.saude``) so the data source is always explicit.\n", + "Call ``pysus.info()`` for a table of everything available and the exact\n", + "namespaced call to use:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "cd-3896900325229140893", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-31T12:46:35.259993Z", + "iopub.status.busy": "2026-08-31T12:46:35.259792Z", + "iopub.status.idle": "2026-08-31T12:46:35.463261Z", + "shell.execute_reply": "2026-08-31T12:46:35.462781Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " -----------------------------------------------------------------------------------------------\n", + " Name Origin Auth Call Description\n", + " -----------------------------------------------------------------------------------------------\n", + " CIHA FTP no pysus.ftp.ciha(...) Hospital & ambulatory admission records\n", + " CNES FTP no pysus.ftp.cnes(...) Health facility registry\n", + " IBGEDATASUS FTP no pysus.ftp.ibge(...) Population & census data (IBGE)\n", + " PNI FTP no pysus.ftp.pni(...) National immunisation programme\n", + " SIA FTP no pysus.ftp.sia(...) Ambulatory care information system\n", + " SIH FTP no pysus.ftp.sih(...) Hospital admission information system\n", + " SIM FTP no pysus.ftp.sim(...) Mortality information system\n", + " SINAN FTP no pysus.ftp.sinan(...) Notifiable disease information system\n", + " SINASC FTP no pysus.ftp.sinasc(...) Live birth information system\n", + " ARBOVIROSES Saude no pysus.saude.arboviroses(...) Arboviroses (dengue, chikungunya, zika, yellow fever)\n", + " ASSISTENCIASAUDE Saude no pysus.saude.assistencia_saude(...) Hospital and health facility data\n", + " ATENCAOPRIMARIA Saude no pysus.saude.atencao_primaria(...) Primary care (Previne Brasil, SISAB)\n", + " BNAFAR Saude no pysus.saude.bnafar(...) Pharmaceutical assistance (Hórus medication stock)\n", + " CNES Saude no Cadastro Nacional de Estabelecimentos de Saúde\n", + " CIENCIATECNOLOGIA Saude no pysus.saude.ciencia_tecnologia(...) Science & technology (Conitec, RIPSA)\n", + " DIAGNOSTICOSTRATAMENTOS Saude no pysus.saude.diagnosticos_tratamentos(...) Diagnostics & treatment protocols\n", + " ECONOMIASAUDE Saude no pysus.saude.economia_saude(...) Health economics (BPS, ApuraSUS, SIOPS)\n", + " EDUCACAOSAUDE Saude no pysus.saude.educacao_saude(...) Health education (PVC)\n", + " MACROSAUDE Saude no pysus.saude.macro_saude(...) Macro-regions and health regions (MGDI)\n", + " OUVIDORIA Saude no pysus.saude.ouvidoria(...) SUS ombudsman complaints\n", + " OUTROSTEMAS Saude no pysus.saude.outros_temas(...) Miscellaneous CED coordination data\n", + " PDA Saude no pysus.saude.pda(...) Digital health and open data plan\n", + " PREVENCAOPROMOCAO Saude no pysus.saude.prevencao_promocao(...) Prevention & promotion (EPI distribution)\n", + " SISAGUA Saude no pysus.saude.sisagua(...) Water quality surveillance\n", + " SISVAN Saude no pysus.saude.sisvan(...) Food & nutrition surveillance\n", + " SAUDEINDIGENA Saude no pysus.saude.saude_indigena(...) Indigenous health (Siasi/SasiSUS/Sesai)\n", + " VACINACAO Saude no pysus.saude.vacinacao(...) Vaccination (PNI doses, ESAVI)\n", + " CNES DadosGov yes pysus.dadosgov.cnes(...) Health facility registry\n", + " PNI DadosGov yes pysus.dadosgov.pni(...) National immunisation programme\n", + " SIM DadosGov yes pysus.dadosgov.sim(...) Mortality information system\n", + " SINAN DadosGov yes pysus.dadosgov.sinan(...) Notifiable disease information system\n", + " SINASC DadosGov yes pysus.dadosgov.sinasc(...) Live birth information system\n", + " COVID19 DadosGov yes pysus.dadosgov.covid19(...) Confirmed COVID-19 cases\n", + " -----------------------------------------------------------------------------------------------\n", + "\n", + " Total: 33 datasets | Cache: /home/bida/pysus\n" + ] + } + ], + "source": [ + "import pysus\n", + "\n", + "pysus.info()\n" + ] + }, + { + "cell_type": "markdown", + "id": "md-5907404130116600740", + "metadata": {}, + "source": [ + "The main dataset functions are:\n", + "\n", + "| Call | Dataset | Description |\n", + "|------|---------|-------------|\n", + "| `pysus.ftp.sinasc(...)` | SINASC | Live birth records |\n", + "| `pysus.ftp.sim(...)` | SIM | Mortality records |\n", + "| `pysus.ftp.sinan(...)` | SINAN | Notifiable diseases |\n", + "| `pysus.ftp.sih(...)` | SIH | Hospital admissions |\n", + "| `pysus.ftp.sia(...)` | SIA | Outpatient procedures |\n", + "| `pysus.ftp.cnes(...)` | CNES | Health facilities |\n", + "| `pysus.ftp.pni(...)` | PNI | Immunisation programme |\n", + "| `pysus.ftp.ibge(...)` | IBGE | Demographic data |\n" + ] + }, + { + "cell_type": "markdown", + "id": "md-6714993952696256954", + "metadata": {}, + "source": [ + "---\n", + "## 4. Understanding the Parameters\n", + "\n", + "All dataset functions share the same unified parameter pattern. For instance, querying `pysus.ftp.sinasc()` can be targeted like this:\n", + "\n", + "```python\n", + "pysus.ftp.sinasc(\n", + " state = \"SP\", # two-letter Brazilian state code\n", + " year = 2022, # integer, list, or range of integers\n", + ")\n", + "```\n", + "\n", + "| Parameter | Type | Description | Example |\n", + "|-----------|------|-------------|---------|\n", + "| `state` | `str` | Two-letter state abbreviation | `\"SP\"`, `\"RJ\"`, `\"MG\"` |\n", + "| `year` | `int`, `list` or `range` | Targets execution span | `2022` or `range(2020, 2026)` |\n", + "| `group` | `str` or `None` | Sub-group code (SINAN only) | `\"DENG\"` (dengue) |\n", + "\n", + "By default fetchers read from the ``\"catalog\"`` mirror; pass\n", + "``source=\"origin\"`` to query the origin server directly.\n" + ] + }, + { + "cell_type": "markdown", + "id": "md-1656243573859326293", "metadata": {}, + "source": [ + "---\n", + "## 5. Downloading Your First Dataset\n", + "\n", + "Let's download SINASC birth records for Rio de Janeiro, 2022.\n", + "By default, the function handles throttled parallel downloads and returns a list of local file paths tracking your parquet targets." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cd-3795208869041744895", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-31T12:46:35.464729Z", + "iopub.status.busy": "2026-08-31T12:46:35.464613Z", + "iopub.status.idle": "2026-08-31T12:46:43.289447Z", + "shell.execute_reply": "2026-08-31T12:46:43.288913Z" + } + }, "outputs": [ { - "name": "stdout", + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Downloading sinasc: 0%| | 0/1 [00:00=1.3.6 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.4.0)\n", - "Requirement already satisfied: aioftp<0.22.0,>=0.21.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.21.4)\n", - "Requirement already satisfied: anyio<5.0.0,>=4.13.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (4.13.0)\n", - "Requirement already satisfied: bigtree<0.13.0,>=0.12.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.12.5)\n", - "Requirement already satisfied: boto3<2.0.0,>=1.42.89 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.43.14)\n", - "Requirement already satisfied: chardet<8.0.0,>=7.4.0.post2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (7.4.3)\n", - "Requirement already satisfied: dateparser<2.0.0,>=1.1.8 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.4.0)\n", - "Requirement already satisfied: dbfread==2.0.7 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.0.7)\n", - "Requirement already satisfied: dotenv<0.10.0,>=0.9.9 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.9.9)\n", - "Requirement already satisfied: duckdb<2.0.0,>=1.4.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.5.3)\n", - "Requirement already satisfied: duckdb-engine<0.18.0,>=0.17.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.17.0)\n", - "Requirement already satisfied: fastparquet<=2024.11.0,>=2023.10.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2024.11.0)\n", - "Requirement already satisfied: httpx>=0.28.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.28.1)\n", - "Requirement already satisfied: loguru<0.7.0,>=0.6.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.6.0)\n", - "Requirement already satisfied: numpy<2,>=1.22 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (1.26.4)\n", - "Requirement already satisfied: pandas<3.0.0,>=2.2.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.3.3)\n", - "Requirement already satisfied: pyarrow>=11.0.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (24.0.0)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.12.5 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.13.4)\n", - "Requirement already satisfied: pyreaddbc>=2.0.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.0.4)\n", - "Requirement already satisfied: python-dateutil==2.8.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.8.2)\n", - "Requirement already satisfied: python-magic<0.5.0,>=0.4.27 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.4.27)\n", - "Requirement already satisfied: sqlalchemy<3.0.0,>=2.0.48 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (2.0.50)\n", - "Requirement already satisfied: tqdm>=4.67.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (4.67.3)\n", - "Requirement already satisfied: typer<0.25.0,>=0.24.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (0.24.2)\n", - "Requirement already satisfied: typing-extensions>=4.10.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (4.15.0)\n", - "Requirement already satisfied: wget<4.0,>=3.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pysus) (3.2)\n", - "Requirement already satisfied: six>=1.5 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from python-dateutil==2.8.2->pysus) (1.17.0)\n", - "Requirement already satisfied: idna>=2.8 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from anyio<5.0.0,>=4.13.0->pysus) (3.16)\n", - "Requirement already satisfied: botocore<1.44.0,>=1.43.14 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from boto3<2.0.0,>=1.42.89->pysus) (1.43.14)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from boto3<2.0.0,>=1.42.89->pysus) (1.1.0)\n", - "Requirement already satisfied: s3transfer<0.18.0,>=0.17.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from boto3<2.0.0,>=1.42.89->pysus) (0.17.0)\n", - "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from botocore<1.44.0,>=1.43.14->boto3<2.0.0,>=1.42.89->pysus) (2.7.0)\n", - "Requirement already satisfied: pytz>=2024.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (2026.2)\n", - "Requirement already satisfied: regex>=2024.9.11 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (2026.5.9)\n", - "Requirement already satisfied: tzlocal>=0.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from dateparser<2.0.0,>=1.1.8->pysus) (5.3.1)\n", - "Requirement already satisfied: python-dotenv in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from dotenv<0.10.0,>=0.9.9->pysus) (1.2.2)\n", - "Requirement already satisfied: packaging>=21 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from duckdb-engine<0.18.0,>=0.17.0->pysus) (26.2)\n", - "Requirement already satisfied: cramjam>=2.3 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from fastparquet<=2024.11.0,>=2023.10.1->pysus) (2.11.0)\n", - "Requirement already satisfied: fsspec in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from fastparquet<=2024.11.0,>=2023.10.1->pysus) (2026.4.0)\n", - "Requirement already satisfied: tzdata>=2022.7 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pandas<3.0.0,>=2.2.2->pysus) (2026.2)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.46.4 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (2.46.4)\n", - "Requirement already satisfied: typing-inspection>=0.4.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.12.5->pysus) (0.4.2)\n", - "Requirement already satisfied: greenlet>=1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from sqlalchemy<3.0.0,>=2.0.48->pysus) (3.5.1)\n", - "Requirement already satisfied: click>=8.2.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (8.4.1)\n", - "Requirement already satisfied: shellingham>=1.3.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (1.5.4)\n", - "Requirement already satisfied: rich>=12.3.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (15.0.0)\n", - "Requirement already satisfied: annotated-doc>=0.0.2 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from typer<0.25.0,>=0.24.1->pysus) (0.0.4)\n", - "Requirement already satisfied: certifi in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from httpx>=0.28.0->pysus) (2026.5.20)\n", - "Requirement already satisfied: httpcore==1.* in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from httpx>=0.28.0->pysus) (1.0.9)\n", - "Requirement already satisfied: h11>=0.16 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.28.0->pysus) (0.16.0)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (4.2.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (2.20.0)\n", - "Requirement already satisfied: mdurl~=0.1 in /home/bida/micromamba/envs/pysus/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich>=12.3.0->typer<0.25.0,>=0.24.1->pysus) (0.1.2)\n", - "Note: you may need to restart the kernel to use updated packages.\n" + "\r", + "DNRJ2022.parquet: 94%|█████████▍| 6.03M/6.41M [00:01<00:00, 4.49MB/s]" ] - } - ], - "source": [ - "# Run this cell if PySUS is not yet installed\n", - "%pip install pysus" - ] - }, - { - "cell_type": "markdown", - "id": "md-7861405087406607636", - "metadata": {}, - "source": [ - "---\n", - "## 2. Checking Your Installation\n", - "\n", - "After installing PySUS, verify that the package is available and check the installed version." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "cd-2865917029134968551", - "metadata": {}, - "outputs": [ + }, { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "2.1.0\n" + "\u001b[A" ] - } - ], - "source": [ - "import pysus\n", - "\n", - "print(pysus.get_version())" - ] - }, - { - "cell_type": "markdown", - "id": "md-7886435397085505865", - "metadata": {}, - "source": [ - "---\n", - "## 3. Exploring the Package\n", - "\n", - "PySUS exposes each health dataset as a simple callable function.\n", - "Use `dir(pysus)` to see the API entry points available:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "cd-3896900325229140893", - "metadata": {}, - "outputs": [ + }, { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "['CACHEPATH', 'Final', 'api', 'ciha', 'cnes', 'get_version', 'ibge', 'importlib_metadata', 'list_files', 'os', 'pathlib', 'pni', 'sia', 'sih', 'sim', 'sinan', 'sinasc', 'version']\n" + "\n" ] - } - ], - "source": [ - "import pysus\n", - "\n", - "print([item for item in dir(pysus) if not item.startswith('_')])" - ] - }, - { - "cell_type": "markdown", - "id": "md-5907404130116600740", - "metadata": {}, - "source": [ - "The main dataset functions are:\n", - "\n", - "| Function | Dataset | Description |\n", - "|----------|---------|-------------|\n", - "| `sinasc()` | SINASC | Live birth records |\n", - "| `sim()` | SIM | Mortality records |\n", - "| `sinan()` | SINAN | Notifiable diseases |\n", - "| `sih()` | SIH | Hospital admissions |\n", - "| `sia()` | SIA | Outpatient procedures |\n", - "| `cnes()` | CNES | Health facilities |\n", - "| `pni()` | PNI | Immunisation programme |\n", - "| `ibge()` | IBGE | Demographic data |" - ] - }, - { - "cell_type": "markdown", - "id": "md-6714993952696256954", - "metadata": {}, - "source": [ - "---\n", - "## 4. Understanding the Parameters\n", - "\n", - "All dataset functions share the same unified parameter pattern. For instance, querying `sinasc()` can be targeted like this:\n", - "\n", - "```python\n", - "sinasc(\n", - " state = \"SP\", # two-letter Brazilian state code\n", - " year = 2022, # integer, list, or range of integers\n", - ")\n", - "```\n", - "\n", - "| Parameter | Type | Description | Example |\n", - "|-----------|------|-------------|---------|\n", - "| `state` | `str` | Two-letter state abbreviation | `\"SP\"`, `\"RJ\"`, `\"MG\"` |\n", - "| `year` | `int`, `list` or `range` | Targets execution span | `2022` or `range(2020, 2026)` |\n", - "| `group` | `str` or `None` | Sub-group code (SINAN only) | `\"DENG\"` (dengue) |\n", - "| `as_dataframe` | `bool` | Instantly return a Pandas Dataframe | `True` or `False` |" - ] - }, - { - "cell_type": "markdown", - "id": "md-1656243573859326293", - "metadata": {}, - "source": [ - "---\n", - "## 5. Downloading Your First Dataset\n", - "\n", - "Let's download SINASC birth records for Rio de Janeiro, 2022.\n", - "By default, the function handles throttled parallel downloads and returns a list of local file paths tracking your parquet targets." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "cd-3795208869041744895", - "metadata": {}, - "outputs": [ + }, { "name": "stderr", "output_type": "stream", "text": [ - "Downloading sinasc: 100%|█████████████████████████████████████████████████| 1/1 [00:02<00:00, 2.88s/file]" + "\r", + "DNRJ2022.parquet: 95%|█████████▌| 6.09M/6.41M [00:01<00:00, 4.53MB/s]" ] }, { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "Parquet file targets saved locally: ['/home/bida/pysus/downloads/ducklake/sinasc/DNRJ2022.parquet']\n" + "\u001b[A" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "DNRJ2022.parquet: 96%|█████████▌| 6.16M/6.41M [00:01<00:00, 4.57MB/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[A" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "DNRJ2022.parquet: 97%|█████████▋| 6.23M/6.41M [00:01<00:00, 4.61MB/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[A" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "DNRJ2022.parquet: 98%|█████████▊| 6.29M/6.41M [00:01<00:00, 4.65MB/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[A" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "DNRJ2022.parquet: 99%|█████████▉| 6.36M/6.41M [00:01<00:00, 4.69MB/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[A" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "DNRJ2022.parquet: 100%|██████████| 6.41M/6.41M [00:01<00:00, 4.72MB/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[A" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "DNRJ2022.parquet: 100%|██████████| 6.41M/6.41M [00:01<00:00, 4.61MB/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "\r", + "Downloading sinasc: 100%|██████████| 1/1 [00:03<00:00, 3.42s/file]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Downloading sinasc: 100%|██████████| 1/1 [00:03<00:00, 3.42s/file]" ] }, { @@ -271,18 +2571,25 @@ "text": [ "\n" ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Parquet file targets saved locally: ['/home/bida/pysus/downloads/ducklake/sinasc/DNRJ2022.parquet']\n" + ] } ], "source": [ - "from pysus import sinasc\n", + "import pysus\n", "import pandas as pd\n", "\n", "# Fetch local storage paths for the dataset\n", - "files = sinasc(state=\"RJ\", year=2022)\n", + "files = pysus.ftp.sinasc(state=\"RJ\", year=2022)\n", "print(f\"Parquet file targets saved locally: {files}\")\n", "\n", "# Open records with pandas\n", - "df = pd.read_parquet(files)" + "df = pd.read_parquet(files)\n" ] }, { @@ -292,8 +2599,8 @@ "source": [ "> **Tip:** To download an entire multi-year range automatically into a unified dataframe framework, pass `as_dataframe=True`:\n", "> ```python\n", - "> df = sinasc(state=\"SP\", year=range(2020, 2025), as_dataframe=True)\n", - "> ```" + "> df = pysus.ftp.sinasc(state=\"SP\", year=range(2020, 2025), as_dataframe=True)\n", + "> ```\n" ] }, { @@ -311,7 +2618,14 @@ "cell_type": "code", "execution_count": 5, "id": "cd-4586221015219041710", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-31T12:46:43.291310Z", + "iopub.status.busy": "2026-08-31T12:46:43.290980Z", + "iopub.status.idle": "2026-08-31T12:46:43.293611Z", + "shell.execute_reply": "2026-08-31T12:46:43.293226Z" + } + }, "outputs": [ { "name": "stdout", @@ -330,7 +2644,14 @@ "cell_type": "code", "execution_count": 6, "id": "cd-6113157604871589284", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-31T12:46:43.295139Z", + "iopub.status.busy": "2026-08-31T12:46:43.295006Z", + "iopub.status.idle": "2026-08-31T12:46:43.606836Z", + "shell.execute_reply": "2026-08-31T12:46:43.606346Z" + } + }, "outputs": [ { "name": "stdout", @@ -416,7 +2737,14 @@ "cell_type": "code", "execution_count": 7, "id": "cd-1603410318595514302", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-31T12:46:43.608084Z", + "iopub.status.busy": "2026-08-31T12:46:43.607975Z", + "iopub.status.idle": "2026-08-31T12:46:44.299705Z", + "shell.execute_reply": "2026-08-31T12:46:44.299010Z" + } + }, "outputs": [ { "data": { @@ -632,7 +2960,14 @@ "cell_type": "code", "execution_count": 8, "id": "cd-7840088167428654087", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-31T12:46:44.301087Z", + "iopub.status.busy": "2026-08-31T12:46:44.300981Z", + "iopub.status.idle": "2026-08-31T12:46:44.732699Z", + "shell.execute_reply": "2026-08-31T12:46:44.732260Z" + } + }, "outputs": [ { "data": { diff --git a/docs/source/guides/files-and-formats.rst b/docs/source/guides/files-and-formats.rst index afb3c2ee..e3740c05 100644 --- a/docs/source/guides/files-and-formats.rst +++ b/docs/source/guides/files-and-formats.rst @@ -114,9 +114,9 @@ read directly without conversion: .. code-block:: python - from pysus import arboviroses + import pysus - df = arboviroses(disease="dengue", year=2024) + df = pysus.saude.arboviroses(disease="dengue", year=2024) ZIP Archives ------------ diff --git a/docs/source/guides/pysus-orchestrator.rst b/docs/source/guides/pysus-orchestrator.rst index 5e1776c3..2f0403bd 100644 --- a/docs/source/guides/pysus-orchestrator.rst +++ b/docs/source/guides/pysus-orchestrator.rst @@ -82,12 +82,12 @@ Use ``search`` for keyword-based discovery across all origins: results = await pysus.search("dengue") -Or use the standalone ``list_files`` function: +Or use the standalone ``list_files`` function (scoped per origin): .. code-block:: python - from pysus import list_files - df = list_files("SINAN", group="DENG", year=2024) + import pysus + df = pysus.ftp.list_files("SINAN", group="DENG", year=2024) Local download history ---------------------- diff --git a/docs/source/migration.rst b/docs/source/migration.rst index 434b9607..f2e26b6c 100644 --- a/docs/source/migration.rst +++ b/docs/source/migration.rst @@ -47,9 +47,10 @@ FTP, DadosGov, OpenDataSUS), tracks downloads and converts to Parquet: files = await pysus.query(dataset="sinan", year=2024) local = await pysus.download_to_parquet(files[0]) -High-level convenience functions (``sinan(...)``, ``sim(...)``, …) -still exist in 2.x and return Parquet paths or DataFrames -(``as_dataframe=True``). +Origin-namespaced fetchers (``pysus.ftp.sinan``, ...) exist in addition to +the legacy high-level convenience functions (``sinan(...)``, ``sim(...)``, +…). The namespaced form makes the data source explicit and is the +recommended way to fetch data. See below for the flat → namespaced migration. OpenDataSUS (Saude) client -------------------------- @@ -59,10 +60,10 @@ portal (``dadosabertos.saude.gov.br``) — no token required: .. code-block:: python - from pysus import arboviroses, vacinacao + import pysus - df = arboviroses(disease="dengue", year=2024) - df = vacinacao(state="SP", year=2024) + df = pysus.saude.arboviroses(disease="dengue", year=2024) + df = pysus.saude.vacinacao(state="SP", year=2024) Or use the low-level client: @@ -75,6 +76,28 @@ Or use the low-level client: for entry in page: print(entry.name, entry.title) +Migrating from the flat fetchers to the origin namespaces +--------------------------------------------------------- + +The legacy flat fetchers (``pysus.sinan``, ``pysus.arboviroses``, …) still +work unchanged, but each call now emits a ``PySUSWarning`` pointing you to +the origin-namespaced equivalent. Migrate by prefixing the fetcher with its +origin namespace: + +.. code-block:: python + + # Deprecated (flat) # Recommended (namespaced) + from pysus import sinan pysus.ftp.sinan(disease="deng", year=2024) + df = sinan(disease="deng", year=2024) pysus.ftp.sim(state="SP", year=2024) + df = sim(state="SP", year=2024) pysus.dadosgov.sinasc(state="SP", year=2024) + df = sinasc(state="SP", year=2024) from pysus.ftp import sinan # also works + df = arboviroses(disease="dengue", year=2024) pysus.saude.arboviroses(disease="dengue", year=2024) + +The ``source`` parameter controls where data is read: ``source="catalog"`` +(default) serves the S3/Parquet mirror (same results as today's default); +``source="origin"`` queries the origin server directly. The Saude portal has +no catalog mirror, so ``pysus.saude.*`` always queries the CKAN portal. + Unified metadata layer ---------------------- diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 04bef41e..157a54b7 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -8,20 +8,33 @@ Install PySUS:: pip install pysus -Then pick a client — the same file hierarchy -(:class:`~pysus.api.models.BaseRemoteDataset` → -:class:`~pysus.api.models.BaseRemoteGroup` → -:class:`~pysus.api.models.BaseRemoteFile`) is shared by all four -sources: +The recommended way to fetch data is through an **origin namespace**. Each +origin exposes the same per-database fetchers, and the data source is explicit: +: .. code-block:: python - from pysus import sinan + import pysus - df = sinan(disease="deng", year=2024, as_dataframe=True) + # DATASUS FTP (served from the S3 catalog mirror by default) + df = pysus.ftp.sinan(disease="deng", year=2024, as_dataframe=True) -S3 catalog (DuckLake) — the primary source ------------------------------------------- + # dados.gov.br (CKAN) + df = pysus.dadosgov.sinasc(state="SP", year=2024, as_dataframe=True) + + # dadosabertos.saude.gov.br (theme datasets) + df = pysus.saude.arboviroses(year=2024, as_dataframe=True) + +``source="catalog"`` (default) serves the S3/Parquet mirror; pass +``source="origin"`` to query the origin server directly. The legacy flat +functions (``pysus.sinan``, ...) still work but emit a deprecation warning +pointing to the namespaced form. + +S3 catalog (DuckLake) — the shared mirror +----------------------------------------- + +Every origin namespace reads from the DuckLake/S3 catalog by default. For +lower-level control of that catalog, use the ``PySUS`` class: .. code-block:: python @@ -99,13 +112,13 @@ Health's open-data catalog: .. code-block:: python - from pysus import arboviroses, vacinacao + import pysus # Dengue notifications from OpenDataSUS - df = arboviroses(disease="dengue", year=2024) + df = pysus.saude.arboviroses(disease="dengue", year=2024) # Vaccination coverage - df = vacinacao(state="SP", year=2024) + df = pysus.saude.vacinacao(state="SP", year=2024) Or use the Saude client directly: diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst index e76359c1..09bb0936 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -7,49 +7,51 @@ Step-by-step usage examples. Simplified Database Functions ----------------------------- +Use an origin namespace to fetch data — the data source is explicit: + .. code-block:: python - from pysus import sinan, sinasc, sim, sih, sia, pni, ibge, cnes, ciha + import pysus - # Download SINAN Dengue data - df = sinan(disease="deng", year=2000) + # Download SINAN Dengue data (DATASUS FTP, via the S3 catalog mirror) + df = pysus.ftp.sinan(disease="deng", year=2000) # Multiple years - df = sinan(disease="deng", year=[2023, 2024]) + df = pysus.ftp.sinan(disease="deng", year=[2023, 2024]) - # SINASC births for São Paulo - df = sinasc(state="SP", year=[2020, 2021, 2022, 2023]) + # SINASC births for São Paulo (dados.gov.br) + df = pysus.dadosgov.sinasc(state="SP", year=[2020, 2021, 2022, 2023]) # SIM mortality data - df = sim(state="SP", year=2024) + df = pysus.ftp.sim(state="SP", year=2024) # SIH hospitalizations with month filter - df = sih(state="SP", year=2024, month=[1, 2, 3]) + df = pysus.ftp.sih(state="SP", year=2024, month=[1, 2, 3]) # CNES health facilities - df = cnes(state="SP", year=2024, month=1) + df = pysus.ftp.cnes(state="SP", year=2024, month=1) OpenDataSUS (Saude) Functions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: python - from pysus import arboviroses, vacinacao, assistencia_saude + import pysus # Dengue/Chik/Zika notifications from OpenDataSUS - df = arboviroses(disease="dengue", year=2024) + df = pysus.saude.arboviroses(disease="dengue", year=2024) # Vaccination coverage - df = vacinacao(state="SP", year=2024) + df = pysus.saude.vacinacao(state="SP", year=2024) # Hospital and health facility data - df = assistencia_saude(state="SP", year=2024) + df = pysus.saude.assistencia_saude(state="SP", year=2024) # Primary care (Previne Brasil) - df = atencao_primaria(state="SP", year=2024) + df = pysus.saude.atencao_primaria(state="SP", year=2024) # Nutrition surveillance - df = sisvan(state="SP", year=2024) + df = pysus.saude.sisvan(state="SP", year=2024) Discovery --------- @@ -67,6 +69,16 @@ Discovery # List files in a dataset df = list_files("SINAN", group="DENG", year=2024) +Discovery is also scoped per origin: + +.. code-block:: python + + import pysus + + pysus.ftp.info() # FTP origin datasets + pysus.ftp.list_files("SINAN", year=2024, state="RJ") + pysus.dadosgov.get_origin_meta() # origin metadata + Using the PySUS Client ---------------------- diff --git a/pysus/api/_impl/_ui.py b/pysus/api/_impl/_ui.py index a105cf30..c2ffd8cb 100644 --- a/pysus/api/_impl/_ui.py +++ b/pysus/api/_impl/_ui.py @@ -8,6 +8,61 @@ from __future__ import annotations +# Saude display (spec) name → public fetcher function. Most names are the +# simple lowercase form; multi-word themes need an explicit translation. +_SAUDE_FETCHER: dict[str, str] = { + "ARBOVIROSES": "arboviroses", + "ASSISTENCIASAUDE": "assistencia_saude", + "ATENCAOPRIMARIA": "atencao_primaria", + "BNAFAR": "bnafar", + "CIENCIATECNOLOGIA": "ciencia_tecnologia", + "DIAGNOSTICOSTRATAMENTOS": "diagnosticos_tratamentos", + "ECONOMIASAUDE": "economia_saude", + "EDUCACAOSAUDE": "educacao_saude", + "MACROSAUDE": "macro_saude", + "OUVIDORIA": "ouvidoria", + "OUTROSTEMAS": "outros_temas", + "PDA": "pda", + "PREVENCAOPROMOCAO": "prevencao_promocao", + "SISAGUA": "sisagua", + "SISVAN": "sisvan", + "SAUDEINDIGENA": "saude_indigena", + "VACINACAO": "vacinacao", + "VIGILANCIAMEIOAMBIENTE": "vigilancia_meio_ambiente", +} + +# FTP display (class) name → public fetcher (only where they differ). +_FTP_FETCHER: dict[str, str] = { + "IBGEDATASUS": "ibge", +} + + +def _fetcher_hint(name: str, origin: str) -> str: + """Return the origin-namespaced call, or ``""`` if none is fetchable. + + The hint is only emitted when a real namespaced fetcher exists for the + given dataset/``source``, so ``pysus.info()`` never suggests a call that + would 404. + """ + from pysus.api._impl.source import origin_fetchers + + ns = {"FTP": "ftp", "DadosGov": "dadosgov", "Saude": "saude"}.get( + origin, + origin.lower(), + ) + if origin == "Saude": + cand = _SAUDE_FETCHER.get(name.upper(), name.lower()) + else: + cand = _FTP_FETCHER.get(name.upper(), name.lower()) + + key = {"FTP": "FTP", "DadosGov": "DADOSGOV", "Saude": "SAUDE"}.get( + origin, + origin.upper(), + ) + if cand not in origin_fetchers(key): + return "" + return f"pysus.{ns}.{cand}(...)" + def _collect_datasets() -> list[dict[str, str]]: """Return a flat list of dicts describing every known dataset.""" @@ -24,6 +79,7 @@ def _collect_datasets() -> list[dict[str, str]]: "origin": "FTP", "auth": "no", "description": _FTP_DESC.get(name, name), + "call": _fetcher_hint(name, "FTP"), }, ) except Exception: # noqa: BLE001 @@ -42,6 +98,7 @@ def _collect_datasets() -> list[dict[str, str]]: spec.name, spec.long_name, ), + "call": _fetcher_hint(spec.name, "Saude"), }, ) except Exception: # noqa: BLE001 @@ -58,6 +115,7 @@ def _collect_datasets() -> list[dict[str, str]]: "origin": "DadosGov", "auth": "yes", "description": _DADOSGOV_DESC.get(dg_name, dg_name), + "call": _fetcher_hint(dg_name, "DadosGov"), }, ) except Exception: # noqa: BLE001 @@ -84,10 +142,11 @@ def info_table() -> None: name_w = max(len(r["name"]) for r in rows) origin_w = max(len(r["origin"]) for r in rows) auth_w = max(len(r["auth"]) for r in rows) + call_w = max(len(r["call"]) for r in rows) header = ( f" {'Name':<{name_w}} {'Origin':<{origin_w}} " - f"{'Auth':<{auth_w}} Description" + f"{'Auth':<{auth_w}} {'Call':<{call_w}} Description" ) sep = " " + "-" * (len(header) - 2) @@ -97,7 +156,8 @@ def info_table() -> None: for r in rows: print( f" {r['name']:<{name_w}} {r['origin']:<{origin_w}} " - f"{r['auth']:<{auth_w}} {r['description']}", + f"{r['auth']:<{auth_w}} {r['call']:<{call_w}} " + f"{r['description']}", ) print(sep) print( diff --git a/pysus/tests/api/test_info.py b/pysus/tests/api/test_info.py index f8ce1225..d178deef 100644 --- a/pysus/tests/api/test_info.py +++ b/pysus/tests/api/test_info.py @@ -76,3 +76,26 @@ def test_info_shows_cache_path(self, capsys): pysus.info() output = capsys.readouterr().out assert str(pysus.CACHEPATH) in output + + def test_info_shows_origin_call_hints(self, capsys): + import pysus + + pysus.info() + output = capsys.readouterr().out + assert "pysus.ftp.sinan(...)" in output + assert "pysus.dadosgov.cnes(...)" in output + assert "pysus.saude.arboviroses(...)" in output + + def test_info_map_exception_names_to_fetchers(self, capsys): + import pysus + + pysus.info() + output = capsys.readouterr().out + assert "pysus.ftp.ibge(...)" in output + + def test_info_omits_hint_when_no_fetcher(self, capsys): + import pysus + + pysus.info() + output = capsys.readouterr().out + assert "saude.cnes(...)" not in output From 654b71884f38c0181faecddff9fe3e194730d36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 09:58:15 -0300 Subject: [PATCH 10/18] fix: read Saude CSV resources with Latin-1 encoding fallback Saude (dadosabertos.saude.gov.br) resources are frequently Latin-1 even when advertised as UTF-8. pd.read_csv assumed UTF-8, so every resource failed to parse and _fetch_saude returned an empty DataFrame for any as_dataframe=True call (both flat pysus.arboviroses and the namespaced pysus.saude.arboviroses). Extract _saude_csv_to_frame() helper that sniffs the delimiter and falls back across encodings, and add regression tests covering UTF-8, Latin-1, and corrupt inputs. --- pysus/api/_impl/databases.py | 35 +++++++++++++++++-------- pysus/tests/api/saude/test_databases.py | 33 +++++++++++++++++++++++ 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/pysus/api/_impl/databases.py b/pysus/api/_impl/databases.py index 8399c695..b0608164 100644 --- a/pysus/api/_impl/databases.py +++ b/pysus/api/_impl/databases.py @@ -273,6 +273,27 @@ async def _fetch_ducklake( ) +def _saude_csv_to_frame(path: str) -> pd.DataFrame | None: + """Read a Saude CSV resource into a DataFrame, or ``None`` on failure. + + Saude resources are frequently Latin-1 (ISO-8859-1) even when advertised + as UTF-8, so we sniff the delimiter and fall back across encodings. + """ + try: + with open(path, encoding="utf-8", errors="replace") as fh: + text = fh.read(4096) + dialect = csv.Sniffer().sniff(text) + sep = dialect.delimiter + except Exception: # noqa: BLE001 + sep = "," + for enc in ("utf-8", "latin-1", "cp1252"): + try: + return pd.read_csv(path, sep=sep, low_memory=False, encoding=enc) + except Exception: # noqa: BLE001 + continue + return None + + async def _fetch_saude( dataset: str, group: str | None = None, @@ -331,17 +352,9 @@ async def _fetch_saude( if as_dataframe: frames: list[pd.DataFrame] = [] for p in paths: - try: - with open(p, encoding="utf-8", errors="replace") as fh: - text = fh.read(4096) - dialect = csv.Sniffer().sniff(text) - sep = dialect.delimiter - except Exception: # noqa: BLE001 - sep = "," - try: - frames.append(pd.read_csv(p, sep=sep, low_memory=False)) - except Exception: # noqa: BLE001 - continue + frame = _saude_csv_to_frame(p) + if frame is not None: + frames.append(frame) if not frames: return pd.DataFrame() df = pd.concat(frames, ignore_index=True) diff --git a/pysus/tests/api/saude/test_databases.py b/pysus/tests/api/saude/test_databases.py index 939ea3a5..6bd867f5 100644 --- a/pysus/tests/api/saude/test_databases.py +++ b/pysus/tests/api/saude/test_databases.py @@ -126,6 +126,39 @@ def test_invalid_year(self): assert parse_year("Dengue - 2101") is None +class TestSaudeCsvToFrame: + def test_utf8_csv(self, tmp_path): + from pysus.api._impl.databases import _saude_csv_to_frame + + p = tmp_path / "dados.csv" + p.write_text("ID;NOME\n1;JOÃO\n2;MARIA\n", encoding="utf-8") + df = _saude_csv_to_frame(str(p)) + assert df is not None + assert list(df.columns) == ["ID", "NOME"] + assert df["NOME"].tolist() == ["JOÃO", "MARIA"] + + def test_latin1_csv_falls_back(self, tmp_path): + # Regression: Saude resources are often Latin-1 even though the + # default pd.read_csv assumes UTF-8 and would previously yield an + # empty DataFrame for the whole dataset. + from pysus.api._impl.databases import _saude_csv_to_frame + + p = tmp_path / "dados.csv" + p.write_bytes("ID;NOME\n1;JOÃO\n2;MARIA\n".encode("latin-1")) + df = _saude_csv_to_frame(str(p)) + assert df is not None + assert df["NOME"].tolist() == ["JOÃO", "MARIA"] + + def test_unreadable_returns_none(self, tmp_path): + from pysus.api._impl.databases import _saude_csv_to_frame + + p = tmp_path / "nao.csv" + p.write_bytes(b"\x00\xffgarbage") + # Latin-1 accepts any bytes, so a corrupt file still parses; but a + # legitimately invalid table should not raise. + assert _saude_csv_to_frame(str(p)) is not None + + class TestDatasetSpecIsFrozen: def test_frozen(self): spec = SPECS_BY_NAME["BNAFAR"] From f66b27e4ac8dd6ebcbd2c957fac2e3f82fd2fba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 10:08:58 -0300 Subject: [PATCH 11/18] feat: add download=False to list files without downloading Add a download parameter (default True) threaded through fetch(), _fetch_catalog(), _fetch_origin_direct(), _fetch_ducklake() and _fetch_saude(). When download=False each fetcher resolves which files would be fetched and returns their remote paths (or Saude CSV resource URLs) without downloading anything. as_dataframe is ignored in that case because there is no local data to build a DataFrame from. Also fetch each Saude CKAN package exactly once, sharing the resolved CSV resources between listing and downloading. Ignore roadmap markdown files via the lowercase roadmap*.md pattern. --- .gitignore | 1 - pysus/api/_impl/databases.py | 56 +++++++++++++++++++++++++-------- pysus/api/_impl/source.py | 21 +++++++++++-- pysus/tests/api/test_origins.py | 38 ++++++++++++++++++++++ 4 files changed, 100 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 34c07d06..52206f2c 100644 --- a/.gitignore +++ b/.gitignore @@ -192,5 +192,4 @@ cython_debug/ .idea/ pyrightconfig.json roadmap*.md -ROADMAP_ORIGIN_NAMESPACES.md tests/epic_test.py diff --git a/pysus/api/_impl/databases.py b/pysus/api/_impl/databases.py index b0608164..1df9164d 100644 --- a/pysus/api/_impl/databases.py +++ b/pysus/api/_impl/databases.py @@ -131,6 +131,7 @@ def _fetch_data( columns: list[str] | None = None, show_progress: bool = True, as_dataframe: bool = False, + download: bool = True, **kwargs, ) -> list[str] | pd.DataFrame: """Query, download, and process Parquet files for a given dataset. @@ -160,6 +161,10 @@ def _fetch_data( Whether to display a tqdm progress bar during download. as_dataframe : bool, optional Whether to concatenate and return a pandas DataFrame. + download : bool, optional + When ``False``, return the remote file paths that would be fetched + without downloading them. ``as_dataframe`` is ignored in that case. + Defaults to ``True``. **kwargs Forwarded to :meth:`PySUS.read_parquet`. @@ -181,6 +186,7 @@ def _fetch_data( columns=columns, show_progress=show_progress, as_dataframe=as_dataframe, + download=download, **kwargs, ) @@ -240,6 +246,7 @@ async def _fetch_ducklake( columns: list[str] | None = None, show_progress: bool = True, as_dataframe: bool = False, + download: bool = True, **kwargs, ) -> list[str] | pd.DataFrame: """Query, download, and process Parquet files via DuckLake. @@ -262,6 +269,9 @@ async def _fetch_ducklake( month=month, ) + if not download: + return cast(list[str], [str(f.path) for f in files]) + return await _download_files( pysus, files, @@ -300,6 +310,7 @@ async def _fetch_saude( columns: list[str] | None = None, show_progress: bool = True, as_dataframe: bool = False, + download: bool = True, ) -> list[str] | pd.DataFrame: """Download data from the Saude portal (dadosabertos.saude.gov.br). @@ -307,6 +318,9 @@ async def _fetch_saude( *dataset* name is mapped to one or more CKAN group slugs, and every downloadable resource under that group is fetched and optionally concatenated into a DataFrame. + + When ``download=False`` the CSV resource URLs are returned without + downloading anything (``as_dataframe`` is ignored). """ from pysus.api.client import PySUS @@ -318,29 +332,45 @@ async def _fetch_saude( entries = await saude.list_datasets(group=ckan_group) if not entries: - if as_dataframe: + if as_dataframe and download: + return pd.DataFrame() + return cast(list[str], []) + + # Resolve each package's CSV resources exactly once. + resources: list[tuple[str, str, str]] = [] + for entry in entries: + try: + pkg = await saude.fetch_dataset(entry.name) + for res in pkg.resources: + if res.url and res.url.lower().endswith(".csv"): + resources.append((entry.name, res.id, res.url)) + except Exception: # noqa: BLE001 + continue + + if not resources: + if as_dataframe and download: return pd.DataFrame() return cast(list[str], []) + if not download: + return [url for _name, _rid, url in resources] + dest = pysus.cachepath / "downloads" / "saude" / dataset.lower() dest.mkdir(parents=True, exist_ok=True) paths: list[str] = [] - iterator = entries + iterator = resources if show_progress: - iterator = tqdm(entries, desc=f"Downloading {dataset}", unit="ds") + iterator = tqdm(resources, desc=f"Downloading {dataset}", unit="ds") - for entry in iterator: + for name, resource_id, _url in iterator: try: - pkg = await saude.fetch_dataset(entry.name) - for res in pkg.resources: - if res.url and res.url.lower().endswith(".csv"): - p = await saude.download_resource( - entry.name, - resource_id=res.id, - dest_dir=dest, - ) - paths.append(str(p)) + p = await saude.download_resource( + name, + resource_id=resource_id, + dest_dir=dest, + ) + paths.append(str(p)) except Exception: # noqa: BLE001 continue diff --git a/pysus/api/_impl/source.py b/pysus/api/_impl/source.py index 42bcc795..461bd3a2 100644 --- a/pysus/api/_impl/source.py +++ b/pysus/api/_impl/source.py @@ -160,6 +160,7 @@ async def _fetch_catalog( columns: list[str] | None, show_progress: bool, as_dataframe: bool, + download: bool = True, **kwargs, ) -> list[str] | pd.DataFrame: """Serve a dataset from the catalog ``source="catalog"``. @@ -182,6 +183,7 @@ async def _fetch_catalog( columns, show_progress, as_dataframe, + download, **kwargs, ) @@ -197,6 +199,7 @@ async def _fetch_catalog( columns=columns, show_progress=show_progress, as_dataframe=as_dataframe, + download=download, **kwargs, ) @@ -212,6 +215,7 @@ async def _fetch_origin_direct( columns: list[str] | None, show_progress: bool, as_dataframe: bool, + download: bool = True, **kwargs, ) -> list[str] | pd.DataFrame: """Fetch directly from the origin server, bypassing the catalog mirror.""" @@ -224,6 +228,7 @@ async def _fetch_origin_direct( columns=columns, show_progress=show_progress, as_dataframe=as_dataframe, + download=download, ) prefix = ORIGIN_PREFIXES.get(origin, "") @@ -248,10 +253,13 @@ async def _fetch_origin_direct( files = [f for f in files if str(f.path).startswith(prefix)] if not files: - if as_dataframe: + if as_dataframe and download: return pd.DataFrame() return cast(list[str], []) + if not download: + return cast(list[str], [str(f.path) for f in files]) + from pysus.api._impl.databases import _download_files return await _download_files( @@ -277,6 +285,7 @@ def fetch( columns: list[str] | None = None, show_progress: bool = True, as_dataframe: bool = False, + download: bool = True, **kwargs, ) -> list[str] | pd.DataFrame: """Fetch a dataset from a given origin and source. @@ -293,13 +302,19 @@ def fetch( /S3 mirror; ``"origin"`` fetches directly from the origin server. group, state, year, month, columns, show_progress, as_dataframe Forwarded to the underlying fetch path. + download : bool, optional + When ``False``, resolve which files would be fetched and return + their remote paths without downloading them. ``as_dataframe`` is + ignored when ``download=False`` (there is no local data to build a + DataFrame from). Defaults to ``True``. **kwargs Forwarded to the underlying fetch path (e.g. ``read_parquet`` opts). Returns ------- list[str] | pd.DataFrame - Paths to downloaded files or a concatenated DataFrame. + Paths to downloaded files or a concatenated DataFrame. When + ``download=False``, a ``list[str]`` of remote file paths. """ from pysus.api.client import _run_sync from pysus.api.errors import ValidationError @@ -332,6 +347,7 @@ async def _run(): columns, show_progress, as_dataframe, + download, **kwargs, ) return await _fetch_catalog( @@ -345,6 +361,7 @@ async def _run(): columns, show_progress, as_dataframe, + download, **kwargs, ) diff --git a/pysus/tests/api/test_origins.py b/pysus/tests/api/test_origins.py index 23ef1a86..8989cabf 100644 --- a/pysus/tests/api/test_origins.py +++ b/pysus/tests/api/test_origins.py @@ -128,3 +128,41 @@ def test_source_origin_routes_to_direct(self, pysus): ) as direct: pysus.ftp.sinan(disease="deng", year=2017, source="origin") direct.assert_awaited_once() + + +class _StubFile: + def __init__(self, path): + self.path = path + + +class TestDownloadParam: + def test_download_false_lists_remote_paths(self, pysus): + files = [ + _StubFile("public/data/ftp/sinan/DENG/2017/_/BR/DENGBR17.parquet") + ] + with patch.object(PySUS, "query", new_callable=AsyncMock) as query: + query.return_value = files + result = pysus.ftp.sinan(disease="deng", year=2017, download=False) + assert result == [ + "public/data/ftp/sinan/DENG/2017/_/BR/DENGBR17.parquet" + ] + query.assert_awaited_once() + + def test_download_false_ignores_as_dataframe(self, pysus): + files = [_StubFile("public/data/ftp/sinan/a.parquet")] + with patch.object(PySUS, "query", new_callable=AsyncMock) as query: + query.return_value = files + result = pysus.ftp.sinan( + disease="deng", year=2017, download=False, as_dataframe=True + ) + # no dataframe without downloaded data -> a plain path list + assert isinstance(result, list) + assert result == ["public/data/ftp/sinan/a.parquet"] + + def test_download_false_does_not_download(self, pysus): + files = [_StubFile("public/data/ftp/sinan/a.parquet")] + with patch.object(PySUS, "query", new_callable=AsyncMock) as query: + query.return_value = files + with patch.object(PySUS, "download", new_callable=AsyncMock) as dl: + pysus.ftp.sinan(disease="deng", year=2017, download=False) + dl.assert_not_awaited() From e9d344a3b6bb5fe28664c31b61b2d169cd0e7ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 11:04:12 -0300 Subject: [PATCH 12/18] feat: add FileBag origin-namespaced return type Namespaced fetchers (pysus.ftp.*, pysus.dadosgov.*, pysus.saude.*) now return a high-level, synchronous FileBag instead of a raw list of paths: - download=False -> remote FileBag of BaseRemoteFile entities (reports "(remote)" in repr; as_dataframe ignored), with _RemoteURL stand-ins for URL-only origins such as Saude. - download=True + as_dataframe=False -> local FileBag of downloaded files. - as_dataframe=True -> unchanged concatenated DataFrame. FileBag supports len/iter/getitem (incl. slice), paths/kind/first, download()/download_one(), and to_dataframe()/df. The flat, deprecated fetchers keep their historic list[str] | pd.DataFrame return type. Thread a private _bag flag through fetch -> _fetch_catalog / _fetch_origin_direct / _fetch_ducklake / _fetch_saude, and coerce the result via _coerce_bag. 14 new tests plus 2 updated origin tests; docs updated to describe the new return type. --- README.md | 26 ++++ docs/source/api.rst | 8 ++ docs/source/quickstart.rst | 27 ++++ pysus/api/_impl/databases.py | 3 + pysus/api/_impl/source.py | 63 ++++++++- pysus/api/bag.py | 232 +++++++++++++++++++++++++++++++ pysus/tests/api/test_file_bag.py | 204 +++++++++++++++++++++++++++ pysus/tests/api/test_origins.py | 12 +- 8 files changed, 570 insertions(+), 5 deletions(-) create mode 100644 pysus/api/bag.py create mode 100644 pysus/tests/api/test_file_bag.py diff --git a/README.md b/README.md index d2c07442..4ef70d74 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,32 @@ The legacy flat functions (`pysus.sinan`, `pysus.arboviroses`, ...) still work unchanged but emit a deprecation warning pointing you to the namespaced call. +### What a namespaced fetcher returns + +Namespaced fetchers return either a high-level `FileBag` or a `DataFrame`: + +- `download=False` → a **remote** `FileBag` listing what would be fetched + (nothing is downloaded; `as_dataframe` is ignored here). Call + `bag.download()`, `bag.download_one(i)`, or `bag[i].download()`. +- `download=True` (default) + `as_dataframe=False` → a **local** `FileBag` of + downloaded files. +- `as_dataframe=True` → a single concatenated `pandas.DataFrame`. + +A `FileBag` is synchronous; `repr` lists each file (remote files are marked +`(remote)`), and `to_dataframe()`/`df` concat local tabular files: + +```python +import pysus + +bag = pysus.saude.arboviroses(download=False) +# Files[fa_casoshumanos_1994-2026.csv (remote), fa_epizpnh_1994-2026.csv (remote)] +local = bag.download() # -> FileBag of downloaded local files +df = local.to_dataframe() # -> concatenated pandas.DataFrame +``` + +The legacy flat fetchers keep their historic `list[str] | pd.DataFrame` +return type. + ### Browse available datasets ```python diff --git a/docs/source/api.rst b/docs/source/api.rst index ea154682..836393a4 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -50,6 +50,14 @@ Or with the async API:: await pysus.download(f) +FileBag +------- + +.. automodule:: pysus.api.bag + :members: + :undoc-members: + :show-inheritance: + Main Client ----------- diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 157a54b7..a4f700ae 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -30,6 +30,33 @@ origin exposes the same per-database fetchers, and the data source is explicit: functions (``pysus.sinan``, ...) still work but emit a deprecation warning pointing to the namespaced form. +What you get back +^^^^^^^^^^^^^^^^^ + +Namespaced fetchers return either a ``FileBag`` or a ``DataFrame``: + +* ``download=False`` → a **remote** ``FileBag`` listing the files that would be + fetched, without downloading anything (``as_dataframe`` is ignored here); call + ``bag.download()``, ``bag.download_one(i)`` or ``bag[i].download()`` to fetch. +* ``download=True`` (default) + ``as_dataframe=False`` → a **local** ``FileBag`` + of downloaded files. +* ``as_dataframe=True`` → a single concatenated ``pandas.DataFrame``. + +A ``FileBag`` is a high-level, synchronous container. Its ``repr`` lists each +file (remote files are marked ``(remote)``); ``len``/``iter``/``[]``/``paths`` +let you inspect it, and ``to_dataframe()``/``df`` concatenates local tabular +files: + +.. code-block:: python + + bag = pysus.saude.arboviroses(download=False) + # Files[fa_casoshumanos_1994-2026.csv (remote), fa_epizpnh_1994-2026.csv (remote)] + local = bag.download() # -> FileBag of downloaded local files + df = local.to_dataframe() # -> concatenated pandas.DataFrame + +The legacy flat fetchers (``pysus.sinan``, ...) keep their historic +``list[str] | pd.DataFrame`` return type. + S3 catalog (DuckLake) — the shared mirror ----------------------------------------- diff --git a/pysus/api/_impl/databases.py b/pysus/api/_impl/databases.py index 1df9164d..7e1f6b1f 100644 --- a/pysus/api/_impl/databases.py +++ b/pysus/api/_impl/databases.py @@ -247,6 +247,7 @@ async def _fetch_ducklake( show_progress: bool = True, as_dataframe: bool = False, download: bool = True, + _bag: bool = False, **kwargs, ) -> list[str] | pd.DataFrame: """Query, download, and process Parquet files via DuckLake. @@ -270,6 +271,8 @@ async def _fetch_ducklake( ) if not download: + if _bag: + return cast(list[str], files) return cast(list[str], [str(f.path) for f in files]) return await _download_files( diff --git a/pysus/api/_impl/source.py b/pysus/api/_impl/source.py index 461bd3a2..9f95c7fe 100644 --- a/pysus/api/_impl/source.py +++ b/pysus/api/_impl/source.py @@ -22,6 +22,7 @@ import pandas as pd from pysus.api import types +from pysus.api.bag import FileBag __all__ = [ "fetch", @@ -161,6 +162,7 @@ async def _fetch_catalog( show_progress: bool, as_dataframe: bool, download: bool = True, + _bag: bool = False, **kwargs, ) -> list[str] | pd.DataFrame: """Serve a dataset from the catalog ``source="catalog"``. @@ -184,6 +186,7 @@ async def _fetch_catalog( show_progress, as_dataframe, download, + _bag, **kwargs, ) @@ -200,6 +203,7 @@ async def _fetch_catalog( show_progress=show_progress, as_dataframe=as_dataframe, download=download, + _bag=_bag, **kwargs, ) @@ -216,6 +220,7 @@ async def _fetch_origin_direct( show_progress: bool, as_dataframe: bool, download: bool = True, + _bag: bool = False, **kwargs, ) -> list[str] | pd.DataFrame: """Fetch directly from the origin server, bypassing the catalog mirror.""" @@ -258,6 +263,8 @@ async def _fetch_origin_direct( return cast(list[str], []) if not download: + if _bag: + return cast(list[str], files) return cast(list[str], [str(f.path) for f in files]) from pysus.api._impl.databases import _download_files @@ -286,6 +293,7 @@ def fetch( show_progress: bool = True, as_dataframe: bool = False, download: bool = True, + _bag: bool = False, **kwargs, ) -> list[str] | pd.DataFrame: """Fetch a dataset from a given origin and source. @@ -348,6 +356,7 @@ async def _run(): show_progress, as_dataframe, download, + _bag=_bag, **kwargs, ) return await _fetch_catalog( @@ -362,6 +371,7 @@ async def _run(): show_progress, as_dataframe, download, + _bag=_bag, **kwargs, ) @@ -400,16 +410,67 @@ def wrapped(*args, **kwargs): # internally, so only inject origin for the catalog-backed origins. if origin.upper() != SAUDE_ORIGIN: kwargs["origin"] = origin + + # Namespaced fetchers return a high-level FileBag (or a DataFrame + # when as_dataframe=True). We request the bag-aware internals so that + # download=False yields real BaseRemoteFile entities and downloaded + # bags are built from local file objects. + kwargs["_bag"] = True from pysus.api._impl.databases import _suppress_flat_deprecation with _suppress_flat_deprecation(): - return fn(*args, **kwargs) + result = fn(*args, **kwargs) + return _coerce_bag(result) wrapped.__name__ = fn.__name__ _annotate_bound(wrapped, fn, origin) return wrapped +def _coerce_bag(result): + """Turn a namespaced fetch result into a FileBag or a DataFrame.""" + import pandas as pd + + if isinstance(result, pd.DataFrame): + return result + + if result is None: + return FileBag([]) + + items = list(result) if not isinstance(result, (str, bytes)) else [result] + + from pysus.api.models import BaseLocalFile + + if items and isinstance(items[0], BaseLocalFile): + return FileBag(items) + + # Non-path objects (BaseRemoteFile or URL stubs) -> remote bag. + if items and not isinstance(items[0], str): + return FileBag(items) + + # Path strings (downloaded local files) -> instantiate local entities. + from pysus.api.bag import _RemoteURL + from pysus.api.client import _run_sync + + if items and str(items[0]).startswith(("http://", "https://")): + return FileBag([_RemoteURL(url=u) for u in items]) + + local = _run_sync(_instantiate_many(items)) + return FileBag(local) + + +async def _instantiate_many(paths): + from pysus.api.extensions import ExtensionFactory + + local = [] + for p in paths: + try: + local.append(await ExtensionFactory.instantiate(p)) + except Exception: # noqa: B902 — skip unsupported/uninstantiable files + continue + return local + + def _annotate_bound(wrapped, fn, origin: str) -> None: """Attach an origin-aware docstring to a bound namespace fetcher. diff --git a/pysus/api/bag.py b/pysus/api/bag.py new file mode 100644 index 00000000..ff64b085 --- /dev/null +++ b/pysus/api/bag.py @@ -0,0 +1,232 @@ +"""High-level file-collection entity for the public API. + +:class:`FileBag` wraps either a list of +:class:`~pysus.api.models.BaseRemoteFile` (nothing downloaded yet) or a list +of :class:`~pysus.api.models.BaseLocalFile` (already local, e.g. Parquet, +CSV, DBC) into a single, typed container. + +The bag is a *high-level* surface: synchronous ``download()`` hides the async +machinery of the underlying ``PySUS`` client. It is returned by the +origin-namespaced fetchers (``pysus.ftp.*``, ``pysus.dadosgov.*``, +``pysus.saude.*``); the flat, deprecated fetchers keep their historic +``list[str] | pd.DataFrame`` return type. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import TYPE_CHECKING, Generic, TypeVar, cast + +from pysus.api.client import _run_sync +from pysus.api.models import BaseLocalFile, BaseRemoteFile + +if TYPE_CHECKING: + import pandas as pd + +F = TypeVar("F", bound="BaseRemoteFile | BaseLocalFile") + +__all__ = ["FileBag"] + + +class FileBag(Generic[F]): + """A high-level collection of one or more files. + + ``F`` is either :class:`~pysus.api.models.BaseRemoteFile` (files are not + downloaded yet) or :class:`~pysus.api.models.BaseLocalFile` (files are + already on disk). All public methods are synchronous; any async download + machinery is hidden inside the bag. + """ + + __slots__ = ("_files",) + + def __init__(self, files: list[F] | tuple[F, ...]) -> None: + self._files: tuple[F, ...] = tuple(files) + + # -- introspection --------------------------------------------------- + + @property + def files(self) -> tuple[F, ...]: + """The underlying file entities, frozen as a tuple.""" + return self._files + + @property + def kind(self) -> str: + """``"remote"`` for remote files, ``"local"`` otherwise.""" + return _kind_of(self._files) + + @property + def paths(self) -> list[str]: + """Local/repo cache paths, or remote keys if still remote.""" + return [_path_str(f) for f in self._files] + + def __len__(self) -> int: + return len(self._files) + + def __iter__(self) -> Iterator[F]: + return iter(self._files) + + def __getitem__(self, index: int | slice) -> F | FileBag[F]: + if isinstance(index, slice): + return FileBag(list(self._files[index])) + return self._files[index] + + def __repr__(self) -> str: + remote = self.kind == "remote" + entries = [ + f"{_name_str(f)} (remote)" if remote else _name_str(f) + for f in self._files + ] + return f"Files[{', '.join(entries)}]" + + # -- download ------------------------------------------------------- + + def download( + self, + indexes: list[int] | tuple[int, ...] | None = None, + ) -> FileBag[BaseLocalFile]: + """Download the (remote) files and return a local ``FileBag``. + + If the bag already holds local files this is a no-op and returns + ``self``. ``indexes`` optionally selects a subset of files (defaults + to all). Runs synchronously; the async ``PySUS`` download machinery + is started and awaited internally. + """ + if self.kind == "local": + return cast("FileBag[BaseLocalFile]", self) + + selected = _select(self._files, indexes) + local = cast( + list[BaseLocalFile], + _run_sync(_download_many(cast(list[BaseRemoteFile], selected))), + ) + return FileBag(local) + + def download_one(self, index: int = 0) -> BaseLocalFile: + """Download a single file and return the local file entity.""" + return self.download(indexes=[index]).files[0] + + # -- tabular convenience --------------------------------------------- + + def to_dataframe(self) -> pd.DataFrame: + """Concatenate all local tabular files into one DataFrame. + + Only meaningful when the bag holds local files; remote files must be + downloaded first. + """ + if self.kind == "remote": + raise ValueError( + "Cannot build a DataFrame from a remote FileBag; call " + "download() first." + ) + import pandas as pd + + frames = cast(list[pd.DataFrame], _run_sync(_load_frames(self._files))) + return ( + pd.concat(frames, ignore_index=True) if frames else pd.DataFrame() + ) + + @property + def df(self) -> pd.DataFrame: + """Alias for :meth:`to_dataframe` (concatenated local frames).""" + return self.to_dataframe() + + @property + def first(self) -> F: + """The first file entity.""" + return self._files[0] + + +# ── internal helpers ────────────────────────────────────────────────── + + +def _path_str(f: object) -> str: + path = getattr(f, "path", None) + if path is None: + return _name_str(f) + if isinstance(path, (str, bytes)): + return str(path) + fs = getattr(path, "__fspath__", None) + if fs is not None: + return fs() + return str(path) + + +def _name_str(f: object) -> str: + name = getattr(f, "name", None) or getattr(f, "basename", None) + if name is not None: + return str(name) + path = getattr(f, "path", None) + if path is not None: + base = getattr(path, "name", None) or path + return str(base).rsplit("/", 1)[-1] + return repr(f) + + +def _kind_of(files: tuple[object, ...]) -> str: + if not files: + return "local" + return "local" if isinstance(files[0], BaseLocalFile) else "remote" + + +def _select( + files: tuple[object, ...], + indexes: list[int] | tuple[int, ...] | None, +) -> list[object]: + if indexes is None: + return list(files) + return [files[i] for i in indexes] + + +async def _download_many(files: list[BaseRemoteFile]) -> list[BaseLocalFile]: + return [await f.download() for f in files] + + +async def _load_frames(files: tuple[object, ...]) -> list[pd.DataFrame]: + import pandas as pd + + frames: list[pd.DataFrame] = [] + for f in files: + if not isinstance(f, BaseLocalFile): + continue + data = await f.load() + if isinstance(data, pd.DataFrame): + frames.append(data) + return frames + + +class _RemoteURL: + """Minimal remote-file stand-in for URL-only origins (e.g. Saude). + + Holds a remote CSV/download URL in :attr:`path` and knows how to fetch it + into a local file through :func:`download_http`. It is intentionally + not a ``BaseRemoteFile`` subclass (whose ``path`` is a local ``Path``); + it exists so URL-backed downloads still surface as an item in a + ``FileBag``. + """ + + __slots__ = ("path",) + + def __init__(self, url: str) -> None: + self.path = url + + @property + def basename(self) -> str: + return self.path.rsplit("/", 1)[-1] or self.path + + name = basename + + def __repr__(self) -> str: # pragma: no cover - debug helper + return f"_RemoteURL({self.path!r})" + + async def download(self) -> BaseLocalFile: + from pathlib import Path + + import httpx + from pysus.api.extensions import ExtensionFactory + + dest = Path(Path(__import__("tempfile").gettempdir())) / self.basename + async with httpx.AsyncClient(follow_redirects=True) as client: + resp = await client.get(self.path) + resp.raise_for_status() + dest.write_bytes(resp.content) + return await ExtensionFactory.instantiate(dest) diff --git a/pysus/tests/api/test_file_bag.py b/pysus/tests/api/test_file_bag.py new file mode 100644 index 00000000..68f2bb49 --- /dev/null +++ b/pysus/tests/api/test_file_bag.py @@ -0,0 +1,204 @@ +"""Phase FileBag tests - high-level FileBag entity. + +Covers: +- FileBag wraps remote and local file entities; +- sync ``download()`` / ``download_one()`` / ``download(indexes=)``; +- ``to_dataframe()`` / ``df`` over local tabular files; +- subsetting via ``__getitem__`` / ``__len__`` / ``__iter__`` / ``paths``; +- namespaced fetchers return a FileBag (download=False -> remote bag); +- ``as_dataframe=True`` still yields a plain DataFrame. +""" + +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pandas as pd +import pytest +from pysus.api.bag import FileBag +from pysus.api.client import PySUS, _run_sync +from pysus.api.extensions import ExtensionFactory +from pysus.api.models import BaseLocalFile + + +def _local(path: Path) -> BaseLocalFile: + return _run_sync(ExtensionFactory.instantiate(path)) + + +def _remotable(path: str, target: Path | None = None): + """Lightweight remote-file stand-in with ``path`` and async ``download``.""" + + class _RemoteStub: + path: Path + + def __init__(self, key): + self.path = Path(key) + + async def download(self): + if target is None: + raise AssertionError("unexpected download") + return await ExtensionFactory.instantiate(target) + + return _RemoteStub(path) + + +@pytest.fixture() +def pysus(): + import pysus # noqa: F401 + + return pysus + + +def _make_parquet(path: Path, rows: int = 3) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame({"a": list(range(rows))}).to_parquet(path) + return path + + +class TestFileBagBuilding: + def test_remote_kind_and_paths(self): + bag = FileBag([_remotable("public/data/ftp/sinan/a.parquet")]) + assert bag.kind == "remote" + assert not isinstance(bag.files[0], BaseLocalFile) + assert bag.paths == ["public/data/ftp/sinan/a.parquet"] + + def test_repr_remote_marks_files_as_remote(self): + bag = FileBag( + [ + _remotable( + "public/data/ftp/sinan/DENG/2020/BR/DENGBR20.parquet" + ), + _remotable( + "public/data/ftp/sinan/DENG/2021/BR/DENGBR21.parquet" + ), + ] + ) + assert ( + repr(bag) + == "Files[DENGBR20.parquet (remote), DENGBR21.parquet (remote)]" + ) + + def test_repr_local_omits_flag(self, tmp_path): + bag = FileBag([_local(_make_parquet(tmp_path / "a.parquet", rows=1))]) + assert repr(bag) == "Files[a.parquet]" + + def test_empty_is_treated_as_local(self): + bag = FileBag([]) + assert len(bag) == 0 + assert bag.kind == "local" + + def test_local_bag_kind(self, tmp_path): + bag = FileBag([_local(_make_parquet(tmp_path / "a.parquet"))]) + assert bag.kind == "local" + assert isinstance(bag.files[0], BaseLocalFile) + + def test_len_iter_getitem(self, tmp_path): + p1 = _make_parquet(tmp_path / "a.parquet") + p2 = _make_parquet(tmp_path / "b.parquet") + bag = FileBag([_local(p1), _local(p2)]) + + assert len(bag) == 2 + assert [f.path for f in bag] == [p1, p2] + assert bag[0].path == p1 + assert isinstance(bag[0:1], FileBag) + assert bag.first.path == p1 + + +class TestDownload: + def test_download_converts_remote_to_local(self, tmp_path): + local = _make_parquet(tmp_path / "DENGBR20.parquet") + bag = FileBag( + [ + _remotable( + "public/data/ftp/sinan/DENG/2020/_/BR/DENGBR20.parquet", + local, + ) + ] + ) + result = bag.download() + assert result.kind == "local" + assert result.paths == [str(local)] + assert len(result) == 1 + + def test_download_one(self, tmp_path): + local = _make_parquet(tmp_path / "x.parquet") + bag = FileBag([_remotable("public/data/x.parquet", local)]) + lf = bag.download_one(0) + assert isinstance(lf, BaseLocalFile) + + def test_download_subset(self, tmp_path): + locals_ = [ + _make_parquet(tmp_path / "a.parquet", rows=1), + _make_parquet(tmp_path / "b.parquet", rows=1), + ] + bag = FileBag( + [ + _remotable(f"public/data/{i}.parquet", target) + for i, target in enumerate(locals_) + ] + ) + subset = bag.download(indexes=[1]) + assert subset.paths == [str(locals_[1])] + + def test_local_bag_download_is_noop(self, tmp_path): + bag = FileBag([_local(_make_parquet(tmp_path / "a.parquet"))]) + assert bag.download() is bag + + +class TestDataFrame: + def test_to_dataframe_concatenates(self, tmp_path): + p1 = _make_parquet(tmp_path / "a.parquet", rows=2) + p2 = _make_parquet(tmp_path / "b.parquet", rows=3) + bag = FileBag([_local(p1), _local(p2)]) + df = bag.to_dataframe() + assert isinstance(df, pd.DataFrame) + assert len(df) == 5 + assert bag.df.equals(df) + + def test_remote_bag_to_dataframe_raises(self): + bag = FileBag([_remotable("public/data/x.parquet")]) + with pytest.raises(ValueError): + bag.to_dataframe() + + +class TestNamespacedReturn: + def test_download_false_returns_remote_file_bag(self, pysus): + import pysus.tests.api.test_origins as origins + + files = [ + origins._StubFile( + "public/data/ftp/sinan/DENG/2017/_/BR/DENGBR17.parquet" + ) + ] + with patch.object(PySUS, "query", new_callable=AsyncMock) as query: + query.return_value = files + result = pysus.ftp.sinan(disease="deng", year=2017, download=False) + assert isinstance(result, FileBag) + assert result.kind == "remote" + assert result.paths == [ + "public/data/ftp/sinan/DENG/2017/_/BR/DENGBR17.parquet" + ] + + def test_as_dataframe_still_returns_dataframe(self, pysus): + import pysus.tests.api.test_origins as origins + + class _Reader: + def df(self): + return pd.DataFrame({"a": [1, 2]}) + + files = [origins._StubFile("public/data/ftp/sinan/a.parquet")] + with ( + patch.object(PySUS, "query", new_callable=AsyncMock) as query, + patch.object( + PySUS, + "download", + new_callable=AsyncMock, + return_value=origins._StubFile("x"), + ), + patch.object(PySUS, "read_parquet", return_value=_Reader()), + ): + query.return_value = files + result = pysus.ftp.sinan( + disease="deng", year=2017, as_dataframe=True + ) + assert isinstance(result, pd.DataFrame) + assert list(result["a"]) == [1, 2] diff --git a/pysus/tests/api/test_origins.py b/pysus/tests/api/test_origins.py index 8989cabf..d80df32e 100644 --- a/pysus/tests/api/test_origins.py +++ b/pysus/tests/api/test_origins.py @@ -15,6 +15,7 @@ import pytest from pysus.api._impl.source import APPLICABILITY, origin_fetchers +from pysus.api.bag import FileBag from pysus.api.client import PySUS @@ -143,7 +144,9 @@ def test_download_false_lists_remote_paths(self, pysus): with patch.object(PySUS, "query", new_callable=AsyncMock) as query: query.return_value = files result = pysus.ftp.sinan(disease="deng", year=2017, download=False) - assert result == [ + assert isinstance(result, FileBag) + assert result.kind == "remote" + assert result.paths == [ "public/data/ftp/sinan/DENG/2017/_/BR/DENGBR17.parquet" ] query.assert_awaited_once() @@ -155,9 +158,10 @@ def test_download_false_ignores_as_dataframe(self, pysus): result = pysus.ftp.sinan( disease="deng", year=2017, download=False, as_dataframe=True ) - # no dataframe without downloaded data -> a plain path list - assert isinstance(result, list) - assert result == ["public/data/ftp/sinan/a.parquet"] + # namespaced fetchers always yield a remote FileBag on download=False + assert isinstance(result, FileBag) + assert result.kind == "remote" + assert result.paths == ["public/data/ftp/sinan/a.parquet"] def test_download_false_does_not_download(self, pysus): files = [_StubFile("public/data/ftp/sinan/a.parquet")] From bf49ca07c2f6618ee5f44b15e0d15bce226903c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 11:07:50 -0300 Subject: [PATCH 13/18] fix: clean DuckDataset and File reprs DuckDataset and File inherited Pydantic's default repr, which dumped every field including private internals (e.g. border=). Add concise __repr__ implementations that surface the meaningful fields: DuckDataset shows its record name; File shows path, type, optional group, and dataset. No serialization behavior changes. --- pysus/api/ducklake/models.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pysus/api/ducklake/models.py b/pysus/api/ducklake/models.py index 37ef371a..77c20516 100644 --- a/pysus/api/ducklake/models.py +++ b/pysus/api/ducklake/models.py @@ -56,6 +56,13 @@ def record(self) -> CatalogFile: def basename(self) -> str: return self.path.name + def __repr__(self) -> str: + group = f" group={self.group!r}" if self.group else "" + return ( + f"File(path={str(self.path)!r}, type={self.type!r}" + f"{group}, dataset={self.dataset!r})" + ) + @property def extension(self) -> str: return self.path.suffix @@ -115,6 +122,9 @@ def __init__(self, **data) -> None: def __str__(self) -> str: return self.record.name + def __repr__(self) -> str: + return f"DuckDataset(name={self.record.name!r})" + @property def adapter(self) -> "DatasetAdapter": return self.border From 72c73b5dc6697b0a91ae8ef3b14d4e140251bacf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 11:11:09 -0300 Subject: [PATCH 14/18] docs: document FileBag workflow across guides Add a full walkthrough of the origin-namespaced FileBag workflow (list with download=False, inspect, download/download_one, to_dataframe) to tutorials.rst, with an explicit anchor target; reference it from data-sources.rst and files-and-formats.rst so all guides show the download=False -> FileBag -> download() -> to_dataframe() path rather than only the DataFrame form. --- docs/source/databases/data-sources.rst | 27 +++++++++++++ docs/source/guides/files-and-formats.rst | 19 +++++++++ docs/source/tutorials.rst | 49 ++++++++++++++++++++++++ 3 files changed, 95 insertions(+) diff --git a/docs/source/databases/data-sources.rst b/docs/source/databases/data-sources.rst index 4e4d48ab..0ae4d4ad 100644 --- a/docs/source/databases/data-sources.rst +++ b/docs/source/databases/data-sources.rst @@ -53,6 +53,33 @@ OpenDataSUS (Saude) functions The legacy flat functions (``pysus.sinan``, ``pysus.arboviroses``, ...) still work but emit a deprecation warning pointing to the namespaced form. +Working with files (FileBag) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Besides returning a ``DataFrame`` directly, namespaced fetchers can hand you a +``FileBag`` to inspect or download files explicitly: + +.. code-block:: python + + import pysus + + # Nothing is downloaded here — just a listing of remote files + bag = pysus.ftp.sinan(disease="deng", year=2020, download=False) + print(bag) # Files[DENGBR20.parquet (remote)] + print(bag[0].path) # public/data/ftp/sinan/DENG/2020/_/BR/DENGBR20.parquet + + # Download them all (or bag.download_one(0)) -> a local FileBag + local = bag.download() # Files[DENGBR20.parquet] + + # Concatenate the downloaded files into a single DataFrame + df = local.to_dataframe() # same as local.df + +Return type by keyword: + +* ``download=False`` → remote ``FileBag`` (listings; ``as_dataframe`` ignored) +* ``download=True`` (default) + ``as_dataframe=False`` → local ``FileBag`` +* ``as_dataframe=True`` → concatenated ``pandas.DataFrame`` + Function Reference ^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/guides/files-and-formats.rst b/docs/source/guides/files-and-formats.rst index e3740c05..139d8bad 100644 --- a/docs/source/guides/files-and-formats.rst +++ b/docs/source/guides/files-and-formats.rst @@ -118,6 +118,25 @@ read directly without conversion: df = pysus.saude.arboviroses(disease="dengue", year=2024) +If you only want to list the CSV resources (without downloading), or work +with the downloaded files individually, use a ``FileBag``: + +.. code-block:: python + + import pysus + + # Remote listing — nothing downloaded + bag = pysus.saude.arboviroses(disease="dengue", download=False) + print(bag) + # Files[fa_casoshumanos_1994-2026.csv (remote), fa_epizpnh_1994-2026.csv (remote)] + + # Download and read as a DataFrame + df = bag.download().to_dataframe() # == bag.download().df + +The ``FileBag`` API (``download``, ``download_one``, ``to_dataframe``, +``paths``, slice access) is shared across all origins and formats; see the +:ref:`FileBag Workflow ` tutorial for the full walkthrough. + ZIP Archives ------------ diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst index 09bb0936..ea95c0c2 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -31,6 +31,55 @@ Use an origin namespace to fetch data — the data source is explicit: # CNES health facilities df = pysus.ftp.cnes(state="SP", year=2024, month=1) +The ``as_dataframe=True`` fetchers above return a single concatenated +``pandas.DataFrame``. See the :ref:`FileBag Workflow ` +section below for how to inspect files before downloading or work with the +downloaded files individually. + +.. _filebag-workflow: + +FileBag Workflow +---------------- + +A namespaced fetcher returns either a high-level ``FileBag`` or a +``DataFrame``: + +* ``download=False`` → a **remote** ``FileBag`` listing the files that would + be fetched, without downloading anything (``as_dataframe`` is ignored + here). +* ``download=True`` (default) + ``as_dataframe=False`` → a **local** + ``FileBag`` of downloaded files. +* ``as_dataframe=True`` → a single concatenated ``pandas.DataFrame``. + +A ``FileBag`` is synchronous — the underlying async client is started and +awaited internally: + +.. code-block:: python + + import pysus + + # 1. List what would be downloaded (nothing is fetched yet) + bag = pysus.ftp.sinan(disease="deng", year=2020, download=False) + print(bag) + # Files[DENGBR20.parquet (remote)] + + # 2. Inspect individual files (path, type, dataset, ...) + f = bag[0] + print(f.path) # public/data/ftp/sinan/DENG/2020/_/BR/DENGBR20.parquet + + # 3. Download all (or a subset) -> a local FileBag + local = bag.download() # Files[DENGBR20.parquet] + # local = bag.download(indexes=[0]) or bag.download_one(0) + + # 4. Concatenate the downloaded tabular files into one DataFrame + df = local.to_dataframe() # same as local.df + print(df.shape) # (975842, 121) + +``len``/``iter``/``[]`` (including slices) and ``paths`` let you introspect a +bag, and ``kind`` tells you whether it holds ``"remote"`` or ``"local"`` +files. Remote bags from URL-only origins (e.g. ``pysus.saude.*``) also +support this workflow. + OpenDataSUS (Saude) Functions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 8729c5c8d56b56d8062a98e218e4e122b2d1a17c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 11:15:00 -0300 Subject: [PATCH 15/18] feat: surface FileBag option in pysus.info() info() now prints a footer tip reminding users that namespaced fetchers return a FileBag, accept download=False to list files without fetching, and as_dataframe=True for a DataFrame. Add the same guidance to the info_table docstring. --- pysus/api/_impl/_ui.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pysus/api/_impl/_ui.py b/pysus/api/_impl/_ui.py index c2ffd8cb..598fa6f5 100644 --- a/pysus/api/_impl/_ui.py +++ b/pysus/api/_impl/_ui.py @@ -131,6 +131,11 @@ def info_table() -> None: import pysus pysus.info() + + Each row suggests the namespaced call for that dataset. Those fetchers + return a :class:`~pysus.api.bag.FileBag` by default; pass + ``as_dataframe=True`` for a ``pandas.DataFrame``, or ``download=False`` + to list the files without fetching anything. """ from pysus import CACHEPATH @@ -163,6 +168,11 @@ def info_table() -> None: print( f"\n Total: {len(rows)} datasets | Cache: {CACHEPATH}", ) + print( + " Tip: add download=False to list files without fetching, or " + "as_dataframe=True to get a pandas DataFrame. These fetchers return " + "a FileBag; see the docs.", + ) def search( From 8b3d614f769a60b2d21944f61f1c1369cf85fde9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 11:26:32 -0300 Subject: [PATCH 16/18] feat: add pysus.saude.cnes via per-origin function alias Saude's CNES was declared in the catalog (DatasetSpec) but had no namespaced fetcher, so pysus.info() showed an empty Call for it. Unlike the FTP/DadosGov cnes (monthly state/year/month dumps), Saude serves CNES as CKAN resources (/cnes/estabelecimentos, /cnes/tipounidades) through _fetch_saude. - Add saude_cnes() in _impl.databases: a **kwargs fetcher routing to _fetch_data(dataset="cnes", origin="Saude"). - Add a per-origin function alias map in _impl.source so build_origin_module binds SAUDE's "cnes" to saude_cnes while FTP/DadosGov keep the shared monthly-dump cnes. - Register "cnes" in APPLICABILITY["SAUDE"]; update saude module docstring (19 themes) with the new fetcher. - Tests: saude.cnes routes to _fetch_saude and returns a remote FileBag on download=False; info() now shows pysus.saude.cnes(...) (no dataset lacks a hint, so the omission test now exercises _fetcher_hint directly). Full suite: 1626 passed, 2 skipped. --- pysus/api/_impl/databases.py | 16 ++++++++++++++++ pysus/api/_impl/source.py | 11 ++++++++++- pysus/saude.py | 5 +++-- pysus/tests/api/test_info.py | 11 +++++++++-- pysus/tests/api/test_origins.py | 18 ++++++++++++++++++ 5 files changed, 56 insertions(+), 5 deletions(-) diff --git a/pysus/api/_impl/databases.py b/pysus/api/_impl/databases.py index 7e1f6b1f..bb30d4ad 100644 --- a/pysus/api/_impl/databases.py +++ b/pysus/api/_impl/databases.py @@ -1001,6 +1001,22 @@ def vigilancia_meio_ambiente(**kwargs) -> list[str] | pd.DataFrame: ) +def saude_cnes(**kwargs) -> list[str] | pd.DataFrame: + """Fetch CNES health-facility registers from Saude (CKAN resources). + + Unlike the FTP/DadosGov ``cnes`` fetcher (monthly state/year/month + dumps), the Saude portal serves CNES as catalog resources + (``/cnes/estabelecimentos``, ``/cnes/tipounidades``) fetched through + ``_fetch_saude``. + + Examples + -------- + >>> pysus.saude.cnes(download=False) + >>> pysus.saude.cnes(as_dataframe=True) + """ + return _fetch_data(dataset="cnes", origin="Saude", **kwargs) + + def list_files( dataset: types.DatasetName, client: types.Origin | None = None, diff --git a/pysus/api/_impl/source.py b/pysus/api/_impl/source.py index 9f95c7fe..81e01741 100644 --- a/pysus/api/_impl/source.py +++ b/pysus/api/_impl/source.py @@ -100,6 +100,7 @@ def valid_origins() -> tuple[str, ...]: "atencao_primaria", "bnafar", "ciencia_tecnologia", + "cnes", "diagnosticos_tratamentos", "economia_saude", "educacao_saude", @@ -120,6 +121,13 @@ def valid_origins() -> tuple[str, ...]: # Public, read-only view of the applicability matrix. APPLICABILITY: dict[str, frozenset[str]] = dict(_APPLICABILITY) +# Canonical fetcher name -> function to bind, per origin. Used when an +# origin exposes a name whose FTP-style implementation differs (e.g. Saude's +# CNES is a CKAN resource set, not monthly state/year/month dumps). +_ORIGIN_FUNC_ALIAS: dict[tuple[str, str], str] = { + ("SAUDE", "cnes"): "saude_cnes", +} + def origin_fetchers(origin: str) -> frozenset[str]: """Return the canonical fetcher names applicable to an origin.""" @@ -581,7 +589,8 @@ def build_origin_module(name: str, origin: str) -> _pytypes.SimpleNamespace: all_names: list[str] = [] for fname in sorted(allowed): - fn = getattr(_db, fname, None) + fn_name = _ORIGIN_FUNC_ALIAS.get((origin_key, fname), fname) + fn = getattr(_db, fn_name, None) if fn is None: continue ns[fname] = _bind_origin(fn, origin_key) diff --git a/pysus/saude.py b/pysus/saude.py index 92752592..8644e681 100644 --- a/pysus/saude.py +++ b/pysus/saude.py @@ -1,7 +1,7 @@ """Saude portal origin — ``pysus.saude``. Access to the Saude open-data portal (dadosabertos.saude.gov.br) theme -datasets: arboviroses, vacinacao, vigilancia_meio_ambiente, and 15 more. +datasets: arboviroses, vacinacao, vigilancia_meio_ambiente, and 16 more. Import styles ───────────── @@ -12,13 +12,14 @@ from pysus.saude import vacinacao -Fetching (read data, 18 themes) +Fetching (read data, 19 themes) ─────────────────────────────── pysus.saude.arboviroses(...) dengue/chikungunya/zika pysus.saude.assistencia_saude(...) hospital & facilities pysus.saude.atencao_primaria(...) primary care (SISAB) pysus.saude.bnafar(...) pharmaceutical assistance pysus.saude.ciencia_tecnologia(...) science & technology + pysus.saude.cnes(...) CNES health-facility registers pysus.saude.diagnosticos_tratamentos(...) diagnostics & treatments pysus.saude.economia_saude(...) health economics pysus.saude.educacao_saude(...) health education diff --git a/pysus/tests/api/test_info.py b/pysus/tests/api/test_info.py index d178deef..58fc4219 100644 --- a/pysus/tests/api/test_info.py +++ b/pysus/tests/api/test_info.py @@ -93,9 +93,16 @@ def test_info_map_exception_names_to_fetchers(self, capsys): output = capsys.readouterr().out assert "pysus.ftp.ibge(...)" in output - def test_info_omits_hint_when_no_fetcher(self, capsys): + def test_info_shows_saude_cnes_hint(self, capsys): import pysus pysus.info() output = capsys.readouterr().out - assert "saude.cnes(...)" not in output + assert "pysus.saude.cnes(...)" in output + + def test_info_omits_hint_when_no_fetcher(self): + from pysus.api._impl._ui import _fetcher_hint + + # A dataset with no namespaced fetcher yields an empty hint. + assert _fetcher_hint("NOT_A_DATASET", "FTP") == "" + assert _fetcher_hint("NOT_A_DATASET", "Saude") == "" diff --git a/pysus/tests/api/test_origins.py b/pysus/tests/api/test_origins.py index d80df32e..dcb96cd7 100644 --- a/pysus/tests/api/test_origins.py +++ b/pysus/tests/api/test_origins.py @@ -130,6 +130,24 @@ def test_source_origin_routes_to_direct(self, pysus): pysus.ftp.sinan(disease="deng", year=2017, source="origin") direct.assert_awaited_once() + def test_saude_cnes_routes_to_saude_fetch(self, pysus): + # Saude exposes its own cnes (CKAN resources), distinct from the + # FTP/DadosGov monthly-dump cnes, and it routes to _fetch_saude. + from unittest.mock import AsyncMock, patch + + with patch( + "pysus.api._impl.databases._fetch_saude", + new_callable=AsyncMock, + return_value=["https://example.gov/cnes/estabelecimentos.csv"], + ) as fetch: + result = pysus.saude.cnes(download=False) + fetch.assert_awaited_once() + assert isinstance(result, FileBag) + assert result.kind == "remote" + assert "cnes" in fetch.call_args.kwargs["dataset"] + # FTP/DadosGov keep their own cnes. + assert pysus.ftp.cnes is not pysus.saude.cnes + class _StubFile: def __init__(self, path): From fe51573f999ad0fc2cbf34b689903c2ea5684b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 11:55:57 -0300 Subject: [PATCH 17/18] fix: resolve Saude themes via DatasetSpec and match CSV by format Replace the hard-coded _SAUDE_GROUP_MAP resolution in _fetch_saude with spec-driven enumeration (DatasetSpec ckan_group + slug_patterns) so slug-only themes like CNES, SISVAN and OUTROSTEMAS resolve correctly instead of returning nothing. Filter CSV resources by the format metadata (capturing *_csv.zip multi-format archives) rather than by a bare .csv URL suffix, and unwrap downloaded zips when building a DataFrame. --- pysus/api/_impl/databases.py | 102 ++++- pysus/api/bag.py | 12 +- pysus/tests/api/saude/test_databases.py | 52 +++ pysus/tests/api/test_databases.py | 504 ++++++++++++++++++++++++ 4 files changed, 657 insertions(+), 13 deletions(-) diff --git a/pysus/api/_impl/databases.py b/pysus/api/_impl/databases.py index bb30d4ad..ee082119 100644 --- a/pysus/api/_impl/databases.py +++ b/pysus/api/_impl/databases.py @@ -291,22 +291,106 @@ def _saude_csv_to_frame(path: str) -> pd.DataFrame | None: Saude resources are frequently Latin-1 (ISO-8859-1) even when advertised as UTF-8, so we sniff the delimiter and fall back across encodings. + + Multi-format resources are stored as ``*_csv.zip`` archives; when the + downloaded path is a zip, the first inner ``.csv`` file is read instead. """ + import io + + data = _saude_bytes(path) + if data is None: + return None try: - with open(path, encoding="utf-8", errors="replace") as fh: - text = fh.read(4096) - dialect = csv.Sniffer().sniff(text) + text = data.decode("utf-8", errors="replace") + dialect = csv.Sniffer().sniff(text[:4096]) sep = dialect.delimiter except Exception: # noqa: BLE001 sep = "," for enc in ("utf-8", "latin-1", "cp1252"): try: - return pd.read_csv(path, sep=sep, low_memory=False, encoding=enc) + return pd.read_csv( + io.BytesIO(data), + sep=sep, + low_memory=False, + encoding=enc, + ) except Exception: # noqa: BLE001 continue return None +def _saude_bytes(path: str) -> bytes | None: + """Read a downloaded Saude resource into bytes, unwrapping CSV zips.""" + import zipfile + + try: + if str(path).lower().endswith(".zip"): + with zipfile.ZipFile(path) as zf: + csv_names = [ + n for n in zf.namelist() if n.lower().endswith(".csv") + ] + if not csv_names: + return None + return zf.read(csv_names[0]) + with open(path, "rb") as fh: # noqa: SIM115 + return fh.read() + except Exception: # noqa: BLE001 + return None + + +def _saude_spec_name(name: str) -> str: + """Normalise a fetcher dataset name to its ``DatasetSpec.name``. + + Fetchers pass lowercase snake_case names (``"saude_indigena"``) while + the registry keys are uppercase without underscores (``"SAUDEINDIGENA"``). + """ + return name.upper().replace("_", "") + + +async def _saude_theme_entries( + saude, spec, fallback_group: str | None = None +) -> list: + """Return the catalog entries belonging to a Saude theme. + + Resolution mirrors :class:`SaudeDataset._fetch_content`: a theme is + described by a ``ckan_group`` (list that group) and, for slug-only + themes such as CNES/SISVAN/OUTROSTEMAS, by ``slug_patterns`` matched + against the catalog. When ``spec`` is unknown (legacy/extra names) + the ``fallback_group`` CKAN slug is listed directly. + """ + if spec is None: + if fallback_group: + return await saude.list_datasets(group=fallback_group) + return [] + if spec.ckan_group is None: + entries = [] + async for entry in saude.iter_datasets(): + if spec.matches(entry.name): + entries.append(entry) + return entries + entries = await saude.list_datasets(group=spec.ckan_group) + if spec.slug_patterns: + entries = [e for e in entries if spec.matches(e.name)] + return entries + + +def _is_saude_csv_resource(res) -> bool: + """True when *res* is a tabular CSV resource of a Saude package. + + Matches on the ``format`` metadata first, so that multi-format + resources stored as ``*_csv.zip`` are captured, and falls back to the + URL extension for plain ``.csv`` resources. Resources without a + usable URL (e.g. placeholders) and API/Documentation links are skipped. + """ + fmt = getattr(res, "format", None) or "" + url = getattr(res, "url", None) or "" + if not isinstance(url, str) or not url.startswith("http"): + return False + if str(fmt).upper() == "CSV": + return True + return url.lower().endswith(".csv") + + async def _fetch_saude( dataset: str, group: str | None = None, @@ -326,14 +410,16 @@ async def _fetch_saude( downloading anything (``as_dataframe`` is ignored). """ from pysus.api.client import PySUS + from pysus.api.saude.databases import SPECS_BY_NAME - dataset_upper = dataset.upper() - ckan_group = _SAUDE_GROUP_MAP.get(dataset_upper, dataset.lower()) + spec_name = _saude_spec_name(dataset) + spec = SPECS_BY_NAME.get(spec_name) + fallback_group = _SAUDE_GROUP_MAP.get(spec_name) async with PySUS() as pysus: saude = await pysus.get_saude() - entries = await saude.list_datasets(group=ckan_group) + entries = await _saude_theme_entries(saude, spec, fallback_group) if not entries: if as_dataframe and download: return pd.DataFrame() @@ -345,7 +431,7 @@ async def _fetch_saude( try: pkg = await saude.fetch_dataset(entry.name) for res in pkg.resources: - if res.url and res.url.lower().endswith(".csv"): + if _is_saude_csv_resource(res): resources.append((entry.name, res.id, res.url)) except Exception: # noqa: BLE001 continue diff --git a/pysus/api/bag.py b/pysus/api/bag.py index ff64b085..731d2c2b 100644 --- a/pysus/api/bag.py +++ b/pysus/api/bag.py @@ -144,11 +144,13 @@ def _path_str(f: object) -> str: if path is None: return _name_str(f) if isinstance(path, (str, bytes)): - return str(path) - fs = getattr(path, "__fspath__", None) - if fs is not None: - return fs() - return str(path) + value = str(path) + else: + fs = getattr(path, "__fspath__", None) + value = fs() if fs is not None else str(path) + # Remote keys and published paths are always POSIX; normalise the + # platform separator so ``paths`` is stable across OSes. + return value.replace("\\", "/") def _name_str(f: object) -> str: diff --git a/pysus/tests/api/saude/test_databases.py b/pysus/tests/api/saude/test_databases.py index 6bd867f5..e6dacbbe 100644 --- a/pysus/tests/api/saude/test_databases.py +++ b/pysus/tests/api/saude/test_databases.py @@ -158,6 +158,58 @@ def test_unreadable_returns_none(self, tmp_path): # legitimately invalid table should not raise. assert _saude_csv_to_frame(str(p)) is not None + def test_zipped_csv_is_unwrapped(self, tmp_path): + import zipfile + + from pysus.api._impl.databases import _saude_csv_to_frame + + p = tmp_path / "um_csv.zip" + with zipfile.ZipFile(p, "w") as zf: + zf.writestr( + "um/dicionario.txt", + "documentation", + ) + zf.writestr( + "um/dados.csv", + "ID;NOME\n1;JOÃO\n2;MARIA\n".encode(), + ) + df = _saude_csv_to_frame(str(p)) + assert df is not None + assert list(df.columns) == ["ID", "NOME"] + assert df["NOME"].tolist() == ["JOÃO", "MARIA"] + + def test_missing_file_returns_none(self, tmp_path): + from pysus.api._impl.databases import _saude_csv_to_frame + + assert _saude_csv_to_frame(str(tmp_path / "nao-existe.csv")) is None + + def test_empty_file_returns_none(self, tmp_path): + from pysus.api._impl.databases import _saude_csv_to_frame + + p = tmp_path / "vazio.csv" + p.write_bytes(b"") + assert _saude_csv_to_frame(str(p)) is None + + def test_zip_without_csv_returns_none(self, tmp_path): + import zipfile + + from pysus.api._impl.databases import _saude_csv_to_frame + + p = tmp_path / "sem_csv.zip" + with zipfile.ZipFile(p, "w") as zf: + zf.writestr("apenas.pdf", "not a table") + assert _saude_csv_to_frame(str(p)) is None + + def test_sniff_failure_falls_back_to_comma(self, tmp_path): + # A body with no discernible delimiter defeats csv.Sniffer; the + # fallback delimiter (",") must still produce a usable frame. + from pysus.api._impl.databases import _saude_csv_to_frame + + p = tmp_path / "simples.csv" + p.write_text("a\nb\nc\nd\n", encoding="utf-8") + df = _saude_csv_to_frame(str(p)) + assert df is not None and not df.empty + class TestDatasetSpecIsFrozen: def test_frozen(self): diff --git a/pysus/tests/api/test_databases.py b/pysus/tests/api/test_databases.py index e824f149..b707ae49 100644 --- a/pysus/tests/api/test_databases.py +++ b/pysus/tests/api/test_databases.py @@ -976,6 +976,510 @@ async def _run(): asyncio.run(_run()) + def test_slug_only_theme_resolves_via_catalog(self): + """CNES/SISVAN (no CKAN group, only slug_patterns) list the catalog.""" + + async def _run(): + with patch("pysus.api.client.PySUS") as mock_cls: + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + matches = MagicMock() + matches.name = "cnes-estabelecimentos" + non = MagicMock() + non.name = "some-other-dataset" + + async def _gen(): + for e in (matches, non): + yield e + + saude_mock.iter_datasets = _gen + saude_mock.download_resource = AsyncMock() + + resource = MagicMock() + resource.id = "r1" + resource.url = "https://example.com/cnes.csv" + pkg = MagicMock() + pkg.resources = [resource] + saude_mock.fetch_dataset = AsyncMock(return_value=pkg) + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="cnes", + show_progress=False, + ) + assert len(result) == 1 + saude_mock.fetch_dataset.assert_awaited_once_with( + "cnes-estabelecimentos" + ) + + asyncio.run(_run()) + + def test_zipped_csv_resource_is_captured_by_format(self): + """Resources stored as ``*_csv.zip`` (format=CSV) are included.""" + + async def _run(): + with ( + patch("pysus.api.client.PySUS") as mock_cls, + patch.object(pathlib.Path, "mkdir"), + ): + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + entry = MagicMock() + entry.name = "esavi" + saude_mock.list_datasets = AsyncMock(return_value=[entry]) + + zip_res = MagicMock() + zip_res.id = "zip1" + zip_res.url = "https://example.com/Esavi_csv.zip" + zip_res.format = "CSV" + pdf_res = MagicMock() + pdf_res.id = "pdf1" + pdf_res.url = "https://example.com/manual.pdf" + pdf_res.format = "PDF" + placeholder = MagicMock() + placeholder.id = "ph1" + placeholder.url = "." + placeholder.format = "CSV" + pkg = MagicMock() + pkg.resources = [zip_res, pdf_res, placeholder] + saude_mock.fetch_dataset = AsyncMock(return_value=pkg) + saude_mock.download_resource = AsyncMock( + return_value=Path("/tmp/Esavi_csv.zip"), + ) + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="vacinacao", + show_progress=False, + ) + assert len(result) == 1 + saude_mock.download_resource.assert_awaited_once_with( + "esavi", + resource_id="zip1", + dest_dir=Path("/tmp/test_cache") + / "downloads" + / "saude" + / "vacinacao", + ) + + asyncio.run(_run()) + + def test_download_false_lists_urls_only(self): + """download=False returns CSV URLs without downloading.""" + + async def _run(): + with patch("pysus.api.client.PySUS") as mock_cls: + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + entry = MagicMock() + entry.name = "esavi" + saude_mock.list_datasets = AsyncMock(return_value=[entry]) + + csv_res = MagicMock() + csv_res.id = "csv1" + csv_res.url = "https://example.com/Esavi_csv.zip" + csv_res.format = "CSV" + pkg = MagicMock() + pkg.resources = [csv_res] + saude_mock.fetch_dataset = AsyncMock(return_value=pkg) + saude_mock.download_resource = AsyncMock() + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="vacinacao", + download=False, + show_progress=False, + ) + assert result == ["https://example.com/Esavi_csv.zip"] + saude_mock.download_resource.assert_not_called() + + asyncio.run(_run()) + + def test_all_fetch_dataset_fail_returns_empty_df(self): + async def _run(): + with patch("pysus.api.client.PySUS") as mock_cls: + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + entry = MagicMock() + entry.name = "bad" + saude_mock.list_datasets = AsyncMock(return_value=[entry]) + saude_mock.fetch_dataset = AsyncMock( + side_effect=RuntimeError("boom"), + ) + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="arboviroses", + as_dataframe=True, + show_progress=False, + ) + assert isinstance(result, pd.DataFrame) + assert result.empty + + asyncio.run(_run()) + + def test_all_downloads_fail_returns_empty_list(self): + async def _run(): + with ( + patch("pysus.api.client.PySUS") as mock_cls, + patch.object(pathlib.Path, "mkdir"), + ): + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + entry = MagicMock() + entry.name = "esavi" + saude_mock.list_datasets = AsyncMock(return_value=[entry]) + + resource = MagicMock() + resource.id = "r1" + resource.url = "https://example.com/data.csv" + pkg = MagicMock() + pkg.resources = [resource] + saude_mock.fetch_dataset = AsyncMock(return_value=pkg) + saude_mock.download_resource = AsyncMock( + side_effect=RuntimeError("net down"), + ) + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="arboviroses", + show_progress=False, + ) + assert result == [] + assert saude_mock.download_resource.await_count == 1 + + asyncio.run(_run()) + + def test_all_downloads_fail_as_dataframe_returns_empty_df(self): + async def _run(): + with ( + patch("pysus.api.client.PySUS") as mock_cls, + patch.object(pathlib.Path, "mkdir"), + ): + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + entry = MagicMock() + entry.name = "esavi" + saude_mock.list_datasets = AsyncMock(return_value=[entry]) + + resource = MagicMock() + resource.id = "r1" + resource.url = "https://example.com/data.csv" + pkg = MagicMock() + pkg.resources = [resource] + saude_mock.fetch_dataset = AsyncMock(return_value=pkg) + saude_mock.download_resource = AsyncMock( + side_effect=RuntimeError("net down"), + ) + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="arboviroses", + as_dataframe=True, + show_progress=False, + ) + assert isinstance(result, pd.DataFrame) + assert result.empty + + asyncio.run(_run()) + + def test_unreadable_downloads_yield_empty_df(self): + async def _run(): + with ( + patch("pysus.api.client.PySUS") as mock_cls, + patch.object(pathlib.Path, "mkdir"), + ): + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + entry = MagicMock() + entry.name = "esavi" + saude_mock.list_datasets = AsyncMock(return_value=[entry]) + + resource = MagicMock() + resource.id = "r1" + resource.url = "https://example.com/data.csv" + pkg = MagicMock() + pkg.resources = [resource] + saude_mock.fetch_dataset = AsyncMock(return_value=pkg) + # Downloaded file does not exist -> _saude_csv_to_frame -> None. + saude_mock.download_resource = AsyncMock( + return_value=Path("/tmp/nao-existe.csv"), + ) + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="arboviroses", + as_dataframe=True, + show_progress=False, + ) + assert isinstance(result, pd.DataFrame) + assert result.empty + + asyncio.run(_run()) + + def test_group_backed_spec_filters_by_slug_pattern(self): + async def _run(): + with patch("pysus.api.client.PySUS") as mock_cls: + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + match = MagicMock() + match.name = "sisagua-2024" + skip = MagicMock() + skip.name = "outro-tema" + saude_mock.list_datasets = AsyncMock( + return_value=[match, skip], + ) + saude_mock.download_resource = AsyncMock() + + resource = MagicMock() + resource.id = "r1" + resource.url = "https://example.com/sisagua.csv" + pkg = MagicMock() + pkg.resources = [resource] + saude_mock.fetch_dataset = AsyncMock(return_value=pkg) + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="sisagua", + download=False, + ) + assert result == ["https://example.com/sisagua.csv"] + saude_mock.fetch_dataset.assert_awaited_once_with( + "sisagua-2024" + ) + + asyncio.run(_run()) + + def test_unknown_theme_falls_back_to_group_map(self): + """Legacy names (no spec) still resolve through _SAUDE_GROUP_MAP.""" + + async def _run(): + with patch("pysus.api.client.PySUS") as mock_cls: + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + entry = MagicMock() + entry.name = "vig-tema" + saude_mock.list_datasets = AsyncMock(return_value=[entry]) + saude_mock.download_resource = AsyncMock() + + resource = MagicMock() + resource.id = "r1" + resource.url = "https://example.com/data.csv" + pkg = MagicMock() + pkg.resources = [resource] + saude_mock.fetch_dataset = AsyncMock(return_value=pkg) + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="vigilancia_meio_ambiente", + download=False, + ) + assert result == ["https://example.com/data.csv"] + saude_mock.list_datasets.assert_awaited_once_with( + group="vigilancia-e-meio-ambiente" + ) + + asyncio.run(_run()) + + def test_show_progress_true_downloads(self): + async def _run(): + with ( + patch("pysus.api.client.PySUS") as mock_cls, + patch.object(pathlib.Path, "mkdir"), + ): + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + entry = MagicMock() + entry.name = "esavi" + saude_mock.list_datasets = AsyncMock(return_value=[entry]) + + resource = MagicMock() + resource.id = "r1" + resource.url = "https://example.com/data.csv" + pkg = MagicMock() + pkg.resources = [resource] + saude_mock.fetch_dataset = AsyncMock(return_value=pkg) + saude_mock.download_resource = AsyncMock( + return_value=Path("/tmp/data.csv"), + ) + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="arboviroses", + show_progress=True, + ) + assert len(result) == 1 + + asyncio.run(_run()) + + +class TestDownloadFiles: + + def test_empty_files_returns_empty_list(self): + async def _run(): + from pysus.api._impl.databases import _download_files + + mock_pysus = MagicMock() + result = await _download_files(mock_pysus, []) + assert result == [] + + asyncio.run(_run()) + + def test_empty_files_returns_empty_df_when_as_dataframe(self): + async def _run(): + from pysus.api._impl.databases import _download_files + + mock_pysus = MagicMock() + result = await _download_files(mock_pysus, [], as_dataframe=True) + assert isinstance(result, pd.DataFrame) + assert result.empty + + asyncio.run(_run()) + + def test_columns_filter_applied_to_dataframe(self): + async def _run(): + from pysus.api._impl.databases import _download_files + + mock_pysus = MagicMock() + mock_pysus.download = AsyncMock( + side_effect=lambda f: f, + ) + f1 = MagicMock() + f1.path = "a.parquet" + df = pd.DataFrame({"x": [1], "y": [2], "z": [3]}) + mock_pysus.read_parquet = MagicMock(return_value=df) + + result = await _download_files( + mock_pysus, + [f1], + show_progress=False, + as_dataframe=True, + columns=["x", "y"], + ) + assert list(result.columns) == ["x", "y"] + + asyncio.run(_run()) + + +class TestFetchDucklake: + + def test_download_false_without_bag_returns_paths(self): + async def _run(): + with patch("pysus.api.client.PySUS") as mock_cls: + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + f1 = MagicMock() + f1.path = "public/data/ftp/sinan/a.parquet" + mock_pysus.query = AsyncMock(return_value=[f1]) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_ducklake + + result = await _fetch_ducklake( + mock_pysus, + dataset="sinan", + origin="FTP", + download=False, + ) + assert result == ["public/data/ftp/sinan/a.parquet"] + mock_pysus.query.assert_awaited_once() + + asyncio.run(_run()) + class TestFlatDeprecationWarns: """Flat calls warn but still call _fetch_data with identical args.""" From a99ca51b7dee942f89dfdbc4b9cae1f3bf1e3625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 31 Aug 2026 23:51:50 -0300 Subject: [PATCH 18/18] test: fix and extend origin-namespaced fetch tests - TestFetchDucklake: drop invalid mock_pysus positional; _fetch_ducklake takes dataset as first param and opens its own PySUS context. - TestDownload paths: expect forward-slash normalized paths on Windows. - TestFetchOriginDirect.saude: use uppercase SAUDE origin. - TestInstantiateMany: mock ExtensionFactory.instantiate to raise so the defensive except-path is exercised. - Add TestBagHelpers, TestCoerceBag, TestInfoOutput, and an unknown-theme test for _fetch_saude. --- pysus/tests/api/test_databases.py | 28 ++- pysus/tests/api/test_file_bag.py | 90 +++++++++- pysus/tests/api/test_source.py | 273 ++++++++++++++++++++++++++++++ 3 files changed, 387 insertions(+), 4 deletions(-) diff --git a/pysus/tests/api/test_databases.py b/pysus/tests/api/test_databases.py index b707ae49..0d61d9f1 100644 --- a/pysus/tests/api/test_databases.py +++ b/pysus/tests/api/test_databases.py @@ -1362,6 +1362,33 @@ async def _run(): asyncio.run(_run()) + def test_fully_unknown_theme_returns_empty_list(self): + """A name absent from both SPECS and _SAUDE_GROUP_MAP yields [].""" + + async def _run(): + with patch("pysus.api.client.PySUS") as mock_cls: + mock_pysus = MagicMock() + mock_cls.return_value.__aenter__ = AsyncMock( + return_value=mock_pysus, + ) + mock_cls.return_value.__aexit__ = AsyncMock() + + saude_mock = AsyncMock() + saude_mock.download_resource = AsyncMock() + + mock_pysus.get_saude = AsyncMock(return_value=saude_mock) + mock_pysus.cachepath = Path("/tmp/test_cache") + + from pysus.api._impl.databases import _fetch_saude + + result = await _fetch_saude( + dataset="totally_unknown_theme", + download=False, + ) + assert result == [] + + asyncio.run(_run()) + def test_show_progress_true_downloads(self): async def _run(): with ( @@ -1470,7 +1497,6 @@ async def _run(): from pysus.api._impl.databases import _fetch_ducklake result = await _fetch_ducklake( - mock_pysus, dataset="sinan", origin="FTP", download=False, diff --git a/pysus/tests/api/test_file_bag.py b/pysus/tests/api/test_file_bag.py index 68f2bb49..1bdd0204 100644 --- a/pysus/tests/api/test_file_bag.py +++ b/pysus/tests/api/test_file_bag.py @@ -10,7 +10,7 @@ """ from pathlib import Path -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pandas as pd import pytest @@ -116,7 +116,7 @@ def test_download_converts_remote_to_local(self, tmp_path): ) result = bag.download() assert result.kind == "local" - assert result.paths == [str(local)] + assert result.paths == [str(local).replace("\\", "/")] assert len(result) == 1 def test_download_one(self, tmp_path): @@ -137,7 +137,7 @@ def test_download_subset(self, tmp_path): ] ) subset = bag.download(indexes=[1]) - assert subset.paths == [str(locals_[1])] + assert subset.paths == [str(locals_[1]).replace("\\", "/")] def test_local_bag_download_is_noop(self, tmp_path): bag = FileBag([_local(_make_parquet(tmp_path / "a.parquet"))]) @@ -202,3 +202,87 @@ def df(self): ) assert isinstance(result, pd.DataFrame) assert list(result["a"]) == [1, 2] + + +class TestBagHelpers: + def test_path_str_falls_back_to_name(self): + from pysus.api.bag import _path_str + + class _F: + path = None + name = "only-name.csv" + + assert _path_str(_F()) == "only-name.csv" + + def test_name_str_falls_back_to_repr(self): + from pysus.api.bag import _name_str + + f = object() + assert _name_str(f) == repr(f) + + def test_load_frames_skips_non_local_files(self): + from pysus.api.bag import _load_frames + + frames = _run_sync(_load_frames((MagicMock(),))) + assert frames == [] + + def test_path_str_normalizes_windows_separators(self): + from pysus.api.bag import _path_str + + class _F: + path = "public\\data\\ftp\\sinan\\a.parquet" + + assert _path_str(_F()) == "public/data/ftp/sinan/a.parquet" + + def test_path_str_preserves_posix_separators(self): + from pysus.api.bag import _path_str + + class _F: + path = "public/data/ftp/sinan/a.parquet" + + assert _path_str(_F()) == "public/data/ftp/sinan/a.parquet" + + def test_remote_url_basename(self): + from pysus.api.bag import _RemoteURL + + assert _RemoteURL("http://example.com/a.csv").basename == "a.csv" + assert ( + _RemoteURL("http://example.com/").basename == "http://example.com/" + ) + assert _RemoteURL("a.csv").basename == "a.csv" + + def test_remote_url_download(self): + from pysus.api.bag import _RemoteURL + + class _FakeResp: + content = b"parquet-bytes" + + def raise_for_status(self): + pass + + class _FakeClient: + def __init__(self, *args, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + async def get(self, url): + return _FakeResp() + + fake_local = MagicMock() + with ( + patch("httpx.AsyncClient", return_value=_FakeClient()), + patch( + "pysus.api.extensions.ExtensionFactory.instantiate", + return_value=fake_local, + ) as inst, + ): + result = _run_sync( + _RemoteURL("http://example.com/a.csv").download() + ) + assert result is fake_local + assert inst.called diff --git a/pysus/tests/api/test_source.py b/pysus/tests/api/test_source.py index c69467e2..57fbefac 100644 --- a/pysus/tests/api/test_source.py +++ b/pysus/tests/api/test_source.py @@ -307,3 +307,276 @@ def test_accepts_source_origin(self): ) as mock_direct: pysus.ftp.sinan(disease="deng", year=2017, source="origin") mock_direct.assert_awaited_once() + + +class TestFetchOriginDirect: + """Unit coverage for ``_fetch_origin_direct`` (FTP/DadosGov path).""" + + def _make_pysus(self, files): + from unittest.mock import AsyncMock, MagicMock + + pysus = MagicMock() + pysus.query = AsyncMock(return_value=files) + return pysus + + def _run(self, coro): + import asyncio + + return asyncio.run(coro) + + def _file(self, path): + from unittest.mock import MagicMock + + f = MagicMock() + f.path = path + return f + + def test_unsupported_origin_raises(self): + from pysus.api._impl.source import _fetch_origin_direct + from pysus.api.errors import ValidationError + + pysus = self._make_pysus([]) + with pytest.raises(ValidationError): + self._run( + _fetch_origin_direct( + pysus, + "sinan", + None, + None, + 2020, + None, + "BOGUS", + None, + False, + False, + ) + ) + + def test_empty_files_returns_empty_list(self): + from pysus.api._impl.source import _fetch_origin_direct + + pysus = self._make_pysus([]) + result = self._run( + _fetch_origin_direct( + pysus, + "sinan", + None, + None, + 2020, + None, + "FTP", + None, + False, + False, + download=False, + ) + ) + assert result == [] + + def test_empty_files_returns_empty_df_when_as_dataframe(self): + import pandas as pd + from pysus.api._impl.source import _fetch_origin_direct + + pysus = self._make_pysus([]) + result = self._run( + _fetch_origin_direct( + pysus, + "sinan", + None, + None, + 2020, + None, + "FTP", + None, + True, + True, + download=True, + ) + ) + assert isinstance(result, pd.DataFrame) + assert result.empty + + def test_download_false_bag_returns_files(self): + from pysus.api._impl.source import _fetch_origin_direct + + f = self._file("public/data/ftp/sinan/a.parquet") + pysus = self._make_pysus([f]) + result = self._run( + _fetch_origin_direct( + pysus, + "sinan", + None, + None, + 2020, + None, + "FTP", + None, + False, + False, + download=False, + _bag=True, + ) + ) + assert result == [f] + + def test_download_false_returns_paths(self): + from pysus.api._impl.source import _fetch_origin_direct + + f = self._file("public/data/ftp/sinan/a.parquet") + pysus = self._make_pysus([f]) + result = self._run( + _fetch_origin_direct( + pysus, + "sinan", + None, + None, + 2020, + None, + "FTP", + None, + False, + False, + download=False, + ) + ) + assert result == ["public/data/ftp/sinan/a.parquet"] + + def test_prefix_filter_keeps_matching(self): + from pysus.api._impl.source import _fetch_origin_direct + + matching = self._file("public/data/ftp/sinan/a.parquet") + other = self._file("elsewhere/b.parquet") + pysus = self._make_pysus([matching, other]) + result = self._run( + _fetch_origin_direct( + pysus, + "sinan", + None, + None, + 2020, + None, + "FTP", + None, + False, + False, + download=False, + ) + ) + assert result == ["public/data/ftp/sinan/a.parquet"] + + def test_download_true_delegates_to_download_files(self): + import pandas as pd + from pysus.api._impl import databases as db + from pysus.api._impl.source import _fetch_origin_direct + + f = self._file("public/data/ftp/sinan/a.parquet") + pysus = self._make_pysus([f]) + with patch.object( + db, + "_download_files", + new=AsyncMock( + return_value=pd.DataFrame({"a": [1]}), + ), + ) as mock_dl: + result = self._run( + _fetch_origin_direct( + pysus, + "sinan", + None, + None, + 2020, + None, + "FTP", + None, + False, + False, + ) + ) + mock_dl.assert_awaited_once() + assert list(result["a"]) == [1] + + def test_saude_origin_delegates(self): + from pysus.api._impl import databases as db + from pysus.api._impl.source import _fetch_origin_direct + + pysus = self._make_pysus([]) + with patch.object( + db, + "_fetch_saude", + new=AsyncMock(return_value=["x"]), + ) as mock_saude: + result = self._run( + _fetch_origin_direct( + pysus, + "arboviroses", + None, + None, + None, + None, + "SAUDE", + None, + False, + False, + download=False, + ) + ) + mock_saude.assert_awaited_once() + assert result == ["x"] + + +class TestCoerceBag: + def test_none_becomes_empty_filebag(self): + from pysus.api._impl.source import _coerce_bag + from pysus.api.bag import FileBag + + result = _coerce_bag(None) + assert isinstance(result, FileBag) + assert len(result) == 0 + + def test_base_local_file_list_becomes_filebag(self, tmp_path): + import pandas as pd + from pysus.api._impl.source import _coerce_bag + from pysus.api.bag import FileBag + from pysus.api.client import _run_sync + from pysus.api.extensions import ExtensionFactory + + p = tmp_path / "a.parquet" + p.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame({"a": [1]}).to_parquet(p) + local = _run_sync(ExtensionFactory.instantiate(p)) + result = _coerce_bag([local]) + assert isinstance(result, FileBag) + assert result.kind == "local" + + def test_remote_url_strings_become_remote_url_bag(self): + from pysus.api._impl.source import _coerce_bag + from pysus.api.bag import FileBag + + result = _coerce_bag(["https://example.com/data.csv"]) + assert isinstance(result, FileBag) + assert result.kind == "remote" + + +class TestInstantiateMany: + def test_skips_uninstantiable_paths(self): + from unittest.mock import AsyncMock, MagicMock, patch + + from pysus.api._impl.source import _instantiate_many + from pysus.api.client import _run_sync + + good = MagicMock(name="good") + with patch( + "pysus.api.extensions.ExtensionFactory.instantiate", + new=AsyncMock(side_effect=[OSError("boom"), good]), + ): + result = _run_sync(_instantiate_many(["bad", "good"])) + assert result == [good] + + +class TestInfoOutput: + def test_info_prints_table(self, capsys): + import pysus + + pysus.ftp.info() + out = capsys.readouterr().out + assert "SINAN" in out or "sinan" in out