From a4b7fc4529e097e9ecd9016e9a131cca122d4caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 17 Aug 2026 18:01:44 -0300 Subject: [PATCH 1/4] feat: OpenDataSUS catalog client and unified metadata layer - add pysus.api.saude client for dadosabertos.saude.gov.br: Next.js buildId discovery, catalog listing, full CKAN package metadata and resource downloads, with offline tests backed by captured fixtures - introduce the unified metadata architecture: MetadataBag with eight typed facets (identity, description, temporal, spatial, provenance, structure, access, quality), the MetadataExtractor protocol, and per-client extractors for FTP, DadosGov, DuckLake and Saude - wire .metadata/.ametadata() into every remote entity (clients, datasets, groups, files) via MetadataMixin; tabular local files compute structure metadata from their content - document both surfaces in docs/source/guides (saude.rst, metadata.rst) --- CHANGELOG.md | 5 + docs/source/guides/index.rst | 2 + docs/source/guides/metadata.rst | 102 +++ docs/source/guides/saude.rst | 102 +++ pysus/api/dadosgov/metadata.py | 132 ++++ pysus/api/dadosgov/models.py | 10 +- pysus/api/ducklake/metadata.py | 131 ++++ pysus/api/ducklake/models.py | 10 +- pysus/api/ftp/metadata.py | 126 +++ pysus/api/ftp/models.py | 6 +- pysus/api/metadata/__init__.py | 49 ++ pysus/api/metadata/extractors.py | 69 ++ pysus/api/metadata/models.py | 720 +++++++++++++++++- pysus/api/models.py | 96 ++- pysus/api/saude/__init__.py | 44 ++ pysus/api/saude/catalog.py | 321 ++++++++ pysus/api/saude/client.py | 231 ++++++ pysus/api/saude/download.py | 167 ++++ pysus/api/saude/errors.py | 29 + pysus/api/saude/metadata.py | 191 +++++ pysus/api/saude/next_data.py | 173 +++++ pysus/api/saude/resources.py | 255 +++++++ pysus/tests/api/metadata/test_bag.py | 287 +++++++ pysus/tests/api/metadata/test_base_classes.py | 233 ++++++ pysus/tests/api/metadata/test_extractors.py | 185 +++++ pysus/tests/api/saude/__init__.py | 6 + pysus/tests/api/saude/conftest.py | 104 +++ .../api/saude/fixtures/catalog_page1.json | 1 + .../fixtures/dataset_arboviroses-dengue.json | 1 + .../api/saude/fixtures/dengue_2024.csv.zip | Bin 0 -> 232 bytes pysus/tests/api/saude/fixtures/homepage.html | 1 + pysus/tests/api/saude/test_catalog.py | 132 ++++ pysus/tests/api/saude/test_client.py | 94 +++ pysus/tests/api/saude/test_download.py | 154 ++++ pysus/tests/api/saude/test_next_data.py | 158 ++++ pysus/tests/api/saude/test_resources.py | 139 ++++ 36 files changed, 4458 insertions(+), 8 deletions(-) create mode 100644 docs/source/guides/metadata.rst create mode 100644 docs/source/guides/saude.rst create mode 100644 pysus/api/dadosgov/metadata.py create mode 100644 pysus/api/ducklake/metadata.py create mode 100644 pysus/api/ftp/metadata.py create mode 100644 pysus/api/metadata/extractors.py create mode 100644 pysus/api/saude/__init__.py create mode 100644 pysus/api/saude/catalog.py create mode 100644 pysus/api/saude/client.py create mode 100644 pysus/api/saude/download.py create mode 100644 pysus/api/saude/errors.py create mode 100644 pysus/api/saude/metadata.py create mode 100644 pysus/api/saude/next_data.py create mode 100644 pysus/api/saude/resources.py create mode 100644 pysus/tests/api/metadata/test_bag.py create mode 100644 pysus/tests/api/metadata/test_base_classes.py create mode 100644 pysus/tests/api/metadata/test_extractors.py create mode 100644 pysus/tests/api/saude/__init__.py create mode 100644 pysus/tests/api/saude/conftest.py create mode 100644 pysus/tests/api/saude/fixtures/catalog_page1.json create mode 100644 pysus/tests/api/saude/fixtures/dataset_arboviroses-dengue.json create mode 100644 pysus/tests/api/saude/fixtures/dengue_2024.csv.zip create mode 100644 pysus/tests/api/saude/fixtures/homepage.html create mode 100644 pysus/tests/api/saude/test_catalog.py create mode 100644 pysus/tests/api/saude/test_client.py create mode 100644 pysus/tests/api/saude/test_download.py create mode 100644 pysus/tests/api/saude/test_next_data.py create mode 100644 pysus/tests/api/saude/test_resources.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5404ba..201676d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ Release Notes ## [2.9.0](https://github.com/AlertaDengue/PySUS/compare/2.8.0...2.9.0) (2026-08-17) +### Features + +* **metadata:** unified metadata layer (`MetadataBag`, `MetadataExtractor`) with per-client extractors for FTP, DadosGov, DuckLake and Saude; every remote entity now exposes a merged `.metadata` bag +* **saude:** add OpenDataSUS catalog client (`pysus.api.saude`) with Next.js buildId discovery, catalog listing, full CKAN package metadata and resource downloads (stage 1 of the dadosabertos.saude.gov.br integration) + ## [2.8.0](https://github.com/AlertaDengue/PySUS/compare/2.7.0...2.8.0) (2026-08-14) ## [2.7.0](https://github.com/AlertaDengue/PySUS/compare/2.6.5...2.7.0) (2026-07-13) diff --git a/docs/source/guides/index.rst b/docs/source/guides/index.rst index b2d58ac3..d5551383 100644 --- a/docs/source/guides/index.rst +++ b/docs/source/guides/index.rst @@ -8,7 +8,9 @@ Guides pysus-orchestrator ftp dadosgov + saude ducklake + metadata files-and-formats datasets web-ui diff --git a/docs/source/guides/metadata.rst b/docs/source/guides/metadata.rst new file mode 100644 index 00000000..ce317f18 --- /dev/null +++ b/docs/source/guides/metadata.rst @@ -0,0 +1,102 @@ +======== +Metadata +======== + +Every remote entity in PySUS — clients, datasets, groups and files — +exposes a unified metadata surface via the ``.metadata`` property: + +.. code-block:: python + + from pysus.api.saude import SaudeClient + + async with SaudeClient() as c: + pkg = await c.fetch_dataset("arboviroses-dengue") + # pkg.metadata is a MetadataBag + +The MetadataBag +--------------- + +:class:`pysus.api.metadata.models.MetadataBag` organizes metadata into +eight typed facets: + +- ``identity`` — name, slug, aliases, and ``cross_origin_id`` (the + shared CKAN UUID that links the same dataset across the + dados.gov.br and dadosabertos.saude.gov.br portals); +- ``description`` — title, long name, description, tags, themes; +- ``temporal`` — created/modified timestamps, periodicity, year/month; +- ``spatial`` — geographic scope, UF list, municipalities, state; +- ``provenance`` — origin, organization, author, contact, license; +- ``structure`` — columns, row count, file count, format; +- ``access`` — URL, format, size, auth requirement, policy; +- ``quality`` — content fingerprint, integrity, freshness. + +Every bag serializes to JSON (``bag.to_dict()`` / +``MetadataBag.from_dict(...)``) so it can be persisted in the DuckLake +catalogs or snapshot files. + +Extractors +---------- + +Each client ships its own extractors, one per entity type +(:class:`~pysus.api.metadata.extractors.MetadataExtractor`): + ++-------------+------------------------------------------+ +| Client | Extractors | ++=============+==========================================+ +| FTP | ``FtpDatasetExtractor``, | +| | ``FtpGroupExtractor``, | +| | ``FtpFileExtractor`` | ++-------------+------------------------------------------+ +| DadosGov | ``DadosGovDatasetExtractor``, | +| | ``DadosGovGroupExtractor``, | +| | ``DadosGovFileExtractor`` | ++-------------+------------------------------------------+ +| DuckLake | ``DuckLakeDatasetExtractor``, | +| | ``DuckLakeGroupExtractor``, | +| | ``DuckLakeFileExtractor`` | ++-------------+------------------------------------------+ +| Saude | ``SaudeDatasetExtractor``, | +| | ``SaudeGroupExtractor``, | +| | ``SaudeFileExtractor`` | ++-------------+------------------------------------------+ + +Concrete model classes declare their extractors via the +``extractor_types`` class attribute; the base classes only know that +*some* extractor exists: + +.. code-block:: python + + file = await some_dataset.search(name="DENGBR25.csv.zip")[0] + bag = file.metadata # merged bag, cached + print(bag.temporal.year) # 2025 (from the filename formatter) + print(bag.access.size_bytes) + +Merging across origins +---------------------- + +:func:`~pysus.api.metadata.models.merge_bags` combines bags from +different origins with a documented per-facet precedence +(``roadmap_saude.md`` §1.7): Saude wins for descriptive fields, +DuckLake for structure and content fingerprints, and so on: + +.. code-block:: python + + from pysus.api.metadata.models import merge_bags + + merged = merge_bags([ftp_file.metadata, saude_file.metadata]) + + # Descriptive metadata comes from Saude, structure from DuckLake: + print(merged.description.title) + print(merged.structure.row_count) + +Local files +----------- + +Tabular local files (Parquet, CSV, DBF, ...) also expose ``.metadata`` +computed from their content — columns, row count and size — without +any network access: + +.. code-block:: python + + local = await file.download() + print(local.metadata.structure.columns) diff --git a/docs/source/guides/saude.rst b/docs/source/guides/saude.rst new file mode 100644 index 00000000..d71237f1 --- /dev/null +++ b/docs/source/guides/saude.rst @@ -0,0 +1,102 @@ +============== +OpenDataSUS +============== + +:class:`pysus.api.saude.client.SaudeClient` talks to the portal of the +Brazilian Ministry of Health — `dadosabertos.saude.gov.br +`_ — a Next.js frontend over a CKAN +backend. It exposes the catalog of 138 health datasets, their full CKAN +metadata, and the resource (file) downloads. + +.. note:: + This is the *catalog* client (Stage 1 of ``roadmap_saude.md``). The + structured DEMAS REST API (``apidadosabertos.saude.gov.br``) and the + DuckLake sync integration ship in later stages. + +No token required +----------------- + +The portal is public — no authentication header is needed. Instantiate +the client and use it as an async context manager: + +.. code-block:: python + + import asyncio + from pysus.api.saude import SaudeClient + + async def main(): + async with SaudeClient() as client: + ... + +Listing datasets +---------------- + +.. code-block:: python + + # One page (20 entries) of the catalog + page = await client.list_datasets(group="arboviroses") + + # All pages, lazily + async for entry in client.iter_datasets(group="arboviroses"): + print(entry.name, entry.title) + + # Filters: q (text), group, tag, fmt (resource format) + page = await client.list_datasets(q="dengue", fmt="CSV") + +Groups and tags +--------------- + +.. code-block:: python + + groups = await client.list_groups() # 14 themes + tags = await client.list_tags() + +Fetching full metadata +---------------------- + +.. code-block:: python + + package = await client.fetch_dataset("arboviroses-dengue") + print(package.title) # "Sinan/Dengue" + print(package.id) # CKAN UUID (shared with dados.gov.br) + print(package.license_title) # "Creative Commons Atribuição" + print(package.periodicity) # "Semanal" (from extras[]) + print(package.contact) # "arboviroses@saude.gov.br" + + resources = package.resources # 19 fields each + for res in resources[:5]: + print(res.name, res.format, res.size) + +Downloading resources +--------------------- + +.. code-block:: python + + # All CSV resources of the dengue dataset + paths = await client.download_dataset( + "arboviroses-dengue", fmt="CSV", dest_dir="./data" + ) + + # A single resource, by id or name + path = await client.download_resource( + "arboviroses-dengue", name="Dengue - 2024", dest_dir="./data" + ) + +Resources with format ``API`` are documentation links and are skipped +automatically. + +Caching +------- + +The portal's Next.js ``buildId`` and every catalog response are cached +on disk (default TTL 24 h). Override with ``cache_dir`` and +``cache_ttl``: + +.. code-block:: python + + from datetime import timedelta + + client = SaudeClient( + cache_dir="./.cache/saude", + cache_ttl=timedelta(hours=6), + ) diff --git a/pysus/api/dadosgov/metadata.py b/pysus/api/dadosgov/metadata.py new file mode 100644 index 00000000..416c36fd --- /dev/null +++ b/pysus/api/dadosgov/metadata.py @@ -0,0 +1,132 @@ +"""Metadata extractors for the DadosGov (dados.gov.br) client. + +Consumes the client's own models (``File``, ``Group``, ``Dataset`` in +:mod:`pysus.api.dadosgov.models`) and produces +:class:`~pysus.api.metadata.models.MetadataBag` instances. + +The filename-derived ``year`` / ``month`` / ``state`` metadata (from +the per-dataset formatters) maps into the temporal and spatial facets. +""" + +from __future__ import annotations + +from typing import Any + +from pysus.api.metadata.extractors import MetadataExtractor +from pysus.api.metadata.models import ( + AccessFacet, + DescriptionFacet, + IdentityFacet, + MetadataBag, + ProvenanceFacet, + SpatialFacet, + StructureFacet, + TemporalFacet, +) + + +class DadosGovDatasetExtractor(MetadataExtractor): + """Build a dataset-level bag from a DadosGov ``Dataset``.""" + + origin = "dadosgov" + + def supported_facets(self) -> set[str]: + return {"identity", "description", "provenance"} + + def _extract(self, obj: Any) -> MetadataBag: + dataset = obj + return MetadataBag( + identity=IdentityFacet( + name=dataset.name, + slug=getattr(dataset, "slug", "") or dataset.name, + ), + description=DescriptionFacet( + title=dataset.long_name, + long_name=dataset.long_name, + description=getattr(dataset, "description", "") or "", + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +class DadosGovGroupExtractor(MetadataExtractor): + """Build a group-level bag from a DadosGov ``Group``.""" + + origin = "dadosgov" + + def supported_facets(self) -> set[str]: + return {"identity", "description", "provenance"} + + def _extract(self, obj: Any) -> MetadataBag: + group = obj + return MetadataBag( + identity=IdentityFacet( + name=group.name, + slug=getattr(group, "slug", "") or group.name, + ), + description=DescriptionFacet( + title=group.long_name, + long_name=group.long_name, + description=getattr(group, "description", "") or "", + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +class DadosGovFileExtractor(MetadataExtractor): + """Build a file-level bag from a DadosGov ``File``.""" + + origin = "dadosgov" + + def supported_facets(self) -> set[str]: + return { + "identity", + "temporal", + "spatial", + "access", + "structure", + "provenance", + } + + def _extract(self, obj: Any) -> MetadataBag: + file = obj + modified = None + try: + modified = file.modify + except (ValueError, AttributeError): + modified = None + state = None + try: + state = file.state + except (ValueError, AttributeError): + state = None + return MetadataBag( + identity=IdentityFacet( + name=file.basename, + slug=file.basename, + ), + temporal=TemporalFacet( + modified=modified, + year=file.year, + month=file.month, + ), + spatial=SpatialFacet(state=state), + structure=StructureFacet( + format=(file.extension or "").lstrip("."), + ), + access=AccessFacet( + url=str(file.path), + format=(file.extension or "").lstrip("."), + size_bytes=file.size or 0, + download_strategy="http", + requires_auth=True, + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +__all__ = [ + "DadosGovDatasetExtractor", + "DadosGovFileExtractor", + "DadosGovGroupExtractor", +] diff --git a/pysus/api/dadosgov/models.py b/pysus/api/dadosgov/models.py index dbf12d01..7d4bc89f 100644 --- a/pysus/api/dadosgov/models.py +++ b/pysus/api/dadosgov/models.py @@ -6,7 +6,7 @@ from abc import abstractmethod from collections.abc import Callable from datetime import datetime -from typing import Any +from typing import Any, ClassVar import httpx from dateparser import parse # type: ignore[import-untyped] @@ -16,6 +16,11 @@ from pysus.api.types import State from .client import ConjuntoDados, DadosGov, Recurso +from .metadata import ( + DadosGovDatasetExtractor, + DadosGovFileExtractor, + DadosGovGroupExtractor, +) _FORMAT_RE = re.compile(r"[._](csv|json|xml)(\.zip)?$", re.IGNORECASE) @@ -56,6 +61,7 @@ class File(BaseRemoteFile): record: Recurso type: str = "File" + extractor_types: ClassVar[list] = [DadosGovFileExtractor] _metadata: dict[str, Any] = PrivateAttr(default_factory=dict) def __init__(self, **data): @@ -249,6 +255,7 @@ class Group(BaseRemoteGroup): """A group of files within a dataset.""" record: ConjuntoDados + extractor_types: ClassVar[list] = [DadosGovGroupExtractor] _formatter: Callable[[str], dict[str, Any]] | None = PrivateAttr( default=None ) @@ -359,6 +366,7 @@ class Dataset(BaseRemoteDataset): ids: list[str] = [] client: DadosGov group_aliases: dict[str, str] = {} + extractor_types: ClassVar[list] = [DadosGovDatasetExtractor] def __repr__(self): """Return the dataset name as its string representation.""" diff --git a/pysus/api/ducklake/metadata.py b/pysus/api/ducklake/metadata.py new file mode 100644 index 00000000..2c145c74 --- /dev/null +++ b/pysus/api/ducklake/metadata.py @@ -0,0 +1,131 @@ +"""Metadata extractors for the DuckLake (S3 catalog) client. + +Consumes the client's own models (``File``, ``DuckGroup``, +``DuckDataset`` in :mod:`pysus.api.ducklake.models`) and produces +:class:`~pysus.api.metadata.models.MetadataBag` instances. + +DuckLake holds the authoritative parquet artifacts: row counts, the +``sha256`` content digest and the post-ETL schema live in the catalog +rows and map into the structure and quality facets. +""" + +from __future__ import annotations + +from typing import Any + +from pysus.api.metadata.extractors import MetadataExtractor +from pysus.api.metadata.models import ( + AccessFacet, + DescriptionFacet, + IdentityFacet, + MetadataBag, + ProvenanceFacet, + QualityFacet, + SpatialFacet, + StructureFacet, + TemporalFacet, +) + + +class DuckLakeDatasetExtractor(MetadataExtractor): + """Build a dataset-level bag from a ``DuckDataset``.""" + + origin = "ducklake" + + def supported_facets(self) -> set[str]: + return {"identity", "description", "provenance"} + + def _extract(self, obj: Any) -> MetadataBag: + dataset = obj + return MetadataBag( + identity=IdentityFacet(name=dataset.name), + description=DescriptionFacet( + title=dataset.long_name, + long_name=dataset.long_name, + description=getattr(dataset, "description", "") or "", + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +class DuckLakeGroupExtractor(MetadataExtractor): + """Build a group-level bag from a ``DuckGroup``.""" + + origin = "ducklake" + + def supported_facets(self) -> set[str]: + return {"identity", "description", "provenance"} + + def _extract(self, obj: Any) -> MetadataBag: + group = obj + return MetadataBag( + identity=IdentityFacet(name=group.name), + description=DescriptionFacet( + title=group.long_name, + long_name=group.long_name, + description=getattr(group, "description", "") or "", + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +class DuckLakeFileExtractor(MetadataExtractor): + """Build a file-level bag from a DuckLake ``File`` (catalog row).""" + + origin = "ducklake" + + def supported_facets(self) -> set[str]: + return { + "identity", + "temporal", + "spatial", + "structure", + "access", + "quality", + "provenance", + } + + def _extract(self, obj: Any) -> MetadataBag: + file = obj + record = file.record + sha = getattr(record, "sha256", None) or "" + modified = None + try: + modified = file.modify + except (ValueError, AttributeError): + modified = getattr(record, "modified", None) + state = getattr(record, "state", None) + return MetadataBag( + identity=IdentityFacet( + name=file.basename, + slug=file.basename, + ), + temporal=TemporalFacet( + modified=modified, + year=getattr(record, "year", None), + month=getattr(record, "month", None), + ), + spatial=SpatialFacet(state=state), + structure=StructureFacet( + row_count=getattr(record, "rows", 0) or 0, + format=getattr(record, "type", None) or "parquet", + ), + access=AccessFacet( + url=str(file.path), + format="parquet", + size_bytes=file.size or 0, + download_strategy="s3", + ), + quality=QualityFacet( + content_fingerprint=sha, + integrity_verified=bool(sha), + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +__all__ = [ + "DuckLakeDatasetExtractor", + "DuckLakeFileExtractor", + "DuckLakeGroupExtractor", +] diff --git a/pysus/api/ducklake/models.py b/pysus/api/ducklake/models.py index b0f3dde7..71ba8bb9 100644 --- a/pysus/api/ducklake/models.py +++ b/pysus/api/ducklake/models.py @@ -8,7 +8,7 @@ from collections.abc import Callable from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from anyio import to_thread from pydantic import Field, PrivateAttr @@ -20,6 +20,11 @@ from .catalog.orm.dataset import File as CatalogFile from .catalog.orm.dataset import Group from .catalog.orm.default import Dataset +from .metadata import ( + DuckLakeDatasetExtractor, + DuckLakeFileExtractor, + DuckLakeGroupExtractor, +) if TYPE_CHECKING: from .client import DuckLake @@ -28,6 +33,7 @@ class File(BaseRemoteFile): group: Optional["DuckGroup"] = Field(default=None, exclude=True) + extractor_types: ClassVar[list] = [DuckLakeFileExtractor] _record: CatalogFile = PrivateAttr() def __init__(self, **data: Any) -> None: @@ -99,6 +105,7 @@ class DuckDataset(BaseRemoteDataset): client: "DuckLake" = Field(exclude=True) border: Any = Field(exclude=True) update_on_close: bool = Field(default=False, exclude=True) + extractor_types: ClassVar[list] = [DuckLakeDatasetExtractor] def __init__(self, **data) -> None: if "adapter" in data and "border" not in data: @@ -233,6 +240,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: class DuckGroup(BaseRemoteGroup): record: Group = Field(exclude=True) dataset: DuckDataset = Field(exclude=True) + extractor_types: ClassVar[list] = [DuckLakeGroupExtractor] def __str__(self) -> str: return self.name diff --git a/pysus/api/ftp/metadata.py b/pysus/api/ftp/metadata.py new file mode 100644 index 00000000..7228b048 --- /dev/null +++ b/pysus/api/ftp/metadata.py @@ -0,0 +1,126 @@ +"""Metadata extractors for the FTP (DATASUS) client. + +Consumes the client's own models (``File``, ``Group``, ``Dataset`` in +:mod:`pysus.api.ftp.models`) and produces +:class:`~pysus.api.metadata.models.MetadataBag` instances. + +The LIST-parser-derived size/modify timestamps and the per-dataset +formatter output (year/month/state/group) map into the temporal, +spatial and identity facets. +""" + +from __future__ import annotations + +from typing import Any + +from pysus.api.metadata.extractors import MetadataExtractor +from pysus.api.metadata.models import ( + AccessFacet, + DescriptionFacet, + IdentityFacet, + MetadataBag, + ProvenanceFacet, + SpatialFacet, + StructureFacet, + TemporalFacet, +) + + +class FtpDatasetExtractor(MetadataExtractor): + """Build a dataset-level bag from an FTP ``Dataset``.""" + + origin = "ftp" + + def supported_facets(self) -> set[str]: + return {"identity", "description", "provenance"} + + def _extract(self, obj: Any) -> MetadataBag: + dataset = obj + return MetadataBag( + identity=IdentityFacet(name=dataset.name), + description=DescriptionFacet( + title=dataset.long_name, + long_name=dataset.long_name, + description=getattr(dataset, "description", "") or "", + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +class FtpGroupExtractor(MetadataExtractor): + """Build a group-level bag from an FTP ``Group``.""" + + origin = "ftp" + + def supported_facets(self) -> set[str]: + return {"identity", "description", "provenance"} + + def _extract(self, obj: Any) -> MetadataBag: + group = obj + return MetadataBag( + identity=IdentityFacet(name=group.name), + description=DescriptionFacet( + title=group.long_name, + long_name=group.long_name, + description=getattr(group, "description", "") or "", + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +class FtpFileExtractor(MetadataExtractor): + """Build a file-level bag from an FTP ``File``.""" + + origin = "ftp" + + def supported_facets(self) -> set[str]: + return { + "identity", + "temporal", + "spatial", + "access", + "structure", + "provenance", + } + + def _extract(self, obj: Any) -> MetadataBag: + file = obj + modified = None + try: + modified = file.modify + except (ValueError, AttributeError): + modified = None + state = None + try: + state = file.state + except (ValueError, AttributeError): + state = None + return MetadataBag( + identity=IdentityFacet( + name=file.basename, + slug=file.basename, + ), + temporal=TemporalFacet( + modified=modified, + year=file.year, + month=file.month, + ), + spatial=SpatialFacet(state=state), + structure=StructureFacet( + format=(file.extension or "").lstrip("."), + ), + access=AccessFacet( + url=str(file.path), + format=(file.extension or "").lstrip("."), + size_bytes=file.size or 0, + download_strategy="ftp", + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +__all__ = [ + "FtpDatasetExtractor", + "FtpFileExtractor", + "FtpGroupExtractor", +] diff --git a/pysus/api/ftp/models.py b/pysus/api/ftp/models.py index a0ef901a..085a4e84 100644 --- a/pysus/api/ftp/models.py +++ b/pysus/api/ftp/models.py @@ -7,7 +7,7 @@ from collections.abc import Callable, Sequence from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, ClassVar from pydantic import PrivateAttr from pysus import CACHEPATH @@ -20,11 +20,13 @@ from pysus.api.types import State from .client import FTP, FTPFileInfo +from .metadata import FtpDatasetExtractor, FtpFileExtractor, FtpGroupExtractor class File(BaseRemoteFile): """A single file on the DATASUS FTP server with parsed metadata.""" + extractor_types: ClassVar[list] = [FtpFileExtractor] _info: FTPFileInfo = PrivateAttr() def __init__(self, **data): @@ -265,6 +267,7 @@ def __repr__(self) -> str: class Group(BaseRemoteGroup): """A group of related files within a dataset (e.g. all files of a type).""" + extractor_types: ClassVar[list] = [FtpGroupExtractor] path: str _name: str = PrivateAttr() _long_name: str = PrivateAttr() @@ -364,6 +367,7 @@ async def _fetch_files(self) -> list[BaseRemoteFile]: class Dataset(BaseRemoteDataset, ABC): """Abstract base for a DATASUS dataset, providing file discovery via FTP.""" + extractor_types: ClassVar[list] = [FtpDatasetExtractor] paths: list[Directory] = [] group_definitions: dict[str, str] = {} diff --git a/pysus/api/metadata/__init__.py b/pysus/api/metadata/__init__.py index e69de29b..5baae606 100644 --- a/pysus/api/metadata/__init__.py +++ b/pysus/api/metadata/__init__.py @@ -0,0 +1,49 @@ +"""Unified metadata layer: facets, bags, merge and extractors. + +See ``roadmap_saude.md`` §1 for the design. Public API: + +- :class:`~pysus.api.metadata.models.MetadataBag` — the canonical + metadata container (eight facets + ``raw``). +- :func:`~pysus.api.metadata.models.merge_bags` — cross-origin merge + with per-facet precedence. +- :class:`~pysus.api.metadata.extractors.MetadataExtractor` — the + protocol every client implements. +""" + +from .extractors import MetadataExtractor +from .models import ( + AccessFacet, + Column, + Dataset, + DatasetGroup, + DescriptionFacet, + File, + FileMeta, + IdentityFacet, + MetadataBag, + ProvenanceFacet, + QualityFacet, + SpatialFacet, + StructureFacet, + TemporalFacet, + merge_bags, +) + +__all__ = [ + "AccessFacet", + "Column", + "Dataset", + "DatasetGroup", + "DescriptionFacet", + "File", + "FileMeta", + "IdentityFacet", + "MetadataBag", + "MetadataExtractor", + "ProvenanceFacet", + "QualityFacet", + "SpatialFacet", + "StructureFacet", + "TemporalFacet", + "merge_bags", +] diff --git a/pysus/api/metadata/extractors.py b/pysus/api/metadata/extractors.py new file mode 100644 index 00000000..07fcf66e --- /dev/null +++ b/pysus/api/metadata/extractors.py @@ -0,0 +1,69 @@ +"""The ``MetadataExtractor`` protocol and registry helpers. + +One extractor implementation exists per (origin, entity_type) pair: + +- FTP: `FtpDatasetExtractor`, `FtpGroupExtractor`, `FtpFileExtractor` +- DadosGov: `DadosGovDatasetExtractor`, `DadosGovGroupExtractor`, + `DadosGovFileExtractor` +- DuckLake: `DuckLakeDatasetExtractor`, `DuckLakeGroupExtractor`, + `DuckLakeFileExtractor` +- Saude: `SaudeDatasetExtractor`, `SaudeGroupExtractor`, + `SaudeFileExtractor` + +The base classes in :mod:`pysus.api.models` only know that *some* +extractors are registered on an instance; they never hard-code which +one. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + +from .models import MetadataBag + + +class MetadataExtractor(ABC): + """Stateless transformer: a concrete entity → :class:`MetadataBag`. + + Extractor implementations are specific to one client and one + entity type (dataset / group / file). They read whatever they + need from the passed object and return a bag carrying only the + facets they can populate — the ``provenance.origin`` field must + always be set so :func:`~pysus.api.metadata.models.merge_bags` + can apply the precedence rules. + """ + + #: The origin label (``"saude"``, ``"ftp"``, ``"dadosgov"``, + #: ``"ducklake"``). Subclasses override it. + origin: str = "" + + def extract(self, obj: Any) -> MetadataBag: + """Synchronous fast path — uses only data already on ``obj``. + + Subclasses implement :meth:`extract`; the default + :meth:`aextract` just delegates to it. Extractors that need + network or local-file IO override :meth:`aextract` instead. + """ + bag = self._extract(obj) + if not bag.provenance.origin: + bag.provenance.origin = self.origin + return bag + + async def aextract(self, obj: Any) -> MetadataBag: + """Async path — may hit the network or read local files. + + The default implementation delegates to :meth:`extract`. + """ + return self.extract(obj) + + @abstractmethod + def _extract(self, obj: Any) -> MetadataBag: + """Build the bag from ``obj``. Must set ``provenance.origin``.""" + + def supported_facets(self) -> set[str]: + """Return the subset of facet names this extractor populates.""" + return set() + + +__all__ = ["MetadataExtractor"] diff --git a/pysus/api/metadata/models.py b/pysus/api/metadata/models.py index 44bd44c4..4e26165b 100644 --- a/pysus/api/metadata/models.py +++ b/pysus/api/metadata/models.py @@ -1,10 +1,35 @@ +"""Metadata data models: facets, bags and merge logic. + +The metadata layer is origin-agnostic. Every client (FTP, DadosGov, +DuckLake, Saude) extracts metadata from a different source, but the +shape that flows through PySUS is one: a :class:`MetadataBag` with +eight typed facets (identity, description, temporal, spatial, +provenance, structure, access, quality) plus a ``raw`` dict that +preserves unmapped fields. + +The legacy dataclasses (:class:`Dataset`, :class:`DatasetGroup`, +:class:`FileMeta`, :class:`File`, :class:`Column`) remain in this +module for backwards compatibility; new code should use +:class:`MetadataBag`. +""" + +from __future__ import annotations + from dataclasses import dataclass, field +from datetime import datetime +from typing import Any from pysus.api.types import ColumnType, Origin +# ---------------------------------------------------------------------- +# Legacy dataclasses (kept for backwards compatibility) +# ---------------------------------------------------------------------- + @dataclass class Dataset: + """Legacy dataset metadata record.""" + name: str long_name: str description: str @@ -12,6 +37,8 @@ class Dataset: @dataclass class DatasetGroup: + """Legacy group metadata record.""" + name: str long_name: str description: str @@ -19,6 +46,8 @@ class DatasetGroup: @dataclass class FileMeta: + """Legacy file metadata record.""" + name: str path: str size: int @@ -32,15 +61,19 @@ class FileMeta: @dataclass class File: + """Legacy file record.""" + origin: Origin dataset: Dataset | None = None group: DatasetGroup | None = None - columns: list["Column"] = field(default_factory=list) + columns: list[Column] = field(default_factory=list) _meta: FileMeta | None = None @dataclass class Column: + """A column definition: name, description and dtype.""" + name: str description: str dtype: ColumnType @@ -48,10 +81,693 @@ class Column: @classmethod def from_schema( cls, name: str, dtype: ColumnType, description: str = "" - ) -> "Column": + ) -> Column: """Create a Column with a description provided from the database.""" return cls( name=name, description=description, dtype=dtype, ) + + def to_dict(self) -> dict[str, str]: + """Serialize the column.""" + return { + "name": self.name, + "description": self.description, + "dtype": self.dtype, + } + + @classmethod + def from_dict(cls, data: dict[str, str]) -> Column: + """Rehydrate a column from a serialized dict.""" + return cls( + name=data["name"], + description=data.get("description", ""), + dtype=data["dtype"], + ) + + +# ---------------------------------------------------------------------- +# Facets +# ---------------------------------------------------------------------- + +#: Geographic scope levels, from least to most specific. Used to merge +#: spatial facets (never widen coverage). +SCOPE_RANK = { + "national": 0, + "regional": 1, + "state": 2, + "municipal": 3, + "local": 4, +} + +#: Origin precedence for descriptive metadata (titles, descriptions). +#: Saude is the curated source; DuckLake is a mirror. +DESCRIPTIVE_PRECEDENCE = ("saude", "dadosgov", "ftp", "ducklake") + +#: Origin precedence for content/structure metadata. DuckLake holds +#: the authoritative parquet schema and row counts. +STRUCTURE_PRECEDENCE = ("ducklake", "saude", "dadosgov", "ftp") + +#: Origin precedence for temporal.modified (upload time accuracy). +MODIFIED_PRECEDENCE = ("ducklake", "ftp", "dadosgov", "saude") + + +@dataclass +class IdentityFacet: + """Names and identifiers of the entity. + + ``cross_origin_id`` is the shared CKAN UUID that links the same + dataset across the dados.gov.br and dadosabertos.saude.gov.br + portals. + """ + + name: str = "" + slug: str = "" + aliases: list[str] = field(default_factory=list) + cross_origin_id: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "slug": self.slug, + "aliases": list(self.aliases), + "cross_origin_id": self.cross_origin_id, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> IdentityFacet: + return cls( + name=data.get("name", ""), + slug=data.get("slug", ""), + aliases=list(data.get("aliases", [])), + cross_origin_id=data.get("cross_origin_id", ""), + ) + + +@dataclass +class DescriptionFacet: + """Human-readable title, description, tags and themes.""" + + title: str = "" + long_name: str = "" + description: str = "" + tags: list[str] = field(default_factory=list) + themes: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "title": self.title, + "long_name": self.long_name, + "description": self.description, + "tags": list(self.tags), + "themes": list(self.themes), + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DescriptionFacet: + return cls( + title=data.get("title", ""), + long_name=data.get("long_name", ""), + description=data.get("description", ""), + tags=list(data.get("tags", [])), + themes=list(data.get("themes", [])), + ) + + +@dataclass +class TemporalFacet: + """Temporal coverage and freshness.""" + + created: datetime | None = None + modified: datetime | None = None + periodicity: str | None = None + valid_from: datetime | None = None + valid_to: datetime | None = None + year: int | None = None + month: int | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "created": self.created.isoformat() if self.created else None, + "modified": self.modified.isoformat() if self.modified else None, + "periodicity": self.periodicity, + "valid_from": ( + self.valid_from.isoformat() if self.valid_from else None + ), + "valid_to": (self.valid_to.isoformat() if self.valid_to else None), + "year": self.year, + "month": self.month, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> TemporalFacet: + def _parse(value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(str(value)) + except ValueError: + return None + + return cls( + created=_parse(data.get("created")), + modified=_parse(data.get("modified")), + periodicity=data.get("periodicity"), + valid_from=_parse(data.get("valid_from")), + valid_to=_parse(data.get("valid_to")), + year=data.get("year"), + month=data.get("month"), + ) + + +@dataclass +class SpatialFacet: + """Geographic coverage.""" + + geographic_scope: str = "national" + ufs: list[str] = field(default_factory=list) + municipalities: list[str] = field(default_factory=list) + state: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "geographic_scope": self.geographic_scope, + "ufs": list(self.ufs), + "municipalities": list(self.municipalities), + "state": self.state, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> SpatialFacet: + return cls( + geographic_scope=data.get("geographic_scope", "national"), + ufs=list(data.get("ufs", [])), + municipalities=list(data.get("municipalities", [])), + state=data.get("state"), + ) + + +@dataclass +class ProvenanceFacet: + """Where the data came from and how it may be used.""" + + origin: str = "" + organization: str = "" + author: str = "" + maintainer: str = "" + contact: str = "" + license: str = "" + license_id: str = "" + attribution: str = "" + source_url: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "origin": self.origin, + "organization": self.organization, + "author": self.author, + "maintainer": self.maintainer, + "contact": self.contact, + "license": self.license, + "license_id": self.license_id, + "attribution": self.attribution, + "source_url": self.source_url, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> ProvenanceFacet: + return cls( + origin=data.get("origin", ""), + organization=data.get("organization", ""), + author=data.get("author", ""), + maintainer=data.get("maintainer", ""), + contact=data.get("contact", ""), + license=data.get("license", ""), + license_id=data.get("license_id", ""), + attribution=data.get("attribution", ""), + source_url=data.get("source_url", ""), + ) + + +@dataclass +class StructureFacet: + """Schema and volume.""" + + columns: list[Column] = field(default_factory=list) + row_count: int = 0 + file_count: int = 0 + format: str = "" + schema_fingerprint: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "columns": [c.to_dict() for c in self.columns], + "row_count": self.row_count, + "file_count": self.file_count, + "format": self.format, + "schema_fingerprint": self.schema_fingerprint, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> StructureFacet: + return cls( + columns=[Column.from_dict(c) for c in data.get("columns", [])], + row_count=int(data.get("row_count", 0) or 0), + file_count=int(data.get("file_count", 0) or 0), + format=data.get("format", ""), + schema_fingerprint=data.get("schema_fingerprint", ""), + ) + + +@dataclass +class AccessFacet: + """How to reach the data.""" + + url: str = "" + format: str = "" + size_bytes: int = 0 + download_strategy: str = "" + requires_auth: bool = False + policy: str = "" + mime_type: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "url": self.url, + "format": self.format, + "size_bytes": self.size_bytes, + "download_strategy": self.download_strategy, + "requires_auth": self.requires_auth, + "policy": self.policy, + "mime_type": self.mime_type, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> AccessFacet: + return cls( + url=data.get("url", ""), + format=data.get("format", ""), + size_bytes=int(data.get("size_bytes", 0) or 0), + download_strategy=data.get("download_strategy", ""), + requires_auth=bool(data.get("requires_auth", False)), + policy=data.get("policy", ""), + mime_type=data.get("mime_type", ""), + ) + + +@dataclass +class QualityFacet: + """Integrity and completeness signals.""" + + freshness_score: float | None = None + integrity_verified: bool = False + content_fingerprint: str = "" + completeness_pct: float | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "freshness_score": self.freshness_score, + "integrity_verified": self.integrity_verified, + "content_fingerprint": self.content_fingerprint, + "completeness_pct": self.completeness_pct, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> QualityFacet: + return cls( + freshness_score=data.get("freshness_score"), + integrity_verified=bool(data.get("integrity_verified", False)), + content_fingerprint=data.get("content_fingerprint", ""), + completeness_pct=data.get("completeness_pct"), + ) + + +# ---------------------------------------------------------------------- +# MetadataBag +# ---------------------------------------------------------------------- + + +@dataclass +class MetadataBag: + """The canonical metadata container. + + One bag per entity (client, dataset, group, file). Every facet has + a documented merge rule (see :func:`merge_bags`) so bags from + different origins can be combined without losing information. + """ + + identity: IdentityFacet = field(default_factory=IdentityFacet) + description: DescriptionFacet = field(default_factory=DescriptionFacet) + temporal: TemporalFacet = field(default_factory=TemporalFacet) + spatial: SpatialFacet = field(default_factory=SpatialFacet) + provenance: ProvenanceFacet = field(default_factory=ProvenanceFacet) + structure: StructureFacet = field(default_factory=StructureFacet) + access: AccessFacet = field(default_factory=AccessFacet) + quality: QualityFacet = field(default_factory=QualityFacet) + raw: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + """Serialize the bag to JSON-compatible dicts.""" + return { + "identity": self.identity.to_dict(), + "description": self.description.to_dict(), + "temporal": self.temporal.to_dict(), + "spatial": self.spatial.to_dict(), + "provenance": self.provenance.to_dict(), + "structure": self.structure.to_dict(), + "access": self.access.to_dict(), + "quality": self.quality.to_dict(), + "raw": self.raw, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> MetadataBag: + """Rehydrate a bag from a serialized dict.""" + return cls( + identity=IdentityFacet.from_dict(data.get("identity", {})), + description=DescriptionFacet.from_dict(data.get("description", {})), + temporal=TemporalFacet.from_dict(data.get("temporal", {})), + spatial=SpatialFacet.from_dict(data.get("spatial", {})), + provenance=ProvenanceFacet.from_dict(data.get("provenance", {})), + structure=StructureFacet.from_dict(data.get("structure", {})), + access=AccessFacet.from_dict(data.get("access", {})), + quality=QualityFacet.from_dict(data.get("quality", {})), + raw=dict(data.get("raw", {})), + ) + + def merge( + self, + other: MetadataBag, + *, + descriptive_precedence: tuple[str, ...] = DESCRIPTIVE_PRECEDENCE, + structure_precedence: tuple[str, ...] = STRUCTURE_PRECEDENCE, + modified_precedence: tuple[str, ...] = MODIFIED_PRECEDENCE, + ) -> MetadataBag: + """Return a new bag with ``other`` merged into ``self``. + + Per-facet rules (roadmap_saude.md §1.7): + + - identity: first non-empty field; aliases union + - description: winner by ``descriptive_precedence``; tags union + - temporal.created: earliest; temporal.modified: winner by + ``modified_precedence``; year/month first non-None + - spatial: most specific scope; ufs/municipalities union + - provenance: first non-empty per field; license most + permissive; origins joined with ``/`` + - structure: columns winner by ``structure_precedence``; + row/file counts max + - access: url/size first non-zero; ``requires_auth`` OR + - quality: fingerprint first non-empty; integrity OR; freshness + max + """ + return merge_bags( + [self, other], + descriptive_precedence=descriptive_precedence, + structure_precedence=structure_precedence, + modified_precedence=modified_precedence, + ) + + +# ---------------------------------------------------------------------- +# Merge helpers +# ---------------------------------------------------------------------- + + +def _origin_of(bag: MetadataBag) -> str: + return (bag.provenance.origin or "").strip().lower() + + +def _first(*values: str) -> str: + for value in values: + if value: + return str(value) + return "" + + +def _pick_by_precedence( + bags: list[MetadataBag], + getter, + precedence: tuple[str, ...], +) -> MetadataBag | None: + """Return the bag that wins for a facet, or None if all empty.""" + for origin in precedence: + for bag in bags: + if _origin_of(bag) == origin and getter(bag): + return bag + for bag in bags: # unknown origins, keep first non-empty + if getter(bag): + return bag + return None + + +def _is_bag_nonempty(bag: MetadataBag) -> bool: + return _origin_of(bag) != "" + + +def merge_bags( + bags: list[MetadataBag], + *, + descriptive_precedence: tuple[str, ...] = DESCRIPTIVE_PRECEDENCE, + structure_precedence: tuple[str, ...] = STRUCTURE_PRECEDENCE, + modified_precedence: tuple[str, ...] = MODIFIED_PRECEDENCE, +) -> MetadataBag: + """Merge several bags (typically one per origin) into one. + + Bags whose ``provenance.origin`` is empty participate only through + field-level fallbacks. All precedence tuples can be overridden. + """ + if not bags: + return MetadataBag() + if len(bags) == 1: + return bags[0] + + populated = [b for b in bags if _is_bag_nonempty(b)] or bags + + # identity — first non-empty; aliases union + identity = IdentityFacet() + identity.name = _first(*[b.identity.name for b in populated]) + identity.slug = _first(*[b.identity.slug for b in populated]) + identity.cross_origin_id = _first( + *[b.identity.cross_origin_id for b in populated] + ) + seen_aliases: set[str] = set() + for b in populated: + for alias in b.identity.aliases: + if alias and alias not in seen_aliases: + identity.aliases.append(alias) + seen_aliases.add(alias) + + # description — winner by descriptive precedence; tags union + winner = _pick_by_precedence( + populated, + lambda b: ( + b.description.title + or b.description.long_name + or b.description.description + ), + descriptive_precedence, + ) + description = DescriptionFacet() + if winner: + description = DescriptionFacet(**winner.description.__dict__.copy()) + description.tags = list(winner.description.tags) + description.themes = list(winner.description.themes) + seen_tags: set[str] = set() + for b in populated: + for tag in b.description.tags: + if tag and tag not in seen_tags: + description.tags.append(tag) + seen_tags.add(tag) + for theme in b.description.themes: + if theme and theme not in description.themes: + description.themes.append(theme) + + # temporal — created earliest, modified by precedence, year/month + # first non-None + temporal = TemporalFacet() + createds = [b.temporal.created for b in populated if b.temporal.created] + temporal.created = min(createds) if createds else None + mod_winner = _pick_by_precedence( + populated, + lambda b: b.temporal.modified is not None, + modified_precedence, + ) + if mod_winner: + temporal.modified = mod_winner.temporal.modified + for b in populated: + if b.temporal.periodicity: + temporal.periodicity = b.temporal.periodicity + break + if b.temporal.valid_from: + temporal.valid_from = b.temporal.valid_from + break + if b.temporal.valid_to: + temporal.valid_to = b.temporal.valid_to + break + for b in populated: + if b.temporal.year is not None: + temporal.year = b.temporal.year + break + for b in populated: + if b.temporal.month is not None: + temporal.month = b.temporal.month + break + + # spatial — most specific scope; ufs/municipalities union + spatial = SpatialFacet() + scope_winner = max( + populated, + key=lambda b: SCOPE_RANK.get(b.spatial.geographic_scope, 0), + ) + spatial.geographic_scope = scope_winner.spatial.geographic_scope + for b in populated: + for uf in b.spatial.ufs: + if uf and uf not in spatial.ufs: + spatial.ufs.append(uf) + for mun in b.spatial.municipalities: + if mun and mun not in spatial.municipalities: + spatial.municipalities.append(mun) + for b in populated: + if b.spatial.state: + spatial.state = b.spatial.state + break + + # provenance — first non-empty; license most permissive + provenance = ProvenanceFacet() + provenance.organization = _first( + *[b.provenance.organization for b in populated] + ) + provenance.author = _first(*[b.provenance.author for b in populated]) + provenance.maintainer = _first( + *[b.provenance.maintainer for b in populated] + ) + provenance.contact = _first(*[b.provenance.contact for b in populated]) + provenance.attribution = _first( + *[b.provenance.attribution for b in populated] + ) + provenance.source_url = _first( + *[b.provenance.source_url for b in populated] + ) + provenance.license = _pick_license(populated) + provenance.license_id = _pick_license_id(populated) + origins = [] + for b in populated: + if b.provenance.origin and b.provenance.origin not in origins: + origins.append(b.provenance.origin) + provenance.origin = "/".join(origins) + + # structure — columns by structure precedence, counts max + col_winner = _pick_by_precedence( + populated, + lambda b: bool(b.structure.columns), + structure_precedence, + ) + structure = StructureFacet() + if col_winner: + structure.columns = list(col_winner.structure.columns) + structure.row_count = max( + (b.structure.row_count for b in populated), default=0 + ) + structure.file_count = max( + (b.structure.file_count for b in populated), default=0 + ) + structure.format = _first(*[b.structure.format for b in populated]) + structure.schema_fingerprint = _first( + *[b.structure.schema_fingerprint for b in populated] + ) + + # access — url first non-empty, size first non-zero, auth OR + access = AccessFacet() + access.url = _first(*[b.access.url for b in populated]) + access.format = _first(*[b.access.format for b in populated]) + for b in populated: + if b.access.size_bytes: + access.size_bytes = b.access.size_bytes + break + access.download_strategy = _first( + *[b.access.download_strategy for b in populated] + ) + access.requires_auth = any(b.access.requires_auth for b in populated) + access.policy = _first(*[b.access.policy for b in populated]) + access.mime_type = _first(*[b.access.mime_type for b in populated]) + + # quality — fingerprint first non-empty, integrity OR, freshness max + quality = QualityFacet() + quality.content_fingerprint = _first( + *[b.quality.content_fingerprint for b in populated] + ) + quality.integrity_verified = any( + b.quality.integrity_verified for b in populated + ) + scores = [ + b.quality.freshness_score + for b in populated + if b.quality.freshness_score is not None + ] + quality.freshness_score = max(scores) if scores else None + pcts = [ + b.quality.completeness_pct + for b in populated + if b.quality.completeness_pct is not None + ] + quality.completeness_pct = max(pcts) if pcts else None + + # raw — merged dicts (later bags win on key collisions) + raw: dict[str, Any] = {} + for b in populated: + raw.update(b.raw) + + return MetadataBag( + identity=identity, + description=description, + temporal=temporal, + spatial=spatial, + provenance=provenance, + structure=structure, + access=access, + quality=quality, + raw=raw, + ) + + +def _pick_license(bags: list[MetadataBag]) -> str: + """Pick the most permissive license title (CC-BY > CC-BY-SA > custom).""" + winner = _first(*[b.provenance.license for b in bags]) + for b in bags: + lic = b.provenance.license.lower() + if "atribuição" in lic and "sem derivações" not in lic: + return b.provenance.license + return winner + + +def _pick_license_id(bags: list[MetadataBag]) -> str: + """Pick the most permissive license id.""" + ids = [b.provenance.license_id for b in bags if b.provenance.license_id] + for preferred in ("cc-by", "cc-by-sa", "cc0"): + if preferred in ids: + return preferred + return ids[0] if ids else "" + + +__all__ = [ + "AccessFacet", + "Column", + "Dataset", + "DatasetGroup", + "DESCRIPTIVE_PRECEDENCE", + "DescriptionFacet", + "File", + "FileMeta", + "IdentityFacet", + "MetadataBag", + "MODIFIED_PRECEDENCE", + "ProvenanceFacet", + "QualityFacet", + "SCOPE_RANK", + "SpatialFacet", + "STRUCTURE_PRECEDENCE", + "StructureFacet", + "TemporalFacet", + "merge_bags", +] diff --git a/pysus/api/models.py b/pysus/api/models.py index c7acef31..c935d330 100644 --- a/pysus/api/models.py +++ b/pysus/api/models.py @@ -15,7 +15,7 @@ from collections.abc import AsyncGenerator, Callable, Sequence from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import pandas as pd import pyarrow as pa @@ -23,6 +23,13 @@ from anyio import to_thread from pydantic import BaseModel, ConfigDict, Field, PrivateAttr from pysus import CACHEPATH +from pysus.api.metadata.models import ( + AccessFacet, + MetadataBag, + ProvenanceFacet, + StructureFacet, + merge_bags, +) from tqdm.asyncio import tqdm from .errors import ConversionError @@ -30,6 +37,7 @@ if TYPE_CHECKING: # pragma: no cover from extensions import Parquet + from pysus.api.metadata.extractors import MetadataExtractor from pysus.api.metadata.models import Column @@ -159,6 +167,26 @@ class BaseTabularFile(BaseLocalFile, ABC): Subclasses must implement *columns*, *rows*, *load*, and *stream*. """ + @property + def metadata(self) -> MetadataBag: + """Return structure/access metadata computed from the file. + + Columns come from the concrete ``columns`` property; row count + and size from the local filesystem. No network is involved. + """ + return MetadataBag( + provenance=ProvenanceFacet(origin="local"), + structure=StructureFacet( + columns=list(self.columns), + row_count=self.rows, + format=str(self.type) if self.type else "", + ), + access=AccessFacet( + url=str(self.path), + size_bytes=self.size, + ), + ) + @property @abstractmethod def columns(self) -> list[Column]: @@ -319,7 +347,69 @@ def _matches(self, obj: Any, **kwargs) -> bool: return True -class BaseRemoteFile(BaseFile, SearchableMixin, ABC): +class MetadataMixin: + """Mixin providing the ``.metadata`` / ``.ametadata()`` contract. + + Shared by :class:`BaseRemoteFile` and :class:`BaseRemoteObject` so + every remote entity (client, dataset, group, file) surfaces the + same merged :class:`~pysus.api.metadata.models.MetadataBag`. + + Concrete subclasses declare their extractors via + :attr:`extractor_types`; the mixin only knows that *some* + extractors exist and merges their output with the default + cross-origin precedence. + """ + + #: Extractor classes the concrete subclass wants applied. Each + #: client module (ftp/dadosgov/ducklake/saude) overrides this. + extractor_types: ClassVar[list[type[MetadataExtractor]]] = [] + + @property + def metadata(self) -> MetadataBag: + """Return the merged metadata bag for this entity. + + Runs every registered extractor synchronously, merges the + bags with the default cross-origin precedence and caches the + result. Extractor failures are ignored (best-effort). + + Returns + ------- + MetadataBag + The merged metadata bag. + """ + cached = getattr(self, "_metadata_cache", None) + if cached is not None: + return cached + extractors = getattr(self, "_extractors", None) or [ + cls() for cls in self.extractor_types + ] + bags: list[MetadataBag] = [] + for extractor in extractors: + try: + bags.append(extractor.extract(self)) + except Exception: # noqa: B902 — best-effort metadata + continue + merged = merge_bags(bags) if bags else MetadataBag() + self._metadata_cache = merged + return merged + + async def ametadata(self) -> MetadataBag: + """Async variant of :meth:`metadata` (may hit the network).""" + extractors = getattr(self, "_extractors", None) or [ + cls() for cls in self.extractor_types + ] + bags: list[MetadataBag] = [] + for extractor in extractors: + try: + bags.append(await extractor.aextract(self)) + except Exception: # noqa: B902 — best-effort metadata + continue + merged = merge_bags(bags) if bags else MetadataBag() + self._metadata_cache = merged + return merged + + +class BaseRemoteFile(BaseFile, SearchableMixin, MetadataMixin, ABC): """Abstract base for a file stored on a remote server. Subclasses must implement *_download*. *dataset* and *group* link back @@ -394,7 +484,7 @@ async def download( return await ExtensionFactory.instantiate(local_path) -class BaseRemoteObject(BaseModel, ABC): +class BaseRemoteObject(MetadataMixin, BaseModel, ABC): """Abstract base for a named remote entity with a description. Subclasses must implement *name*, *long_name*, and *description*. diff --git a/pysus/api/saude/__init__.py b/pysus/api/saude/__init__.py new file mode 100644 index 00000000..a4a5af79 --- /dev/null +++ b/pysus/api/saude/__init__.py @@ -0,0 +1,44 @@ +"""Async client for the OpenDataSUS portal (dadosabertos.saude.gov.br). + +Stage-1 spike — catalog browsing + resource downloads only. The +DEMAS REST query path and the DuckLake sync engine integration ship +in later stages (see ``roadmap_saude.md``). +""" + +from .client import SaudeClient +from .errors import ( + BuildIdMissing, + DatasetNotFound, + NoUsableBuildId, + PortalChanged, + ResourceNotFound, + SaudeError, +) +from .resources import ( + CatalogEntry, + CatalogPage, + CKANPackage, + Extra, + GroupRef, + Organization, + Resource, + TagRef, +) + +__all__ = [ + "BuildIdMissing", + "CatalogEntry", + "CatalogPage", + "CKANPackage", + "DatasetNotFound", + "Extra", + "GroupRef", + "NoUsableBuildId", + "Organization", + "PortalChanged", + "Resource", + "ResourceNotFound", + "SaudeClient", + "SaudeError", + "TagRef", +] diff --git a/pysus/api/saude/catalog.py b/pysus/api/saude/catalog.py new file mode 100644 index 00000000..b6d90577 --- /dev/null +++ b/pysus/api/saude/catalog.py @@ -0,0 +1,321 @@ +"""Async Next.js data fetcher for dadosabertos.saude.gov.br. + +Talks to the portal's data layer of the form:: + + GET /_next/data//dataset.json?q=&groups=&tags=&res_format=&page= + GET /_next/data//dataset/.json?slug= + +Each call goes through :func:`fetch_json` which handles disk caching +(TTL) and exponential-backoff retries on transient transport errors. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +from collections.abc import AsyncIterator +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +import httpx + +from .errors import DatasetNotFound, PortalChanged +from .next_data import fetch_build_id +from .resources import CatalogEntry, CatalogPage, CKANPackage, GroupRef, TagRef + +logger = logging.getLogger(__name__) + +PAGE_SIZE = 20 +_DEFAULT_TTL = timedelta(hours=24) +_RETRYABLE = (httpx.TransportError, httpx.HTTPStatusError, httpx.RequestError) + + +def _cache_key(url: str, params: dict[str, Any] | None) -> str: + """Stable cache key from the URL + sorted query params.""" + parts = [url.replace("https://", "").replace("/", "_").replace(":", "_")] + if params: + parts.append("_".join(f"{k}-{v}" for k, v in sorted(params.items()))) + digest = hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()[:16] + return digest + + +def _is_fresh(path: Path, ttl: timedelta, now: datetime) -> bool: + if not path.exists(): + return False + age = now - datetime.fromtimestamp(path.stat().st_mtime) + return age < ttl + + +def _read_cache(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as fh: + return json.load(fh) + + +def _write_cache(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + json.dump(data, fh) + + +async def fetch_json( + client: httpx.AsyncClient, + url: str, + *, + params: dict[str, Any] | None = None, + cache_path: Path | None = None, + ttl: timedelta = _DEFAULT_TTL, + use_cache: bool = True, + retries: int = 3, +) -> dict[str, Any]: + """GET a JSON payload with TTL cache and exponential-backoff retries. + + Parameters + ---------- + client : httpx.AsyncClient + url : str + params : dict, optional + cache_path : pathlib.Path, optional + Override the auto-derived cache path. + ttl : datetime.timedelta, optional + use_cache : bool, optional + retries : int, optional + + Returns + ------- + dict + Parsed JSON response. + """ + path = cache_path + if path is None: + path = Path(f"/tmp/saude-{_cache_key(url, params)}.json") + if use_cache and _is_fresh(path, ttl, datetime.now()): + logger.debug("saude: cache hit for %s", url) + return _read_cache(path) + + last_error: Exception | None = None + for attempt in range(retries): + try: + response = await client.get(url, params=params) + response.raise_for_status() + data = response.json() + except _RETRYABLE as exc: + last_error = exc + wait = 2**attempt + attempt + logger.warning( + "saude: GET %s attempt %d/%d failed: %s. Retrying in %ds.", + url, + attempt + 1, + retries, + exc, + wait, + ) + await asyncio.sleep(wait) + continue + if use_cache: + _write_cache(path, data) + return data + + raise last_error if last_error else RuntimeError("unreachable") + + +def _build_search_params( + q: str | None = None, + group: str | None = None, + tag: str | None = None, + fmt: str | None = None, + page: int = 1, +) -> dict[str, Any]: + """Mirror of the epidatasets ``_search_params`` helper.""" + params: dict[str, Any] = {"page": page} + if q: + params["q"] = q + if group: + params["groups"] = group + if tag: + params["tags"] = tag + if fmt: + params["res_format"] = fmt + return params + + +async def fetch_catalog_page( + client: httpx.AsyncClient, + *, + build_id: str, + q: str | None = None, + group: str | None = None, + tag: str | None = None, + fmt: str | None = None, + page: int = 1, + cache_root: Path, + ttl: timedelta, + use_cache: bool = True, +) -> CatalogPage: + """Fetch one page (20 datasets) of the catalog listing.""" + base = "https://dadosabertos.saude.gov.br" + url = f"{base}/_next/data/{build_id}/dataset.json" + params = _build_search_params(q, group, tag, fmt, page) + cache_path = cache_root / "catalog" / f"{_cache_key(url, params)}.json" + data = await fetch_json( + client, + url, + params=params, + cache_path=cache_path, + ttl=ttl, + use_cache=use_cache, + ) + page_props = data.get("pageProps", {}) + if "packages" not in page_props: + raise PortalChanged( + "Catalog response does not contain 'packages'; the portal " + "frontend data format may have changed." + ) + try: + return CatalogPage.model_validate(page_props) + except Exception as exc: # noqa: B902 — wrap any validation error + raise PortalChanged( + f"Could not parse catalog page payload: {exc}" + ) from exc + + +async def fetch_catalog_all( + client: httpx.AsyncClient, + *, + build_id: str, + q: str | None = None, + group: str | None = None, + tag: str | None = None, + fmt: str | None = None, + max_pages: int | None = None, + cache_root: Path, + ttl: timedelta, + use_cache: bool = True, +) -> AsyncIterator[CatalogEntry]: + """Yield every catalog entry across all pages. + + Stops when a page returns no packages, or when the number of + packages so far equals ``numberOfPackages``. + """ + page = 1 + yielded = 0 + total: int | None = None + while True: + if max_pages is not None and page > max_pages: + logger.warning( + "saude: reached max_pages=%d; results may be incomplete.", + max_pages, + ) + return + catalog = await fetch_catalog_page( + client, + build_id=build_id, + q=q, + group=group, + tag=tag, + fmt=fmt, + page=page, + cache_root=cache_root, + ttl=ttl, + use_cache=use_cache, + ) + if not catalog.packages: + return + if total is None: + total = catalog.number_of_packages + for entry in catalog.packages: + yield entry + yielded += 1 + if total and yielded >= total: + return + page += 1 + + +async def fetch_dataset( + client: httpx.AsyncClient, + *, + build_id: str, + slug: str, + cache_root: Path, + ttl: timedelta, + use_cache: bool = True, +) -> CKANPackage: + """Fetch the full CKAN package for a single dataset.""" + base = "https://dadosabertos.saude.gov.br" + url = f"{base}/_next/data/{build_id}/dataset/{slug}.json" + cache_path = cache_root / "dataset" / f"{slug}.json" + data = await fetch_json( + client, + url, + params={"slug": slug}, + cache_path=cache_path, + ttl=ttl, + use_cache=use_cache, + ) + page_props = data.get("pageProps") or {} + if not page_props or page_props.get("name") != slug: + raise DatasetNotFound(f"Dataset '{slug}' not found on OpenDataSUS.") + try: + return CKANPackage.model_validate(page_props) + except Exception as exc: # noqa: B902 — wrap any validation error + raise PortalChanged( + f"Could not parse dataset payload for '{slug}': {exc}" + ) from exc + + +async def list_groups( + client: httpx.AsyncClient, + *, + build_id: str, + cache_root: Path, + ttl: timedelta, + use_cache: bool = True, +) -> list[GroupRef]: + """Return all 14 catalog groups (themes).""" + page = await fetch_catalog_page( + client, + build_id=build_id, + page=1, + cache_root=cache_root, + ttl=ttl, + use_cache=use_cache, + ) + return [ + GroupRef.model_validate(g) + for g in page.available_filters.get("groups", []) + ] + + +async def list_tags( + client: httpx.AsyncClient, + *, + build_id: str, + cache_root: Path, + ttl: timedelta, + use_cache: bool = True, +) -> list[TagRef]: + """Return all catalog tags.""" + page = await fetch_catalog_page( + client, + build_id=build_id, + page=1, + cache_root=cache_root, + ttl=ttl, + use_cache=use_cache, + ) + return [ + TagRef.model_validate(t) for t in page.available_filters.get("tags", []) + ] + + +__all__ = [ + "PAGE_SIZE", + "fetch_build_id", + "fetch_catalog_page", + "fetch_catalog_all", + "fetch_dataset", + "list_groups", + "list_tags", +] diff --git a/pysus/api/saude/client.py b/pysus/api/saude/client.py new file mode 100644 index 00000000..cc2a8347 --- /dev/null +++ b/pysus/api/saude/client.py @@ -0,0 +1,231 @@ +"""Async facade for the Saude (dadosabertos.saude.gov.br) client.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Callable +from datetime import timedelta +from pathlib import Path + +import httpx +from pysus import CACHEPATH + +from .catalog import ( + _DEFAULT_TTL, + fetch_build_id, + fetch_catalog_all, + fetch_catalog_page, + fetch_dataset, + list_groups, + list_tags, +) +from .download import download_dataset as _download_dataset +from .download import download_resource as _download_resource +from .resources import CatalogEntry, CKANPackage, GroupRef, Resource, TagRef + + +class SaudeClient: + """Async client for the OpenDataSUS portal. + + The portal is a Next.js frontend over a CKAN backend. ``SaudeClient`` + owns an ``httpx.AsyncClient`` and the on-disk caches for the Next.js + ``buildId`` and the catalog pages. No authentication is required. + + Example + ------- + >>> import asyncio + >>> from pysus.api.saude import SaudeClient + >>> async def main(): + ... async with SaudeClient() as c: + ... datasets = await c.list_datasets(group="arboviroses") + ... print([d.name for d in datasets]) + >>> asyncio.run(main()) + """ + + BASE_URL = "https://dadosabertos.saude.gov.br" + + def __init__( + self, + *, + cache_dir: Path | None = None, + cache_ttl: timedelta = _DEFAULT_TTL, + timeout: float = 30.0, + user_agent: str | None = None, + ) -> None: + self.cache_dir = ( + Path(cache_dir) if cache_dir else Path(CACHEPATH) / "saude" + ) + self.cache_ttl = cache_ttl + self.cache_dir.mkdir(parents=True, exist_ok=True) + headers = {"User-Agent": user_agent or "pysus-saude/0.1 (research)"} + self._client = httpx.AsyncClient( + headers=headers, timeout=timeout, follow_redirects=True + ) + self._build_id: str | None = None + + async def __aenter__(self) -> SaudeClient: + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + await self.close() + + async def close(self) -> None: + """Close the underlying HTTP client.""" + await self._client.aclose() + + async def _ensure_build_id(self, use_cache: bool = True) -> str: + if self._build_id and not use_cache: + pass + elif self._build_id and use_cache: + return self._build_id + build_id = await fetch_build_id( + self._client, + cache_path=self.cache_dir / "build_id.json", + homepage_url=self.BASE_URL + "/", + ttl=self.cache_ttl, + ) + self._build_id = build_id + return build_id + + async def list_datasets( + self, + *, + q: str | None = None, + group: str | None = None, + tag: str | None = None, + fmt: str | None = None, + page: int = 1, + use_cache: bool = True, + ) -> list[CatalogEntry]: + """Return one page (20 entries) of the catalog listing.""" + build_id = await self._ensure_build_id(use_cache=use_cache) + catalog = await fetch_catalog_page( + self._client, + build_id=build_id, + q=q, + group=group, + tag=tag, + fmt=fmt, + page=page, + cache_root=self.cache_dir, + ttl=self.cache_ttl, + use_cache=use_cache, + ) + return catalog.packages + + async def iter_datasets( + self, + *, + q: str | None = None, + group: str | None = None, + tag: str | None = None, + fmt: str | None = None, + max_pages: int | None = None, + use_cache: bool = True, + ) -> AsyncIterator[CatalogEntry]: + """Yield every catalog entry across all pages.""" + build_id = await self._ensure_build_id(use_cache=use_cache) + async for entry in fetch_catalog_all( + self._client, + build_id=build_id, + q=q, + group=group, + tag=tag, + fmt=fmt, + max_pages=max_pages, + cache_root=self.cache_dir, + ttl=self.cache_ttl, + use_cache=use_cache, + ): + yield entry + + async def list_groups(self, *, use_cache: bool = True) -> list[GroupRef]: + """Return the 14 catalog groups (themes).""" + build_id = await self._ensure_build_id(use_cache=use_cache) + return await list_groups( + self._client, + build_id=build_id, + cache_root=self.cache_dir, + ttl=self.cache_ttl, + use_cache=use_cache, + ) + + async def list_tags(self, *, use_cache: bool = True) -> list[TagRef]: + """Return the catalog tags.""" + build_id = await self._ensure_build_id(use_cache=use_cache) + return await list_tags( + self._client, + build_id=build_id, + cache_root=self.cache_dir, + ttl=self.cache_ttl, + use_cache=use_cache, + ) + + async def fetch_dataset( + self, slug: str, *, use_cache: bool = True + ) -> CKANPackage: + """Fetch the full CKAN package for a single dataset.""" + build_id = await self._ensure_build_id(use_cache=use_cache) + return await fetch_dataset( + self._client, + build_id=build_id, + slug=slug, + cache_root=self.cache_dir, + ttl=self.cache_ttl, + use_cache=use_cache, + ) + + async def fetch_resources( + self, slug: str, *, use_cache: bool = True + ) -> list[Resource]: + """Fetch the resources of a dataset.""" + package = await self.fetch_dataset(slug, use_cache=use_cache) + return package.resources + + async def download_resource( + self, + slug: str, + *, + resource_id: str | None = None, + name: str | None = None, + fmt: str | None = None, + dest_dir: Path | None = None, + progress: Callable[[int, int], None] | None = None, + overwrite: bool = False, + use_cache: bool = True, + ) -> Path: + """Download one resource of a dataset.""" + package = await self.fetch_dataset(slug, use_cache=use_cache) + return await _download_resource( + self._client, + package, + resource_id=resource_id, + name=name, + fmt=fmt, + dest_dir=dest_dir, + progress=progress, + overwrite=overwrite, + ) + + async def download_dataset( + self, + slug: str, + *, + dest_dir: Path | None = None, + fmt: str | None = None, + progress: Callable[[int, int], None] | None = None, + overwrite: bool = False, + use_cache: bool = True, + ) -> list[Path]: + """Download every downloadable resource of a dataset.""" + package = await self.fetch_dataset(slug, use_cache=use_cache) + return await _download_dataset( + self._client, + package, + dest_dir=dest_dir, + fmt=fmt, + progress=progress, + overwrite=overwrite, + ) + + +__all__ = ["SaudeClient"] diff --git a/pysus/api/saude/download.py b/pysus/api/saude/download.py new file mode 100644 index 00000000..ef74bb18 --- /dev/null +++ b/pysus/api/saude/download.py @@ -0,0 +1,167 @@ +"""Async download helpers for Saude (CKAN) resources. + +Mirrors ``epidatasets.sources.opendatasus.OpenDataSUSAccessor +.download_resource`` / ``.download_dataset`` but uses ``httpx.AsyncClient +.stream`` and a progress callback shaped like the rest of PySUS. + +Resources whose ``format`` is ``API`` are documentation links and are +skipped automatically. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from pathlib import Path +from urllib.parse import urlparse + +from .errors import ResourceNotFound +from .resources import CKANPackage, Resource + +_DOWNLOAD_CHUNK = 1 << 16 +_NON_SAFE = re.compile(r"[^\w.\- ]") + + +def filename_for(resource: Resource, slug: str, index: int) -> str: + """Derive a safe filename for a resource download. + + Ported verbatim from the epidatasets reference implementation. + """ + base = ( + resource.name + or Path(urlparse(resource.url).path).name + or f"resource_{index}" + ) + base = _NON_SAFE.sub("_", base).strip() or f"resource_{index}" + fmt = (resource.format or "").strip().lower() + if fmt and fmt != "api" and not base.lower().endswith(f".{fmt}"): + base = f"{base}.{fmt}" + return base + + +def _select_resource( + package: CKANPackage, + *, + resource_id: str | None, + name: str | None, + fmt: str | None, +) -> Resource: + """Pick the unique resource matching the selector, else raise.""" + if (resource_id is None) == (name is None): + raise ValueError("Provide exactly one of 'resource_id' or 'name'.") + matches = [] + for resource in package.resources: + if resource_id is not None and resource.id != resource_id: + continue + if name is not None and resource.name != name: + continue + if fmt is not None and resource.format.upper() != fmt.upper(): + continue + if resource.format.upper() == "API": + continue + matches.append(resource) + if not matches: + raise ResourceNotFound( + f"No downloadable resource matches resource_id={resource_id!r}, " + f"name={name!r}, fmt={fmt!r} for dataset '{package.name}'.", + candidates=len(package.resources), + ) + return matches[0] + + +async def download_resource( + client, # httpx.AsyncClient + package: CKANPackage, + *, + resource_id: str | None = None, + name: str | None = None, + fmt: str | None = None, + dest_dir: Path | None = None, + progress: Callable[[int, int], None] | None = None, + overwrite: bool = False, +) -> Path: + """Download a single resource of a dataset to disk. + + Parameters + ---------- + client : httpx.AsyncClient + package : CKANPackage + resource_id, name, fmt : optional selectors (exactly one of id/name) + dest_dir : pathlib.Path, optional + Defaults to ``/downloads//``. + progress : callable, optional + ``(downloaded_bytes, total_bytes)`` callback. + overwrite : bool, optional + When ``False`` (default), an existing file is reused. + + Returns + ------- + pathlib.Path + Path to the downloaded file. + """ + resource = _select_resource( + package, resource_id=resource_id, name=name, fmt=fmt + ) + target_dir = ( + Path(dest_dir) if dest_dir else Path(f"./{package.name}-downloads") + ) + target_dir.mkdir(parents=True, exist_ok=True) + index = resource.position or 0 + dest_path = target_dir / filename_for(resource, package.name, index) + + if dest_path.exists() and not overwrite: + return dest_path + + async with client.stream("GET", resource.url) as response: + response.raise_for_status() + total = int(response.headers.get("Content-Length", 0)) + downloaded = 0 + with dest_path.open("wb") as fh: + async for chunk in response.aiter_bytes(chunk_size=_DOWNLOAD_CHUNK): + fh.write(chunk) + downloaded += len(chunk) + if progress: + progress(downloaded, total) + return dest_path + + +async def download_dataset( + client, # httpx.AsyncClient + package: CKANPackage, + *, + dest_dir: Path | None = None, + fmt: str | None = None, + progress: Callable[[int, int], None] | None = None, + overwrite: bool = False, +) -> list[Path]: + """Download every downloadable resource of a dataset. + + Returns the list of paths written. API-format resources are + skipped. + """ + dest = Path(dest_dir) if dest_dir else Path(f"./{package.name}-downloads") + dest.mkdir(parents=True, exist_ok=True) + paths: list[Path] = [] + fmt_upper = fmt.upper() if fmt else None + for resource in package.resources: + if resource.format.upper() == "API": + continue + if fmt_upper and resource.format.upper() != fmt_upper: + continue + path = await download_resource( + client, + package, + resource_id=resource.id, + dest_dir=dest, + progress=progress, + overwrite=overwrite, + ) + paths.append(path) + return paths + + +__all__ = [ + "filename_for", + "download_resource", + "download_dataset", +] diff --git a/pysus/api/saude/errors.py b/pysus/api/saude/errors.py new file mode 100644 index 00000000..c2b39241 --- /dev/null +++ b/pysus/api/saude/errors.py @@ -0,0 +1,29 @@ +"""Typed errors raised by the Saude (dadosabertos.saude.gov.br) client.""" + + +class SaudeError(Exception): + """Base class for all errors raised by the Saude client.""" + + +class BuildIdMissing(SaudeError): + """The ``__NEXT_DATA__`` block could not be located on the homepage.""" + + +class PortalChanged(SaudeError): + """The catalog response no longer matches the expected Next.js schema.""" + + +class DatasetNotFound(SaudeError): + """The requested dataset slug does not exist on the portal.""" + + +class ResourceNotFound(SaudeError): + """No downloadable resource matched the given selector.""" + + def __init__(self, message: str, *, candidates: int = 0) -> None: + super().__init__(message) + self.candidates = candidates + + +class NoUsableBuildId(SaudeError): + """The buildId cache and the live homepage both failed to yield a value.""" diff --git a/pysus/api/saude/metadata.py b/pysus/api/saude/metadata.py new file mode 100644 index 00000000..66ec1b9e --- /dev/null +++ b/pysus/api/saude/metadata.py @@ -0,0 +1,191 @@ +"""Metadata extractors for the Saude (dadosabertos.saude.gov.br) client. + +Consumes the Stage-1 models (:class:`~pysus.api.saude.resources +.CKANPackage`, :class:`~pysus.api.saude.resources.Resource`, +:class:`~pysus.api.saude.resources.GroupRef`) and produces +:class:`~pysus.api.metadata.models.MetadataBag` instances. + +The CKAN package is the richest metadata source available for the +Saude origin: title, notes, license, organization, contact, +periodicity, tags and themes all map into the bag. +""" + +from __future__ import annotations + +from typing import Any + +from pysus.api.metadata.extractors import MetadataExtractor +from pysus.api.metadata.models import ( + AccessFacet, + DescriptionFacet, + IdentityFacet, + MetadataBag, + ProvenanceFacet, + QualityFacet, + StructureFacet, + TemporalFacet, +) + +# ---------------------------------------------------------------------- +# Dataset +# ---------------------------------------------------------------------- + + +class SaudeDatasetExtractor(MetadataExtractor): + """Build a dataset-level bag from a CKAN package.""" + + origin = "saude" + + def supported_facets(self) -> set[str]: + return { + "identity", + "description", + "temporal", + "provenance", + "structure", + "access", + } + + def _extract(self, obj: Any) -> MetadataBag: + package = obj + + organization = "" + if package.organization is not None: + organization = ( + package.organization.display_name + or package.organization.name + or "" + ) + + return MetadataBag( + identity=IdentityFacet( + name=package.name, + slug=package.name, + cross_origin_id=package.id, + ), + description=DescriptionFacet( + title=package.title, + long_name=package.title, + description=package.notes or "", + tags=[ + (tag.display_name or tag.name) + for tag in package.tags + if (tag.display_name or tag.name) + ], + themes=[ + (group.display_name or group.name) + for group in package.groups + if (group.display_name or group.name) + ], + ), + temporal=TemporalFacet( + created=package.metadata_created, + modified=package.metadata_modified, + periodicity=package.periodicity, + ), + provenance=ProvenanceFacet( + origin=self.origin, + organization=organization, + author=package.author or "", + maintainer=package.maintainer or "", + contact=package.contact or "", + license=package.license_title or "", + license_id=package.license_id or "", + source_url=package.url or "", + ), + structure=StructureFacet(file_count=package.num_resources), + access=AccessFacet( + policy=f"{package.state or 'active'}/" + f"{'private' if package.private else 'public'}/" + f"{'open' if package.isopen else 'closed'}" + ), + ) + + +# ---------------------------------------------------------------------- +# Group (theme) +# ---------------------------------------------------------------------- + + +class SaudeGroupExtractor(MetadataExtractor): + """Build a group-level bag from a CKAN ``GroupRef``.""" + + origin = "saude" + + def supported_facets(self) -> set[str]: + return {"identity", "description", "provenance"} + + def _extract(self, obj: Any) -> MetadataBag: + group = obj + display = group.display_name or group.name or "" + return MetadataBag( + identity=IdentityFacet(name=group.name, slug=group.name), + description=DescriptionFacet( + title=display, + themes=[group.name], + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +# ---------------------------------------------------------------------- +# File (resource) +# ---------------------------------------------------------------------- + + +class SaudeFileExtractor(MetadataExtractor): + """Build a file-level bag from a CKAN ``Resource``.""" + + origin = "saude" + + def supported_facets(self) -> set[str]: + return { + "identity", + "description", + "temporal", + "structure", + "access", + "quality", + "provenance", + } + + def _extract(self, obj: Any) -> MetadataBag: + resource = obj + return MetadataBag( + identity=IdentityFacet( + name=resource.name, + cross_origin_id=resource.id, + ), + description=DescriptionFacet( + description=resource.description or "", + ), + temporal=TemporalFacet( + created=resource.created, + modified=(resource.last_modified or resource.metadata_modified), + ), + structure=StructureFacet(format=resource.format or ""), + access=AccessFacet( + url=resource.url, + format=resource.format or "", + size_bytes=resource.size or 0, + mime_type=resource.mimetype or "", + download_strategy="http-stream", + ), + quality=QualityFacet( + integrity_verified=bool(resource.hash), + ), + provenance=ProvenanceFacet(origin=self.origin), + raw={ + "position": resource.position, + "hash": resource.hash or "", + "state": resource.state or "", + "resource_type": resource.resource_type or "", + }, + ) + + +__all__ = [ + "SaudeDatasetExtractor", + "SaudeFileExtractor", + "SaudeGroupExtractor", +] diff --git a/pysus/api/saude/next_data.py b/pysus/api/saude/next_data.py new file mode 100644 index 00000000..7c781ba7 --- /dev/null +++ b/pysus/api/saude/next_data.py @@ -0,0 +1,173 @@ +"""Next.js data-layer helpers for dadosabertos.saude.gov.br. + +The portal's CKAN backend is served through a Next.js frontend. The +CKAN Action API (``/api/3/action/*``) is not exposed, but Next.js +serves the same catalog data via its hydration endpoints of the form:: + + /_next/data//dataset.json + /_next/data//dataset/.json + +```` rotates on every frontend deploy, so it must be +discovered from the homepage's ``__NEXT_DATA__`` script tag and +cached. + +This module owns the buildId extraction and the on-disk cache. It +exposes one async helper (:func:`fetch_build_id`) and a small +sentinel value used by tests to bypass the cache. +""" + +from __future__ import annotations + +import json +import re +from datetime import datetime, timedelta +from pathlib import Path + +import httpx + +from .errors import BuildIdMissing, NoUsableBuildId + +_NEXT_DATA_RE = re.compile( + r'__NEXT_DATA__"\s+type="application/json">(.*?)', + re.DOTALL, +) + +_DEFAULT_TTL = timedelta(hours=24) + + +def _parse_build_id(html: str) -> str: + """Extract ``buildId`` from the ``__NEXT_DATA__`` script block. + + Parameters + ---------- + html : str + The full HTML of the homepage. + + Returns + ------- + str + The buildId value. + + Raises + ------ + BuildIdMissing + If the script tag or the ``buildId`` field cannot be located. + """ + match = _NEXT_DATA_RE.search(html) + if not match: + raise BuildIdMissing( + "Could not find __NEXT_DATA__ on the OpenDataSUS homepage; " + "the portal frontend may have changed." + ) + try: + payload = json.loads(match.group(1)) + except json.JSONDecodeError as exc: + raise BuildIdMissing( + "Could not parse __NEXT_DATA__ JSON on the OpenDataSUS homepage." + ) from exc + build_id = payload.get("buildId") + if not build_id: + raise BuildIdMissing( + "No 'buildId' field in the OpenDataSUS homepage __NEXT_DATA__." + ) + return str(build_id) + + +def _read_cache(cache_path: Path) -> dict | None: + """Return the cached buildId payload, or ``None`` if absent.""" + if not cache_path.exists(): + return None + try: + with cache_path.open(encoding="utf-8") as fh: + return json.load(fh) + except (OSError, json.JSONDecodeError): + return None + + +def _is_fresh(path: Path, ttl: timedelta, now: datetime) -> bool: + """True when the cache payload at ``path`` is younger than ``ttl``. + + Freshness is decided by the ``saved_at`` timestamp recorded inside + the payload — not the file mtime, which is not under our control + (copies, git checkouts, etc. would skew it). + """ + payload = _read_cache(path) + if not payload: + return False + saved_at = payload.get("saved_at") + if not saved_at: + return False + try: + ts = datetime.fromisoformat(saved_at) + except ValueError: + return False + return now - ts < ttl + + +def _write_cache(cache_path: Path, build_id: str, now: datetime) -> None: + """Persist ``build_id`` to ``cache_path`` with a timestamp.""" + cache_path.parent.mkdir(parents=True, exist_ok=True) + with cache_path.open("w", encoding="utf-8") as fh: + json.dump( + {"buildId": build_id, "saved_at": now.isoformat()}, + fh, + ) + + +async def fetch_build_id( + client: httpx.AsyncClient, + *, + cache_path: Path, + homepage_url: str, + ttl: timedelta = _DEFAULT_TTL, +) -> str: + """Discover and cache the portal's current Next.js ``buildId``. + + The function consults the on-disk cache first, falls back to + fetching the homepage if the cache is missing or stale, and writes + the result back to the cache. + + Parameters + ---------- + client : httpx.AsyncClient + The HTTP client used for the homepage request. + cache_path : pathlib.Path + Where the buildId is cached on disk. + homepage_url : str + URL of the portal homepage. + ttl : datetime.timedelta, optional + How long a cached buildId is considered fresh. + + Returns + ------- + str + The current buildId. + + Raises + ------ + NoUsableBuildId + If neither the cache nor a fresh homepage fetch yields a value. + """ + now = datetime.now() + if _is_fresh(cache_path, ttl, now): + cached = _read_cache(cache_path) + cached_id = cached.get("buildId") if cached else None + if cached_id: + return str(cached_id) + + try: + response = await client.get(homepage_url) + response.raise_for_status() + except httpx.HTTPError as exc: + stale = _read_cache(cache_path) + stale_id = stale.get("buildId") if stale else None + if stale_id: + return str(stale_id) + raise NoUsableBuildId( + "Could not fetch the OpenDataSUS homepage and no cached " + "buildId is available." + ) from exc + + build_id = _parse_build_id(response.text) + _write_cache(cache_path, build_id, now) + return build_id diff --git a/pysus/api/saude/resources.py b/pysus/api/saude/resources.py new file mode 100644 index 00000000..a2d7fbf7 --- /dev/null +++ b/pysus/api/saude/resources.py @@ -0,0 +1,255 @@ +"""Pydantic models for the CKAN package / resource payloads. + +The portal serves the catalog via a Next.js data layer. Every dataset +exposes a CKAN package dict with the 30 fields documented in +``roadmap_saude.md`` §0.5.2, and each package carries a +``resources[]`` list of 19 fields. We model the subset we care about; +``model_config = ConfigDict(extra="ignore")`` keeps the client +forward-compatible with new CKAN fields. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class _FlexibleModel(BaseModel): + """Base model that ignores unknown keys and normalises timestamps.""" + + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + +def _coerce_str(value: Any) -> str: + """Coerce ``None`` / non-str to an empty string for optional fields.""" + if value is None: + return "" + return str(value) + + +def _parse_iso(value: Any) -> datetime | None: + """Parse an ISO-8601 timestamp; return ``None`` on bad input.""" + if value is None or value == "": + return None + if isinstance(value, datetime): + return value + try: + return datetime.fromisoformat(str(value)) + except ValueError: + return None + + +class GroupRef(_FlexibleModel): + """Reference to a CKAN group (theme).""" + + name: str + display_name: str | None = None + + +class TagRef(_FlexibleModel): + """Reference to a CKAN tag.""" + + name: str + display_name: str | None = None + + +class Organization(_FlexibleModel): + """The owning organization of a CKAN package.""" + + id: str + name: str + title: str | None = None + display_name: str | None = None + description: str | None = None + created: datetime | None = None + approval_status: str | None = None + state: str | None = None + image_url: str | None = None + + +class Extra(_FlexibleModel): + """A single ``extras[]`` entry — free-form key/value annotations.""" + + key: str + value: str + + +class Resource(_FlexibleModel): + """A downloadable file attached to a CKAN package. + + 19 fields documented in ``roadmap_saude.md`` §0.5.2. ``format`` is + normalised to uppercase on the way in. + """ + + id: str + name: str = "" + description: str = "" + format: str = "" + url: str + size: int | None = None + mimetype: str | None = None + created: datetime | None = None + last_modified: datetime | None = None + metadata_modified: datetime | None = None + position: int = 0 + hash: str = "" + cache_last_updated: datetime | None = None + cache_url: str | None = None + datastore_active: bool | None = None + mimetype_inner: str | None = None + package_id: str | None = None + resource_type: str | None = None + state: str | None = None + url_type: str = "" + + @field_validator( + "name", "description", "format", "hash", "url_type", mode="before" + ) + @classmethod + def _coerce_str(cls, value: Any) -> str: + return _coerce_str(value) + + @field_validator("format", mode="before") + @classmethod + def _normalise_format(cls, value: Any) -> str: + if value is None: + return "" + return str(value).strip().upper() + + @field_validator( + "created", + "last_modified", + "metadata_modified", + "cache_last_updated", + mode="before", + ) + @classmethod + def _parse_ts(cls, value: Any) -> datetime | None: + return _parse_iso(value) + + +class CKANPackage(_FlexibleModel): + """The full CKAN package for a dataset. + + 30 fields documented in ``roadmap_saude.md`` §0.5.2. ``extras[]`` + is the source of the Portuguese periodicity + (``Frequência de atualização``) and the contact email + (``Contato``); both are exposed as properties. + """ + + id: str + name: str + title: str + notes: str = "" + author: str | None = None + author_email: str | None = None + creator_user_id: str | None = None + isopen: bool = True + license_id: str | None = None + license_title: str | None = None + license_url: str | None = None + maintainer: str | None = None + maintainer_email: str | None = None + metadata_created: datetime + metadata_modified: datetime + num_resources: int + num_tags: int = 0 + organization: Organization | None = None + owner_org: str | None = None + private: bool = False + state: str = "active" + type: str = "dataset" + url: str | None = None + version: str | None = None + extras: list[Extra] = Field(default_factory=list) + groups: list[GroupRef] = Field(default_factory=list) + tags: list[TagRef] = Field(default_factory=list) + resources: list[Resource] = Field(default_factory=list) + + @field_validator("notes", mode="before") + @classmethod + def _coerce_notes(cls, value: Any) -> str: + return _coerce_str(value) + + @field_validator("metadata_created", "metadata_modified", mode="before") + @classmethod + def _parse_ts(cls, value: Any) -> datetime: + ts = _parse_iso(value) + if ts is None: + raise ValueError(f"Invalid timestamp: {value!r}") + return ts + + @property + def periodicity(self) -> str | None: + """The ``Frequência de atualização`` extra, or ``None``.""" + for extra in self.extras: + if extra.key == "Frequência de atualização": + return extra.value + return None + + @property + def contact(self) -> str | None: + """The ``Contato`` extra, or ``None``.""" + for extra in self.extras: + if extra.key == "Contato": + return extra.value + return None + + @property + def ckan_id(self) -> str: + """Alias for ``id`` (the shared UUID across portals).""" + return self.id + + +class CatalogEntry(_FlexibleModel): + """A package projected for the catalog listing endpoint. + + The paginated ``/dataset.json`` response strips most fields and + keeps only what the catalog UI needs (see + ``roadmap_saude.md`` §0.5.2 for the full mapping). + """ + + name: str + title: str + notes: str = "" + formats: list[str] = Field(default_factory=list) + groups: list[GroupRef] = Field(default_factory=list) + tags: list[TagRef] = Field(default_factory=list) + + @field_validator("notes", mode="before") + @classmethod + def _coerce_notes(cls, value: Any) -> str: + return _coerce_str(value) + + +class CatalogPage(_FlexibleModel): + """One page of the Next.js catalog listing response. + + CKAN uses camelCase keys; pydantic reads them via the + ``AliasChoices`` so the snake_case Python API still works. + """ + + packages: list[CatalogEntry] = Field(default_factory=list, alias="packages") + number_of_packages: int = Field( + default=0, validation_alias="numberOfPackages" + ) + page: int = Field(default=1) + rows: int = Field(default=0) + current_filters: dict[str, Any] = Field( + default_factory=dict, validation_alias="currentFilters" + ) + available_filters: dict[str, list[dict[str, str]]] = Field( + default_factory=dict, validation_alias="availableFilters" + ) + + @field_validator("page", "rows", mode="before") + @classmethod + def _coerce_int(cls, value: Any) -> int: + if value is None or value == "": + return 0 + try: + return int(value) + except (TypeError, ValueError): + return 0 diff --git a/pysus/tests/api/metadata/test_bag.py b/pysus/tests/api/metadata/test_bag.py new file mode 100644 index 00000000..8708410a --- /dev/null +++ b/pysus/tests/api/metadata/test_bag.py @@ -0,0 +1,287 @@ +"""Tests for MetadataBag round-trip and merge precedence.""" + +from __future__ import annotations + +from datetime import datetime + +from pysus.api.metadata.models import ( + AccessFacet, + Column, + DescriptionFacet, + IdentityFacet, + MetadataBag, + ProvenanceFacet, + QualityFacet, + SpatialFacet, + StructureFacet, + TemporalFacet, + merge_bags, +) +from pysus.api.types import VARCHAR + + +def _bag(origin: str, **facets) -> MetadataBag: + bag = MetadataBag(provenance=ProvenanceFacet(origin=origin)) + for name, facet in facets.items(): + setattr(bag, name, facet) + return bag + + +class TestRoundTrip: + def test_empty_bag_roundtrip(self): + bag = MetadataBag() + restored = MetadataBag.from_dict(bag.to_dict()) + assert restored == bag + + def test_full_bag_roundtrip(self): + bag = MetadataBag( + identity=IdentityFacet( + name="SINAN", + slug="arboviroses-dengue", + aliases=["DENG"], + cross_origin_id="4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c", + ), + description=DescriptionFacet( + title="Sinan/Dengue", + description="Notificações de dengue", + tags=["dengue", "arbovirose"], + themes=["Arboviroses"], + ), + temporal=TemporalFacet( + created=datetime(2024, 2, 22, 19, 23, 6), + modified=datetime(2026, 8, 16, 3, 59, 48), + periodicity="Semanal", + year=2025, + month=3, + ), + spatial=SpatialFacet( + geographic_scope="state", + ufs=["RJ", "SP"], + municipalities=["330455"], + state="RJ", + ), + provenance=ProvenanceFacet( + origin="saude", + organization="Ministério da Saúde", + license="Creative Commons Atribuição", + license_id="cc-by", + contact="arboviroses@saude.gov.br", + ), + structure=StructureFacet( + columns=[Column("DT_NOTIFIC", "", VARCHAR)], + row_count=172_855, + file_count=83, + format="CSV", + schema_fingerprint="abc123", + ), + access=AccessFacet( + url="https://example.com/x.csv.zip", + format="CSV", + size_bytes=1024, + requires_auth=False, + policy="active/public/open", + ), + quality=QualityFacet( + content_fingerprint="deadbeef", + integrity_verified=True, + completeness_pct=98.5, + ), + raw={"position": 2}, + ) + restored = MetadataBag.from_dict(bag.to_dict()) + assert restored == bag + + def test_to_dict_is_json_compatible(self): + import json + + bag = MetadataBag( + temporal=TemporalFacet( + modified=datetime(2026, 8, 16, 3, 59, 48), + ), + structure=StructureFacet( + columns=[Column("A", "desc", VARCHAR)], + ), + ) + json.dumps(bag.to_dict()) + + +class TestMergeIdentity: + def test_first_non_empty_wins(self): + a = _bag("ftp", identity=IdentityFacet(name="SINAN")) + b = _bag("saude", identity=IdentityFacet(name="", slug="x")) + merged = merge_bags([a, b]) + assert merged.identity.name == "SINAN" + assert merged.identity.slug == "x" + + def test_aliases_union(self): + a = _bag("ftp", identity=IdentityFacet(aliases=["DENG"])) + b = _bag("dadosgov", identity=IdentityFacet(aliases=["DENG", "DENGUE"])) + merged = merge_bags([a, b]) + assert merged.identity.aliases == ["DENG", "DENGUE"] + + def test_cross_origin_id_first_non_empty(self): + a = _bag("ftp", identity=IdentityFacet()) + b = _bag("saude", identity=IdentityFacet(cross_origin_id="uuid-1")) + merged = merge_bags([a, b]) + assert merged.identity.cross_origin_id == "uuid-1" + + +class TestMergeDescription: + def test_saude_wins_over_ftp(self): + a = _bag( + "ftp", + description=DescriptionFacet(title="SINAN", description="ftp desc"), + ) + b = _bag( + "saude", + description=DescriptionFacet( + title="Sinan/Dengue", description="saude desc" + ), + ) + merged = merge_bags([a, b]) + assert merged.description.title == "Sinan/Dengue" + assert merged.description.description == "saude desc" + + def test_tags_union(self): + a = _bag("ftp", description=DescriptionFacet(tags=["x"])) + b = _bag("saude", description=DescriptionFacet(tags=["y"])) + merged = merge_bags([a, b]) + assert sorted(merged.description.tags) == ["x", "y"] + + +class TestMergeTemporal: + def test_created_earliest(self): + a = _bag( + "ftp", + temporal=TemporalFacet(created=datetime(2024, 1, 1)), + ) + b = _bag( + "saude", + temporal=TemporalFacet(created=datetime(2023, 1, 1)), + ) + merged = merge_bags([a, b]) + assert merged.temporal.created == datetime(2023, 1, 1) + + def test_modified_ducklake_wins(self): + a = _bag( + "dadosgov", + temporal=TemporalFacet(modified=datetime(2026, 1, 1)), + ) + b = _bag( + "ducklake", + temporal=TemporalFacet(modified=datetime(2025, 1, 1)), + ) + merged = merge_bags([a, b]) + # ducklake precedes dadosgov in MODIFIED_PRECEDENCE + assert merged.temporal.modified == datetime(2025, 1, 1) + + def test_year_month_first_non_none(self): + a = _bag("ftp", temporal=TemporalFacet(year=None, month=3)) + b = _bag("saude", temporal=TemporalFacet(year=2025, month=None)) + merged = merge_bags([a, b]) + assert merged.temporal.year == 2025 + assert merged.temporal.month == 3 + + +class TestMergeSpatial: + def test_most_specific_scope_wins(self): + a = _bag("ftp", spatial=SpatialFacet(geographic_scope="national")) + b = _bag("saude", spatial=SpatialFacet(geographic_scope="state")) + merged = merge_bags([a, b]) + assert merged.spatial.geographic_scope == "state" + + def test_ufs_union(self): + a = _bag("ftp", spatial=SpatialFacet(ufs=["RJ"])) + b = _bag("dadosgov", spatial=SpatialFacet(ufs=["SP"])) + merged = merge_bags([a, b]) + assert sorted(merged.spatial.ufs) == ["RJ", "SP"] + + +class TestMergeProvenance: + def test_origins_joined(self): + a = _bag("ftp") + b = _bag("saude") + merged = merge_bags([a, b]) + assert merged.provenance.origin == "ftp/saude" + + def test_license_prefers_cc_by(self): + a = _bag( + "ftp", + provenance=ProvenanceFacet( + license="Creative Commons Atribuição", license_id="cc-by" + ), + ) + b = _bag( + "saude", + provenance=ProvenanceFacet( + license="Creative Commons Atribuição-SemDerivações", + license_id="cc-by-nd", + ), + ) + merged = merge_bags([a, b]) + assert merged.provenance.license_id == "cc-by" + + def test_contact_first_non_empty(self): + a = _bag("ftp", provenance=ProvenanceFacet(contact="")) + b = _bag("saude", provenance=ProvenanceFacet(contact="a@b.c")) + merged = merge_bags([a, b]) + assert merged.provenance.contact == "a@b.c" + + +class TestMergeStructure: + def test_ducklake_columns_win(self): + duck_col = Column("DT_NOTIFIC", "", VARCHAR) + saude_col = Column("DT_NOTIFIC", "saude col", VARCHAR) + a = _bag("saude", structure=StructureFacet(columns=[saude_col])) + b = _bag("ducklake", structure=StructureFacet(columns=[duck_col])) + merged = merge_bags([a, b]) + assert merged.structure.columns == [duck_col] + + def test_row_count_max(self): + a = _bag("ftp", structure=StructureFacet(row_count=100)) + b = _bag("ducklake", structure=StructureFacet(row_count=500)) + merged = merge_bags([a, b]) + assert merged.structure.row_count == 500 + + +class TestMergeAccess: + def test_requires_auth_or(self): + a = _bag("ftp", access=AccessFacet(requires_auth=False)) + b = _bag("dadosgov", access=AccessFacet(requires_auth=True)) + merged = merge_bags([a, b]) + assert merged.access.requires_auth is True + + def test_size_first_non_zero(self): + a = _bag("ftp", access=AccessFacet(size_bytes=0)) + b = _bag("saude", access=AccessFacet(size_bytes=1024)) + merged = merge_bags([a, b]) + assert merged.access.size_bytes == 1024 + + +class TestMergeQuality: + def test_integrity_or(self): + a = _bag("ftp", quality=QualityFacet(integrity_verified=False)) + b = _bag("ducklake", quality=QualityFacet(integrity_verified=True)) + merged = merge_bags([a, b]) + assert merged.quality.integrity_verified is True + + def test_fingerprint_first_non_empty(self): + a = _bag("ftp", quality=QualityFacet(content_fingerprint="")) + b = _bag("ducklake", quality=QualityFacet(content_fingerprint="sha")) + merged = merge_bags([a, b]) + assert merged.quality.content_fingerprint == "sha" + + +class TestMergeEdgeCases: + def test_single_bag_identity(self): + bag = _bag("saude") + assert merge_bags([bag]) is bag + + def test_no_bags(self): + assert merge_bags([]) == MetadataBag() + + def test_bag_merge_method(self): + a = _bag("ftp", description=DescriptionFacet(title="ftp")) + b = _bag("saude", description=DescriptionFacet(title="saude")) + merged = a.merge(b) + assert merged.description.title == "saude" diff --git a/pysus/tests/api/metadata/test_base_classes.py b/pysus/tests/api/metadata/test_base_classes.py new file mode 100644 index 00000000..1dfdcb48 --- /dev/null +++ b/pysus/tests/api/metadata/test_base_classes.py @@ -0,0 +1,233 @@ +"""Tests for the ``.metadata`` wiring on the base classes.""" + +from __future__ import annotations + +from pathlib import Path +from typing import ClassVar + +import pytest +from pysus.api.metadata.extractors import MetadataExtractor +from pysus.api.metadata.models import ( + DescriptionFacet, + MetadataBag, + ProvenanceFacet, +) +from pysus.api.models import ( + BaseRemoteClient, + BaseRemoteDataset, + BaseRemoteFile, + BaseRemoteGroup, +) + + +class _TitleExtractor(MetadataExtractor): + """Test extractor that emits a description with the entity name.""" + + origin = "test" + + def _extract(self, obj) -> MetadataBag: + return MetadataBag( + description=DescriptionFacet(title=obj.name), + provenance=ProvenanceFacet(origin=self.origin), + ) + + +class _FailingExtractor(MetadataExtractor): + """Extractor that always raises — must be ignored.""" + + origin = "test" + + def _extract(self, obj) -> MetadataBag: + raise RuntimeError("boom") + + +class _TestFile(BaseRemoteFile): + extractor_types: ClassVar[list] = [_TitleExtractor] + + def __init__(self, name: str = "file.txt"): + super().__init__( + path=Path("/tmp") / name, + type="FILE", + dataset=_TestDataset(), + ) + + @property + def extension(self) -> str: + return ".txt" + + @property + def size(self) -> int: + return 10 + + @property + def modify(self): + from datetime import datetime + + return datetime(2026, 1, 1) + + async def _download(self, output=None, callback=None): + return output + + +class _TestGroup(BaseRemoteGroup): + extractor_types: ClassVar[list] = [_TitleExtractor] + + def __init__(self): + super().__init__(dataset=_TestDataset()) + + @property + def name(self) -> str: + return "group-1" + + @property + def long_name(self) -> str: + return "Group 1" + + @property + def description(self) -> str: + return "" + + async def _fetch_files(self): + return [] + + +class _TestDataset(BaseRemoteDataset): + extractor_types: ClassVar[list] = [_TitleExtractor] + + def __init__(self): + super().__init__(client=_TestClient()) + + @property + def name(self) -> str: + return "SINAN" + + @property + def long_name(self) -> str: + return "Sistema de Informação" + + @property + def description(self) -> str: + return "" + + async def _fetch_content(self): + return [] + + +class _TestClient(BaseRemoteClient): + extractor_types: ClassVar[list] = [_TitleExtractor] + + @property + def name(self) -> str: + return "TestClient" + + @property + def long_name(self) -> str: + return "Test Client" + + @property + def description(self) -> str: + return "" + + async def connect(self): + pass + + async def close(self): + pass + + async def login(self, **kwargs): + pass + + async def datasets(self, **kwargs): + return [] + + async def download(self, file, output, callback=None): + return output + + +class TestFileMetadata: + def test_returns_merged_bag(self): + file = _TestFile(name="x.txt") + bag = file.metadata + assert isinstance(bag, MetadataBag) + assert bag.description.title == "x.txt" + assert bag.provenance.origin == "test" + + def test_result_is_cached(self): + file = _TestFile(name="x.txt") + first = file.metadata + second = file.metadata + assert first is second + + +class TestGroupMetadata: + def test_returns_bag(self): + group = _TestGroup() + bag = group.metadata + assert bag.description.title == "group-1" + + +class TestDatasetMetadata: + def test_returns_bag(self): + dataset = _TestDataset() + bag = dataset.metadata + assert bag.description.title == "SINAN" + + +class TestClientMetadata: + def test_returns_bag(self): + client = _TestClient() + bag = client.metadata + assert bag.description.title == "TestClient" + + +class TestFailingExtractor: + def test_failures_are_ignored(self): + class _FailingFile(_TestFile): + extractor_types: ClassVar[list] = [ + _FailingExtractor, + _TitleExtractor, + ] + + file = _FailingFile(name="x.txt") + bag = file.metadata + # The failing extractor is skipped; the title extractor wins. + assert bag.description.title == "x.txt" + + +class TestNoExtractors: + def test_empty_bag_when_no_extractors(self): + class _BareFile(BaseRemoteFile): + def __init__(self): + super().__init__( + path=Path("/tmp") / "x", + type="FILE", + dataset=_TestDataset(), + ) + + @property + def extension(self) -> str: + return "" + + @property + def size(self) -> int: + return 0 + + @property + def modify(self): + from datetime import datetime + + return datetime(2026, 1, 1) + + async def _download(self, output=None, callback=None): + return output + + file = _BareFile() + assert file.metadata == MetadataBag() + + +class TestAsyncMetadata: + @pytest.mark.asyncio + async def test_ametadata_delegates_to_extract(self): + file = _TestFile(name="x.txt") + bag = await file.ametadata() + assert bag.description.title == "x.txt" diff --git a/pysus/tests/api/metadata/test_extractors.py b/pysus/tests/api/metadata/test_extractors.py new file mode 100644 index 00000000..d459fad9 --- /dev/null +++ b/pysus/tests/api/metadata/test_extractors.py @@ -0,0 +1,185 @@ +"""Tests for the per-client metadata extractors.""" + +from __future__ import annotations + +import json +import pathlib +from datetime import datetime + +import pytest +from pysus.api.dadosgov.metadata import ( + DadosGovDatasetExtractor, + DadosGovFileExtractor, + DadosGovGroupExtractor, +) +from pysus.api.ducklake.metadata import ( + DuckLakeDatasetExtractor, + DuckLakeFileExtractor, + DuckLakeGroupExtractor, +) +from pysus.api.ftp.metadata import ( + FtpDatasetExtractor, + FtpFileExtractor, + FtpGroupExtractor, +) +from pysus.api.metadata.models import MetadataBag +from pysus.api.saude.metadata import ( + SaudeDatasetExtractor, + SaudeFileExtractor, + SaudeGroupExtractor, +) + +_SAUDE_FIXTURES = pathlib.Path(__file__).parent.parent / "saude" / "fixtures" + + +@pytest.fixture(scope="module") +def saude_dataset_page_props() -> dict: + payload = json.loads( + (_SAUDE_FIXTURES / "dataset_arboviroses-dengue.json").read_text() + ) + return payload["pageProps"] + + +class _Stub: + """Minimal stand-in for the concrete client model classes.""" + + def __init__(self, **attrs): + self.__dict__.update(attrs) + + +class TestSaudeExtractors: + def test_dataset_extractor(self, saude_dataset_page_props): + from pysus.api.saude.resources import CKANPackage + + package = CKANPackage.model_validate(saude_dataset_page_props) + bag = SaudeDatasetExtractor().extract(package) + assert isinstance(bag, MetadataBag) + assert bag.provenance.origin == "saude" + assert bag.identity.name == "arboviroses-dengue" + assert bag.identity.cross_origin_id == package.id + assert bag.description.title == "Sinan/Dengue" + assert bag.description.themes == ["Arboviroses"] + assert bag.temporal.periodicity == "Semanal" + assert bag.provenance.contact == "arboviroses@saude.gov.br" + assert bag.provenance.license_id == "cc-by" + assert bag.structure.file_count == 83 + + def test_file_extractor(self, saude_dataset_page_props): + from pysus.api.saude.resources import CKANPackage + + package = CKANPackage.model_validate(saude_dataset_page_props) + resource = next(r for r in package.resources if r.format == "CSV") + bag = SaudeFileExtractor().extract(resource) + assert bag.provenance.origin == "saude" + assert bag.identity.name == resource.name + assert bag.access.format == "CSV" + assert bag.access.url == resource.url + assert "position" in bag.raw + + def test_group_extractor(self): + from pysus.api.saude.resources import GroupRef + + group = GroupRef(name="arboviroses", display_name="Arboviroses") + bag = SaudeGroupExtractor().extract(group) + assert bag.provenance.origin == "saude" + assert bag.identity.name == "arboviroses" + assert bag.description.title == "Arboviroses" + + +class TestDadosGovExtractors: + def test_file_extractor(self): + file = _Stub( + basename="DENGBR25.csv.zip", + path="/x/DENGBR25.csv.zip", + extension=".zip", + size=1234, + modify=datetime(2026, 1, 1), + year=2025, + month=3, + state=None, + ) + bag = DadosGovFileExtractor().extract(file) + assert bag.provenance.origin == "dadosgov" + assert bag.identity.name == "DENGBR25.csv.zip" + assert bag.temporal.year == 2025 + assert bag.temporal.month == 3 + assert bag.access.size_bytes == 1234 + assert bag.access.requires_auth is True + + def test_dataset_extractor(self): + ds = _Stub( + name="SINAN", long_name="Sistema de Informação", description="x" + ) + bag = DadosGovDatasetExtractor().extract(ds) + assert bag.identity.name == "SINAN" + assert bag.description.title == "Sistema de Informação" + + def test_group_extractor(self): + group = _Stub(name="DENG", long_name="Dengue", description="") + bag = DadosGovGroupExtractor().extract(group) + assert bag.identity.name == "DENG" + + +class TestFtpExtractors: + def test_file_extractor(self): + file = _Stub( + basename="DENGBR25.dbc", + path="/ftp/sinan/DENGBR25.dbc", + extension=".dbc", + size=999, + modify=datetime(2025, 6, 1), + year=2025, + month=None, + state="BR", + ) + bag = FtpFileExtractor().extract(file) + assert bag.provenance.origin == "ftp" + assert bag.structure.format == "dbc" + assert bag.access.download_strategy == "ftp" + assert bag.spatial.state == "BR" + + def test_dataset_extractor(self): + ds = _Stub(name="SINAN", long_name="Sistema", description="") + bag = FtpDatasetExtractor().extract(ds) + assert bag.identity.name == "SINAN" + + def test_group_extractor(self): + group = _Stub(name="DENG", long_name="Dengue", description="") + bag = FtpGroupExtractor().extract(group) + assert bag.identity.name == "DENG" + + +class TestDuckLakeExtractors: + def test_file_extractor(self): + record = _Stub( + rows=1000, + modified=datetime(2026, 2, 1), + sha256="deadbeef", + year=2025, + month=None, + state="BR", + type="PARQUET", + ) + file = _Stub( + basename="DENGBR25.parquet", + path="public/data/ftp/sinan/DENGBR25.parquet", + record=record, + size=2048, + modify=datetime(2026, 2, 1), + ) + bag = DuckLakeFileExtractor().extract(file) + assert bag.provenance.origin == "ducklake" + assert bag.structure.row_count == 1000 + assert bag.quality.content_fingerprint == "deadbeef" + assert bag.quality.integrity_verified is True + assert bag.access.download_strategy == "s3" + + def test_dataset_extractor(self): + ds = _Stub(name="sinan", long_name="Sistema", description="") + bag = DuckLakeDatasetExtractor().extract(ds) + assert bag.identity.name == "sinan" + + def test_group_extractor(self): + group = _Stub(name="DENG", long_name="Dengue", description="") + bag = DuckLakeGroupExtractor().extract(group) + assert bag.identity.name == "DENG" diff --git a/pysus/tests/api/saude/__init__.py b/pysus/tests/api/saude/__init__.py new file mode 100644 index 00000000..9340975d --- /dev/null +++ b/pysus/tests/api/saude/__init__.py @@ -0,0 +1,6 @@ +"""Offline fixtures for the Saude client tests. + +These are real captured payloads from dadosabertos.saude.gov.br (captured +2026-08-17). They keep the test suite network-free while exercising the +exact response shapes the portal returns. +""" diff --git a/pysus/tests/api/saude/conftest.py b/pysus/tests/api/saude/conftest.py new file mode 100644 index 00000000..114606db --- /dev/null +++ b/pysus/tests/api/saude/conftest.py @@ -0,0 +1,104 @@ +"""pytest fixtures for the Saude client. + +Uses ``httpx.MockTransport`` to serve the captured payloads so the test +suite runs entirely offline. +""" + +from __future__ import annotations + +import json +import pathlib +from typing import Any + +import httpx +import pytest + +FIXTURES = pathlib.Path(__file__).parent / "fixtures" + + +def _load(name: str) -> bytes: + return (FIXTURES / name).read_bytes() + + +def _build_mock_transport() -> httpx.MockTransport: + """Return a transport that serves the captured fixtures.""" + + homepage = _load("homepage.html") + catalog_page1 = _load("catalog_page1.json") + dataset_dengue = _load("dataset_arboviroses-dengue.json") + resource_zip = _load("dengue_2024.csv.zip") + + def handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + path = request.url.path + if path in ("/", "") or url.endswith("dadosabertos.saude.gov.br"): + return httpx.Response(200, content=homepage) + if path.endswith("/dataset.json"): + return httpx.Response(200, content=catalog_page1) + if path.endswith("/dataset/arboviroses-dengue.json"): + return httpx.Response(200, content=dataset_dengue) + if "ckan.saude.gov.br" in url: + return httpx.Response(200, content=resource_zip) + return httpx.Response(404, content=b"not found: " + url.encode()) + + return httpx.MockTransport(handler) + + +@pytest.fixture +def mocked_saude() -> httpx.MockTransport: + """A mock transport that serves the captured fixtures.""" + return _build_mock_transport() + + +@pytest.fixture +def saude_client(tmp_path: pathlib.Path) -> Any: + """A ready-to-use SaudeClient wired to the mock transport.""" + import asyncio + + from pysus.api.saude import SaudeClient + + client = SaudeClient(cache_dir=tmp_path, timeout=10.0) + client._client = httpx.AsyncClient( + transport=_build_mock_transport(), timeout=10.0 + ) + yield client + try: + loop = asyncio.get_running_loop() + except RuntimeError: + asyncio.run(client.close()) + else: + loop.create_task(client.close()) + + +@pytest.fixture +def saude_homepage_html() -> str: + """Raw homepage HTML for buildId-extraction tests.""" + return _load("homepage.html").decode("utf-8") + + +@pytest.fixture +def saude_catalog_payload() -> dict[str, Any]: + """Parsed ``dataset.json?page=1`` payload.""" + return json.loads(_load("catalog_page1.json")) + + +@pytest.fixture +def saude_dataset_payload() -> dict[str, Any]: + """Parsed ``dataset/arboviroses-dengue.json`` payload.""" + return json.loads(_load("dataset_arboviroses-dengue.json")) + + +@pytest.fixture +def saude_dataset_page_props( + saude_dataset_payload: dict[str, Any] +) -> dict[str, Any]: + """Just the ``pageProps`` of the dengue dataset payload.""" + return saude_dataset_payload["pageProps"] + + +@pytest.fixture +def saude_resource_zip_path(tmp_path: pathlib.Path) -> pathlib.Path: + """Path to the (synthetic) dengue CSV.ZIP fixture.""" + target = tmp_path / "dengue_2024.csv.zip" + target.write_bytes(_load("dengue_2024.csv.zip")) + return target diff --git a/pysus/tests/api/saude/fixtures/catalog_page1.json b/pysus/tests/api/saude/fixtures/catalog_page1.json new file mode 100644 index 00000000..ce125362 --- /dev/null +++ b/pysus/tests/api/saude/fixtures/catalog_page1.json @@ -0,0 +1 @@ +{"pageProps":{"currentFilters":{"q":null,"groups":null,"tags":null,"res_format":null},"availableFilters":{"groups":[{"display_name":"Arboviroses","name":"arboviroses"},{"display_name":"Assistência à saúde","name":"assistencia-a-saude"},{"display_name":"Assistência Farmacêutica","name":"assistencia-farmaceutica"},{"display_name":"Atenção Primária","name":"atencao-primaria"},{"display_name":"Ciência & Tecnologia","name":"ciencia-tecnologia"},{"display_name":"Diagnósticos e Tratamentos","name":"diagnosticos-e-tratamentos"},{"display_name":"Economia da Saúde","name":"economia-da-saude"},{"display_name":"Educação em Saúde","name":"educacao-em-saude"},{"display_name":"Indicadores de saúde","name":"indicadores-de-saude"},{"display_name":"Prevenção e Promoção da Saúde","name":"prevencao-e-promocao-da-saude"},{"display_name":"Saúde Digital","name":"pda"},{"display_name":"Saúde Indígena","name":"saude-indigena"},{"display_name":"Vacinação","name":"vacinacao"},{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[{"display_name":"acidentes","name":"acidentes"},{"display_name":"Acidentes","name":"Acidentes"},{"display_name":"Acompanhamento Gestacional","name":"Acompanhamento Gestacional"},{"display_name":"Aldeias indígenas","name":"Aldeias indígenas"},{"display_name":"Alimentos ultraprocessados","name":"Alimentos ultraprocessados"},{"display_name":"Amamentação","name":"Amamentação"},{"display_name":"Antropometria","name":"Antropometria"},{"display_name":"API","name":"API"},{"display_name":"Arboviroses","name":"Arboviroses"},{"display_name":"Assistência Farmacêutica no Subsistema de Atenção à Saúde Indígena - SasiSUS","name":"Assistência Farmacêutica no Subsistema de Atenção à Saúde Indígena - SasiSUS"},{"display_name":"Atenção Primária","name":"Atenção Primária"},{"display_name":"Autonomia","name":"Autonomia"},{"display_name":"Cadastro Vinculado","name":"Cadastro Vinculado"},{"display_name":"Caxumba","name":"Caxumba"},{"display_name":"chikungunya","name":"chikungunya"},{"display_name":"CNES","name":"CNES"},{"display_name":"Compras Públicas","name":"Compras Públicas"},{"display_name":"Condições","name":"Condições"},{"display_name":"Confirmação do Caso","name":"Confirmação do Caso"},{"display_name":"Conitec","name":"Conitec"},{"display_name":"Consultas Públicas","name":"Consultas Públicas"},{"display_name":"Consumo alimentar.","name":"Consumo alimentar."},{"display_name":"Coqueluche","name":"Coqueluche"},{"display_name":"Corona","name":"Corona"},{"display_name":"covid-19","name":"covid-19"},{"display_name":"COVID-19","name":"COVID-19"},{"display_name":"Crescimento e Desenvolvimento Infantil","name":"Crescimento e Desenvolvimento Infantil"},{"display_name":"Crianças menores de cinco anos","name":"Crianças menores de cinco anos"},{"display_name":"Crianças menores de cinco anos.","name":"Crianças menores de cinco anos."},{"display_name":"Dados Demográficos","name":"Dados Demográficos"},{"display_name":"Dados vacinais","name":"Dados vacinais"},{"display_name":"DECIT","name":"DECIT"},{"display_name":"dengue","name":"dengue"},{"display_name":"Diagnósticos","name":"Diagnósticos"},{"display_name":"Difteria","name":"Difteria"},{"display_name":"Doenças Diarreicas Agudas","name":"Doenças Diarreicas Agudas"},{"display_name":"DPNI","name":"DPNI"},{"display_name":"Dt","name":"Dt"},{"display_name":"e-SUS","name":"e-SUS"},{"display_name":"Esgotamento Sanitário","name":"Esgotamento Sanitário"},{"display_name":"estabelecimentos","name":"estabelecimentos"},{"display_name":"Estado Nutricional","name":"Estado Nutricional"},{"display_name":"Estado Nutricional.","name":"Estado Nutricional."},{"display_name":"Ética em Pesquisa","name":"Ética em Pesquisa"},{"display_name":"EVENTOS SUPOSTAMENTE ATRIBUÍVEIS À VACINAÇÃO OU IMUNIZAÇÃO","name":"EVENTOS SUPOSTAMENTE ATRIBUÍVEIS À VACINAÇÃO OU IMUNIZAÇÃO"},{"display_name":"Evolução do Caso","name":"Evolução do Caso"},{"display_name":"febre amarela","name":"febre amarela"},{"display_name":"Febre Amarela","name":"Febre Amarela"},{"display_name":"Fomento à Pesquisa","name":"Fomento à Pesquisa"},{"display_name":"Frutas","name":"Frutas"},{"display_name":"Haemophilus Influezae tipo b","name":"Haemophilus Influezae tipo b"},{"display_name":"HeP B","name":"HeP B"},{"display_name":"Hepatite A","name":"Hepatite A"},{"display_name":"Hepatite B ao nascer","name":"Hepatite B ao nascer"},{"display_name":"Hepatites Virais","name":"Hepatites Virais"},{"display_name":"Hib","name":"Hib"},{"display_name":"Hospital","name":"Hospital"},{"display_name":"HPV","name":"HPV"},{"display_name":"INAEP","name":"INAEP"},{"display_name":"Indicadores Antropométricos","name":"Indicadores Antropométricos"},{"display_name":"Indicadores de desempenho","name":"Indicadores de desempenho"},{"display_name":"Infecção meningocócica","name":"Infecção meningocócica"},{"display_name":"Influenza","name":"Influenza"},{"display_name":"Inquéritos Epidemiológicos","name":"Inquéritos Epidemiológicos"},{"display_name":"Inquéritos Epidemiológicos.","name":"Inquéritos Epidemiológicos."},{"display_name":"Legumes e Verduras","name":"Legumes e Verduras"},{"display_name":"Leishmaniose","name":"Leishmaniose"},{"display_name":"Leito","name":"Leito"},{"display_name":"Mais Médicos","name":"Mais Médicos"},{"display_name":"Meningite","name":"Meningite"},{"display_name":"Meningite viral","name":"Meningite viral"},{"display_name":"Meningo ACWY","name":"Meningo ACWY"},{"display_name":"Meningocócica C","name":"Meningocócica C"},{"display_name":"Mortalidade indígena","name":"Mortalidade indígena"},{"display_name":"Mpox","name":"Mpox"},{"display_name":"nascidos vivos","name":"nascidos vivos"},{"display_name":"Notifica","name":"Notifica"},{"display_name":"ofídicos","name":"ofídicos"},{"display_name":"Outros tipos de tétano","name":"Outros tipos de tétano"},{"display_name":"Pandemia","name":"Pandemia"},{"display_name":"PCDT","name":"PCDT"},{"display_name":"Pentavalente DTP","name":"Pentavalente DTP"},{"display_name":"Pesquisa","name":"Pesquisa"},{"display_name":"Pesquisa Saúde","name":"Pesquisa Saúde"},{"display_name":"Plataforma Brasil","name":"Plataforma Brasil"},{"display_name":"PMM","name":"PMM"},{"display_name":"Pneumocócica 10","name":"Pneumocócica 10"},{"display_name":"Pneumocócica 23 valente","name":"Pneumocócica 23 valente"},{"display_name":"Pneumonia","name":"Pneumonia"},{"display_name":"PNI","name":"PNI"},{"display_name":"Poliomelite","name":"Poliomelite"},{"display_name":"Povos Indígenas","name":"Povos Indígenas"},{"display_name":"Pré-Natal","name":"Pré-Natal"},{"display_name":"Preços","name":"Preços"},{"display_name":"Previne Brasil","name":"Previne Brasil"},{"display_name":"Programa De Volta Para Casa","name":"Programa De Volta Para Casa"},{"display_name":"provimento médico","name":"provimento médico"},{"display_name":"raiva","name":"raiva"},{"display_name":"Rede de Atenção Psicossocial","name":"Rede de Atenção Psicossocial"},{"display_name":"Reinserção","name":"Reinserção"},{"display_name":"Resíduos sólidos","name":"Resíduos sólidos"},{"display_name":"Ripsa","name":"Ripsa"},{"display_name":"Rotavírus","name":"Rotavírus"},{"display_name":"Rubéola","name":"Rubéola"},{"display_name":"Saneamento","name":"Saneamento"},{"display_name":"Sarampo","name":"Sarampo"},{"display_name":"Saúde","name":"Saúde"},{"display_name":"Saúde Bucal","name":"Saúde Bucal"},{"display_name":"Saúde da Criança","name":"Saúde da Criança"},{"display_name":"Saúde da Mulher","name":"Saúde da Mulher"},{"display_name":"Saúde Indígena","name":"Saúde Indígena"},{"display_name":"SCPA","name":"SCPA"},{"display_name":"SIASI","name":"SIASI"},{"display_name":"Sinan","name":"Sinan"},{"display_name":"SINASC","name":"SINASC"},{"display_name":"Síndrome Gripal","name":"Síndrome Gripal"},{"display_name":"SIPNI","name":"SIPNI"},{"display_name":"Sisab","name":"Sisab"},{"display_name":"SISAGUA","name":"SISAGUA"},{"display_name":"Sistema de Gerenciamento de Recursos Humanos da SESAI -SESAI RH. Gerenciamento da força de trabalho","name":"Sistema de Gerenciamento de Recursos Humanos da SESAI -SESAI RH. Gerenciamento da força de trabalho"},{"display_name":"Sistema de Informação da Atenção à Saúde Indígena - SIASI","name":"Sistema de Informação da Atenção à Saúde Indígena - SIASI"},{"display_name":"Sistema de Informação da Atenção à Saúde Indígena - SIASI - Dados Demográficos","name":"Sistema de Informação da Atenção à Saúde Indígena - SIASI - Dados Demográficos"},{"display_name":"SRAG","name":"SRAG"},{"display_name":"Subsistema de Atenção à Saúde Indígena - SasiSUS","name":"Subsistema de Atenção à Saúde Indígena - SasiSUS"},{"display_name":"Subsistema de Atenção à Saúde Indígena -SasiSUS","name":"Subsistema de Atenção à Saúde Indígena -SasiSUS"},{"display_name":"SVS","name":"SVS"},{"display_name":"SVSA","name":"SVSA"},{"display_name":"Tecnologias Demandadas","name":"Tecnologias Demandadas"},{"display_name":"Testes","name":"Testes"},{"display_name":"Tétano do recém-nascido ou tétano neonatal","name":"Tétano do recém-nascido ou tétano neonatal"},{"display_name":"Tétano Obstétrico","name":"Tétano Obstétrico"},{"display_name":"Tetraviral SCR","name":"Tetraviral SCR"},{"display_name":"Tríplice Viral SCR","name":"Tríplice Viral SCR"},{"display_name":"Tuberculose","name":"Tuberculose"},{"display_name":"Vacinação","name":"Vacinação"},{"display_name":"vagas","name":"vagas"},{"display_name":"Varicela","name":"Varicela"},{"display_name":"Varicela catapora","name":"Varicela catapora"},{"display_name":"Varicela e BCG","name":"Varicela e BCG"},{"display_name":"Variola","name":"Variola"},{"display_name":"Vigilância Alimentar e Nutricional - VAN","name":"Vigilância Alimentar e Nutricional - VAN"},{"display_name":"Vigilância Epidemiológica","name":"Vigilância Epidemiológica"},{"display_name":"VIP Poliomielite inativada","name":"VIP Poliomielite inativada"},{"display_name":"Vírus","name":"Vírus"},{"display_name":"zika","name":"zika"},{"display_name":"zoonoses","name":"zoonoses"}],"res_format":[{"display_name":"API","name":"API"},{"display_name":"CSV","name":"CSV"},{"display_name":"DO","name":"DO"},{"display_name":"DOC","name":"DOC"},{"display_name":"DOCX","name":"DOCX"},{"display_name":"DTA","name":"DTA"},{"display_name":"JSON","name":"JSON"},{"display_name":"ODT","name":"ODT"},{"display_name":"parquet","name":"parquet"},{"display_name":"PDF","name":"PDF"},{"display_name":"R","name":"R"},{"display_name":"RDF","name":"RDF"},{"display_name":"RDS","name":"RDS"},{"display_name":"XLS","name":"XLS"},{"display_name":"XLSX","name":"XLSX"},{"display_name":"XML","name":"XML"},{"display_name":"zip csv","name":"zip csv"}]},"numberOfPackages":138,"packages":[{"name":"tuberculose_sesai","title":"Sistema de Informação da Atenção à Saúde Indígena (Siasi) – Módulo de Morbidades (Tuberculose)","notes":"Dados de Tuberculose ocorridos na população indígena assistida pelo SasiSUS em 2022. ","formats":["PDF","CSV","JSON","XML"],"groups":[{"display_name":"Saúde Indígena","name":"saude-indigena"}],"tags":[{"display_name":"Tuberculose","name":"Tuberculose"}]},{"name":"siasi_banco_obitos","title":"Sistema de Informação da Atenção à Saúde Indígena (Siasi) – Banco de Óbitos","notes":"Registro de dados individualizados e desidentificados de óbitos gerais, não fetais, ocorridos na população indígena assistida pelo Subsistema de Saúde Indígena (SasiSUS), referente ao ano de 2022. ","formats":["PDF","CSV","JSON","XML"],"groups":[{"display_name":"Saúde Indígena","name":"saude-indigena"}],"tags":[{"display_name":"Mortalidade indígena","name":"Mortalidade indígena"}]},{"name":"bps","title":"Banco de Preços em Saúde - BPS","notes":"O Banco de Preço em Saúde - BPS é um sistema de registro de informações de compras públicas e privadas de medicamentos e dispositivos médicos. Sua principal finalidade é possibilitar o uso de informações de compras públicas e privadas de medicamentos e dispositivos médicos, a fim de subsidiar a compra pública mais eficiente no setor saúde, pelos entes federados e instituições de saúde.","formats":["PDF","API","CSV","JSON","XML"],"groups":[{"display_name":"Economia da Saúde","name":"economia-da-saude"}],"tags":[{"display_name":"Compras Públicas","name":"Compras Públicas"},{"display_name":"Preços","name":"Preços"},{"display_name":"Saúde","name":"Saúde"}]},{"name":"srag-2019-a-2026","title":"Banco de dados da Síndrome Respiratória Aguda Grave (SRAG) - 2019 a 2026","notes":" Vigilância da SRAG em 2019\r\nAntes da pandemia de covid-19, a vigilância de vírus respiratórios no Brasil era voltada principalmente para o vírus influenza. Desde 2000, o país contava com uma rede de vigilância sentinela para monitorar casos de síndrome gripal (SG) e, a partir de 2009, também para síndrome respiratória aguda grave (SRAG). As informações vinham de unidades de saúde e hospitais que coletavam amostras e notificavam casos ao Sistema de Vigilância Epidemiológica da Gripe (Sivep-Gripe). Esse trabalho era apoiado pela Rede Nacional de Laboratórios de Saúde Pública, que identificava os vírus e ajudava a orientar campanhas de vacinação e outras medidas de prevenção.\r\n\r\n Vigilância da SRAG a partir de 2020\r\nCom a chegada da covid-19, a vigilância foi ampliada e passou a incluir outros vírus respiratórios de importância em saúde pública. Surgiu o sistema e-SUS Notifica para registrar casos leves e moderados de covid-19, enquanto o Sivep-Gripe manteve o registro dos casos graves e óbitos. A análise passou a ser mais integrada, usando dados epidemiológicos, laboratoriais e genômicos, e novas ferramentas ajudaram a acompanhar a circulação dos vírus em tempo real. Essa mudança tornou o monitoramento mais rápido, preciso e preparado para responder a surtos e pandemias no futuro.\r\n\r\n Apresentação dos Dados\r\n- Neste repositório estão disponibilizados, de forma pública, dados referentes ao Sistema de Informação da Vigilância Epidemiológica da Gripe (Sivep-Gripe), atualmente o Sistema oficial para o registro dos casos e óbitos por SRAG no Brasil.\r\n- Aqui você encontrará arquivos dos bancos de dados organizados por ano epidemiológico: de 2019 a 2024 os bancos estão “congelados”, ou seja, não estão mais sendo atualizados; e o banco “vivo” correspondente ao ano atual, atualizado semanalmente. Também está disponibilizado o dicionário de variáveis e a ficha de notificação.\r\n\r\n Scripts para Análises e Dados Unificados\r\nhttps://gitlab.com/cgcovid/dados-abertos/-/tree/main\r\n\r\nScripts em R: Este repositório disponibiliza publicamente os scripts em R para análise de dados de SRAG, padronizados pela equipe técnica da CGCOVID/DEDT/SVSA do Ministério da Saúde. Esses scripts são utilizados tanto na elaboração das análises apresentadas no painel da SRAG quanto na produção dos informes epidemiológicos semanais.\r\n\r\nDados unificados: Neste repositório estão disponibilizados de forma pública os dados referentes aos Sistemas de Informação SIVEP-Gripe e e-SUS Notifica que alimentam, respectivamente, os painéis de Síndrome Respiratória Aguda Grave (Srag) e Síndrome Gripal (SG) por covid-19 do Ministério da Saúde. Os dados estão disponíveis de forma agregada por Unidade Federada, Município, Faixa Etária e Semana Epidemiológica.\r\n\r\n Observações\r\n As notificações que alimentam Sivep-Gripe estão sujeitas a possíveis erros de digitação ou preenchimento, que podem afetar um ou mais campos dos registros. Por esse motivo, as notificações passam por revisões contínuas, realizadas pelas equipes locais e pela rede de vigilância, com o objetivo de corrigir inconsistências e assegurar a qualidade das informações.\r\n As bases de dados de SRAG disponibilizadas neste portal passam por tratamento que envolve a anonimização, em cumprimento a Lei Geral de Proteção de Dados Pessoais (LGPD), Lei 13.709/2018.\r\n\r\n Para Saber Mais\r\n Boletins epidemiológicos: https://www.gov.br/saude/pt-br/centrais-de-conteudo/publicacoes/boletins/epidemiologicos/covid-19\r\n Definições de casos, critérios de confirmação e encerramento dos casos, acesse o Guia de vigilância integrada da covid-19, influenza e outros vírus respiratórios de importância em saúde pública (2024) pelo link: https://www.gov.br/saude/pt-br/centrais-de-conteudo/publicacoes/guias-e-manuais/2024/guia-vigilancia-integrada-da-covid-19-influenza-e-outros-virus-respiratorios-de-importancia-em-saude-publica/view\r\n Informes semanais com a atualização dos Casos de Influenza, Covid-19 e outros Vírus Respiratórios: https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/c/covid-19/publicacoes-tecnicas/informes https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/c/covid-19/atualizacao-de-casos)\r\n Notas Técnicas conjuntas entre áreas técnicas sobre Cenário epidemiológico dos casos de Síndrome Respiratória Aguda Grave (SRAG): https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/c/covid-19/notas-tecnicas/2025\r\n Painel informativo do cenário epidemiológico da vigilância universal da síndrome gripal pela Covid-19: https://www.gov.br/saude/pt-br/composicao/svsa/cnie/sindrome-gripal-pela-covid-19\r\n Painel informativo do cenário epidemiológico da vigilância universal da Síndrome Respiratória Aguda Grave (SRAG) decorrente da Covid-19, influenza e outros vírus respiratórios: https://www.gov.br/saude/pt-br/composicao/svsa/cnie/srag\r\n Sala de Situação de Vírus Respiratórios: https://www.gov.br/saude/pt-br/composicao/svsa/resposta-a-emergencias/sala-de-situacao-de-saude/virus-respiratorios\r\n Testes: https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/c/covid-19/publicacoes-tecnicas/guias-e-planos/plano-nacional-de-expansao-da-testagem-para-covid-19.pdf/view\r\n Tratamento e manejo clínico, acesse: \r\n - Guia de orientações para profissionais de saúde, no link: https://www.gov.br/saude/pt-br/centrais-de-conteudo/publicacoes/guias-e-manuais/2025/guia-de-orientacoes-para-profissionais-de-saude-srag.pdf/view\r\n - Guia para uso do antiviral nirmatrelvir/ritonavir em pacientes com covid-19 de alto risco,2ªed.: https://bvsms.saude.gov.br/bvs/publicacoes/guia_uso_nirmatrelvir_ritonavir_covid19.pdf","formats":["PDF","CSV","JSON","","XML","API","parquet"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[{"display_name":"SRAG","name":"SRAG"}]},{"name":"sisagua-controle-mensal-infraestrutura-operacional","title":"SISAGUA - Controle Mensal - Infraestrutura Operacional","notes":"Dados sobre as condições operacionais e de infraestrutura dos sistemas e soluções de abastecimento de água para consumo humano, informados pelo prestador de serviço em frequência mensal.","formats":["ODT","API","CSV","JSON","XML"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[]},{"name":"sisagua-controle-mensal-amostras-fora-do-padrao","title":"SISAGUA - Controle mensal - Amostras fora do padrão","notes":"Dados detalhados sobre as análises de qualidade da água de responsabilidade dos prestadores de serviço que não atenderam ao padrão de potabilidade, informados em frequência mensal.","formats":["ODT","API","CSV","JSON","XML"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[]},{"name":"sisagua-controle-mensal-demais-parametros","title":"SISAGUA - Controle mensal - demais parâmetros","notes":"Dados das análises de qualidade da água de média/alta complexidade, realizado pelas instituições responsáveis por SAA e SAC, em frequência mensal ou inferior.","formats":["ODT","API","CSV","JSON","XML"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[]},{"name":"sisagua-controle-mensal-parametros-basicos","title":"SISAGUA - Controle Mensal - Parâmetros básicos","notes":"Dados das análises de baixa complexidade de qualidade da água para consumo humano, realizadas rotineiramente pelos responsáveis por sistemas e soluções alternativas coletivas de abastecimento de água (SAA e SAC), e informadas mensalmente ao Sisagua.","formats":["PDF","API","CSV","JSON","XML"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[]},{"name":"registro-de-ocupacao-hospitalar-covid-19","title":"Registro de Ocupação Hospitalar COVID-19","notes":"O Ministério da Saúde, por meio da Secretaria de Atenção Especializada em Saúde (SAES), implementou, devido à pandemia, o registro das internações por meio do Sistema ESUS Notifica-Módulo Internações SUS.\r\n\r\nEsta página tem como finalidade disponibilizar o banco de dados da ocupação dos leitos, a partir da incorporação do sistema e-SUS Notifica- Módulo internações SUS, em vigor a partir de abril de 2020.\r\n\r\n Informações disponibilizadas\r\nO módulo Internações foi desenvolvido para registro da ocupação de leitos clínicos e de Unidade de Terapia Intensiva (UTI) SUS destinados para atendimento aos pacientes com casos suspeitos ou confirmados da COVID-19 (ocupação SRAG / COVID-19)\r\n\r\nAlguns estados apresentam sistemas próprios de registro de ocupação e para esses foi disponibilizado uma API que transfere os dados dos estados para o sistema ESUS NOTIFICA- Módulo Internações SUS.\r\n\r\nDevido ao grande número de registros pelos estabelecimentos de internações COVID-19, alguns estados já possuem mais de um milhão de registros, o que impossibilita estes dados de serem abertos no Excel, somente existe a disponibilização CSV.\r\n\r\nA partir do ano de 2022 foi acrescentado novos campos utilizados para descrever a ocupação dos leitos. Registros com ano anterior a 2022 não contém esses campos preenchidos.","formats":["API","CSV","JSON","XML"],"groups":[{"display_name":"Assistência à saúde","name":"assistencia-a-saude"}],"tags":[{"display_name":"Corona","name":"Corona"},{"display_name":"Dados Demográficos","name":"Dados Demográficos"},{"display_name":"Hospital","name":"Hospital"},{"display_name":"Leito","name":"Leito"},{"display_name":"Notifica","name":"Notifica"},{"display_name":"covid-19","name":"covid-19"}]},{"name":"macrorregiao-de-saude","title":"Macrorregião e Região de Saúde","notes":"Macrorregião de Saúde","formats":["API","CSV","XML","JSON"],"groups":[{"display_name":"Indicadores de saúde","name":"indicadores-de-saude"}],"tags":[]},{"name":"doses-aplicadas-pelo-programa-de-nacional-de-imunizacoes-pni-2026","title":"Doses aplicadas pelo Programa de Nacional de Imunizações (PNI) - 2026","notes":"A vacinação é reconhecida como uma estratégia mais eficaz para preservar a saúde da população e fortalecer uma sociedade saudável e resistente. Além de prevenir doenças graves, a imunização contribui para reduzir a disseminação desses agentes infecciosos na comunidade, protegendo aqueles que não podem ser vacinados por motivos de saúde. A política de vacinação é responsabilidade do Programa Nacional de Imunizações (PNI) do Ministério da Saúde. Estabelecido em 1973, o PNI desempenha um papel fundamental na promoção da saúde da população brasileira. Por meio do programa, o governo federal disponibiliza gratuitamente no Sistema Único de Saúde - SUS 47 imunobiológicos: 30 vacinas, 13 soros e 4 imunoglobulinas. Essas vacinas são ofertadas para todos os ciclos de vida conforme calendário nacional de vacinação e também com suas indicações para grupos com condições clínicas especiais. Devido ao grande volume de dados vacinais as informações são apresentadas por ano e mês de vacinação, contendo 60 variáveis a serem disponibilizados em formato aberto em planilhas de CSV.","formats":["PDF","API","CSV","JSON","XML"],"groups":[],"tags":[{"display_name":"DPNI","name":"DPNI"},{"display_name":"Dados vacinais","name":"Dados vacinais"},{"display_name":"PNI","name":"PNI"},{"display_name":"Vacinação","name":"Vacinação"}]},{"name":"sisagua-controle-mensal-plano-amostragem","title":"SISAGUA - Controle Mensal - Plano de Amostragem","notes":"Dados sobre os quantitativos mínimos de análises definidos para os prestadores de serviço, por forma de abastecimento, parâmetro de qualidade da água e ponto de monitoramento, com respectivas frequências de amostragem.","formats":["ODT","API","CSV","JSON","XML"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[]},{"name":"esavi","title":"ESAVI - Dados sobre Eventos Supostamente Atribuíveis a Vacinação","notes":"O Ministério da Saúde, por meio da Coordenação-Geral de Farmacovigilância (CGFAM) do Departamento do Programa Nacional de Imunizações (DPNI) da Secretaria de Vigilância em Saúde e Ambiente (SVSA), realizam a gestão da base nacional do Sistema de Vigilância de Eventos Supostamente Atribuíveis à Vacinação ou Imunização (SNVESAVI). O SNVESAVI tem, em última instância, a finalidade de realizar o monitoramento de segurança das vacinas e imunobiológicos utilizados no Brasil, tanto aqueles ofertados pelo Programa Nacional de Imunizações (PNI) quanto aqueles utilizados apenas em estabelecimentos privados, contribuindo para a vacinação segura e o sucesso do programa brasileiro. Para isso, os eventos ocorridos devem ser reportados diretamente ao programa por qualquer profissional de saúde que venha a ter ciência do caso.\r\n\r\nEvento supostamente atribuível à vacinação ou imunização (ESAVI) é qualquer ocorrência médica indesejada após a vacinação, não possuindo necessariamente uma relação causal com o uso de uma vacina ou outro imunobiológico (imunoglobulinas e soros heterólogos). Um ESAVI pode ser qualquer evento indesejável ou não intencional, isto é, sintoma, doença ou achado laboratorial anormal. ESAVIs graves são de notificação compulsória imediata, ou seja, devem ser notificados em até 24 horas após o atendimento e terem a investigação iniciada até 48 horas após o recebimento da notificação pela autoridade de saúde.\r\n\r\nO sistema de informação utilizado pelo Programa Nacional de Imunizações para o monitoramento dos ESAVIs no território nacional é o e-SUS Notifica – Módulo ESAVI (https://notifica.saude.gov.br). Qualquer profissional de saúde, previamente cadastrado na plataforma gov.br pode notificar ESAVIs e erros de imunização diretamente no sistema que é totalmente online e conta com campos para o preenchimento de dados relacionados a notificação, identificação do paciente, imunobiológicos utilizados, gestação, investigação, evento apresentado, antecedentes clínicos e epidemiológicos, atendimento médico, exames complementares e avaliação de causalidade. Para o registro dos dados, o formulário de notificação/investigação e encerramento de casos de ESAVI deverá ser preenchido corretamente.\r\n\r\nA base de dados gerada conta com dados individualizados e que permitem a investigação dos ESAVIs notificados e, posteriormente, a avaliação do nexo causal entre a vacinação e a ocorrência de cada evento. Além das avaliações individuais dos casos, a referida base permite a análise epidemiológica dos dados de forma a permitir o monitoramento da ocorrência dos ESAVIs na população brasileira.\r\n\r\nAs bases de dados disponibilizadas nesta plataforma referem-se aos dados individualizados e anônimos de todos os ESAVIs notificados e registrados no e-SUS Notifica – Módulo ESAVI desde janeiro de 2021. As bases anônimas serão disponibilizadas com sigilo de dados pessoais e sensíveis de modo a inviabilizar o rastreio ou identificação de qualquer cidadão, mas permite a construção e análise de indicadores da farmacovigilância de vacinas e outros imunobiológicos. Esta medida permite a cessão de dados contidos nas bases nacionais dos sistemas de informação em saúde, atendendo à Lei de Acesso à Informação e à Lei Geral de Proteção de Dados.","formats":["PDF","API","CSV","JSON","XML"],"groups":[{"display_name":"Vacinação","name":"vacinacao"}],"tags":[{"display_name":"EVENTOS SUPOSTAMENTE ATRIBUÍVEIS À VACINAÇÃO OU IMUNIZAÇÃO","name":"EVENTOS SUPOSTAMENTE ATRIBUÍVEIS À VACINAÇÃO OU IMUNIZAÇÃO"}]},{"name":"arboviroses-dengue","title":"Sinan/Dengue","notes":"O Sistema de Informação de Agravos de Notificação (Sinan) tem como objetivo coletar, transmitir e disseminar dados gerados rotineiramente pela vigilância epidemiológica das três esferas de governo, por meio de uma rede informatizada, para apoiar o processo de investigação e dar subsídios à análise das informações das doenças e dos agravos de notificação compulsória. Atualmente, o sistema possui duas versões vigentes, Sinan Online e Sinan Net. \r\n \r\nO Sinan Online visa à inserção e disseminação dos dados de notificação e investigação de dengue e de febre de chikungunya, enquanto que o Sinan Net é alimentado pela notificação e investigação da grande maioria dos agravos e doenças, que constam na Lista Nacional de Notificação Compulsória de Doenças, Agravos e Eventos de Saúde Pública, do Anexo 1 do Anexo V da Portaria de Consolidação nº 4, de 28 de setembro de 2017, que consolida as normas sobre os sistemas e os subsistemas do Sistema Único de Saúde, mas é facultado a estados e municípios incluir outros problemas de saúde importantes para o seu contexto local. \r\n \r\nDestaca-se que a dengue é doença de notificação compulsória, ou seja, todo caso suspeito e/ou confirmado deve ser obrigatoriamente notificado ao Serviço de Vigilância Epidemiológica da Secretaria Municipal de Saúde (SMS). As notificações de casos suspeitos de dengue devem ser registradas na Ficha de Notificação/Investigação da dengue e chikungunya e inseridas no Sistema de Informação de Agravos de Notificação – Sinan Online. Os óbitos suspeitos pela infecção do vírus dengue (DENV) são de notificação compulsória imediata para todas as esferas de gestão do Sistema Único de Saúde (SUS), a ser realizada em até 24 horas a partir do seu conhecimento, pelo meio de comunicação mais rápido disponível. Posteriormente, os dados devem ser inseridos no Sistema de Informação de Agravos de Notificação (Sinan Online). \r\n\r\nDengue - Notas: \r\n\r\nTodos os casos suspeitos de dengue devem ser investigados e encerrados no Sinan Online em até 60 dias da data de notificação. Para os casos prováveis de dengue são considerados todos os casos suspeitos notificados, com exceção dos casos que foram descartados. \r\n \r\nAs bases de dados de dengue aqui disponíveis podem apresentar pequenas divergências com os dados disponibilizados pela CGARB (Coordenação Geral de Vigilância de Arboviroses - CGARB). Para análise da série histórica, são utilizadas as bases de dados congeladas em data específica para cada ano analisado. Para dados recentes (2024), a atualização é feita diariamente no painel Saúde de A a Z. Dengue: https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/d/dengue/dengue. \r\n \r\nPara extração de dados sobre casos graves e óbitos de dengue (é necessário considerar as definições de classificação de casos conforme o Guia de Vigilância em Saúde, 2022. Para casos graves, deve-se aplicar filtros na classificação final, correspondentes à “dengue com complicações”, “febre hemorrágica da dengue”, “síndrome do choque da dengue”, “dengue com sinais de alarme” e “dengue grave”) e filtros no critério de confirmação, correspondentes à “laboratorial” e “clínico-epidemiológico”). Para óbitos, adicionalmente, deve-se aplicar um filtro na evolução, correspondente à “Óbito pelo agravo”.\r\n \r\nPara cálculo da incidência recomenda-se utilizar locais de residência. \r\n \r\nA partir de 2020 o estado do Espírito Santo passou a utilizar o sistema e-SUS Vigilância em Saúde. Portanto, para os casos de Arboviroses urbanas do Espírito Santo foram considerados apenas os dados disponibilizados pelo Sinan online (dengue e chikungunya) e Sinan Net (zika). \r\n\r\nOutras informações podem ser consultadas nos sítios eletrônicos:\r\nPortal Sinan \r\nhttps://portalsinan.saude.gov.br/ \r\nDengue - http://portalsinan.saude.gov.br/dengue\r\n\r\nSaúde de A a Z \r\nDengue: https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/d/dengue/dengue ","formats":["PDF","API","CSV","JSON","XML"],"groups":[{"display_name":"Arboviroses","name":"arboviroses"}],"tags":[]},{"name":"arboviroses-febre-de-chikungunya","title":"Sinan/Febre de Chikungunya","notes":"O Sistema de Informação de Agravos de Notificação (Sinan) tem como objetivo coletar, transmitir e disseminar dados gerados rotineiramente pela vigilância epidemiológica das três esferas de governo, por meio de uma rede informatizada, para apoiar o processo de investigação e dar subsídios à análise das informações das doenças e dos agravos de notificação compulsória. Atualmente, o sistema possui duas versões vigentes, Sinan Online e Sinan Net. \r\n \r\nO Sinan Online visa à inserção e disseminação dos dados de notificação e investigação de dengue e de febre de chikungunya, enquanto que o Sinan Net é alimentado pela notificação e investigação da grande maioria dos agravos e doenças, que constam na Lista Nacional de Notificação Compulsória de Doenças, Agravos e Eventos de Saúde Pública, do Anexo 1 do Anexo V da Portaria de Consolidação nº 4, de 28 de setembro de 2017, que consolida as normas sobre os sistemas e os subsistemas do Sistema Único de Saúde, mas é facultado a estados e municípios incluir outros problemas de saúde importantes para o seu contexto local. \r\n \r\nDestaca-se que a febre de chikungunya é doença de notificação compulsória, ou seja, todo caso suspeito e/ou confirmado deve ser obrigatoriamente notificado ao Serviço de Vigilância Epidemiológica da Secretaria Municipal de Saúde (SMS). As notificações de casos suspeitos de dengue devem ser registradas na Ficha de Notificação/Investigação da dengue e chikungunya e inseridas no Sistema de Informação de Agravos de Notificação – Sinan Online. \r\n\r\nFebre de Chikungunya - Notas: \r\n\r\nTodos os casos suspeitos de chikungunya devem ser investigados e encerrados no Sinan Online em até 60 dias da data de notificação. Para os casos prováveis de chikungunya são considerados todos os casos suspeitos notificados, com exceção dos casos que foram descartados.\r\nAs bases de dados de chikungunya aqui disponíveis podem apresentar pequenas divergências com os dados disponibilizados pela CGARB (Coordenação Geral de Vigilância de Arboviroses - CGARB). Para análise da série histórica, são utilizadas as bases de dados congeladas em data específica para cada ano analisado. Para dados recentes (2024), a atualização é feita diariamente no painel Saúde de A a Z. \r\n\r\nPara cálculo da incidência recomenda-se utilizar locais de residência.\r\n\r\nA partir de 2020 o estado do Espírito Santo passou a utilizar o sistema e-SUS Vigilância em Saúde. Portanto, para os casos de Arboviroses urbanas do Espírito Santo foram considerados apenas os dados disponibilizados pelo Sinan online (dengue e chikungunya) e Sinan Net (zika). \r\n\r\nOutras informações podem ser consultadas nos sítios eletrônicos: \r\n\r\nPortal Sinan: \r\nhttps://portalsinan.saude.gov.br \r\nFebre de Chikungunya: https://portalsinan.saude.gov.br/chikungunya \r\n\r\nSaúde de A a Z\r\nChikungunya: https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/c/chikungunya","formats":["PDF","API","CSV","JSON","XML"],"groups":[{"display_name":"Arboviroses","name":"arboviroses"}],"tags":[]},{"name":"arboviroses-zika-virus","title":"Sinan/Vírus Zika","notes":"O Sistema de Informação de Agravos de Notificação (Sinan) tem como objetivo coletar, transmitir e disseminar dados gerados rotineiramente pela vigilância epidemiológica das três esferas de governo, por meio de uma rede informatizada, para apoiar o processo de investigação e dar subsídios à análise das informações das doenças e dos agravos de notificação compulsória. Atualmente, o sistema possui duas versões vigentes, Sinan Online e Sinan Net. \r\n \r\nO Sinan Online visa à inserção e disseminação dos dados de notificação e investigação de dengue e de febre de chikungunya, enquanto que o Sinan Net é alimentado pela notificação e investigação da grande maioria dos agravos e doenças, que constam na Lista Nacional de Notificação Compulsória de Doenças, Agravos e Eventos de Saúde Pública, do Anexo 1 do Anexo V da Portaria de Consolidação nº 4, de 28 de setembro de 2017, que consolida as normas sobre os sistemas e os subsistemas do Sistema Único de Saúde, mas é facultado a estados e municípios incluir outros problemas de saúde importantes para o seu contexto local. Destaca-se que Zika é doença de notificação compulsória, ou seja, todo caso suspeito e/ou confirmado deve ser obrigatoriamente notificado ao Serviço de Vigilância Epidemiológica da Secretaria Municipal de Saúde (SMS), no Sinan Net. \r\n\r\nZika - Notas: \r\n\r\nTodos os casos suspeitos de Zika devem ser investigados e encerrados no Sinan Net em até 60 dias da data de notificação. Para os casos prováveis de Zika são considerados todos os casos suspeitos notificados, com exceção dos casos que foram descartados.\r\nAs bases de dados de Zika aqui disponíveis podem apresentar pequenas divergências com os dados disponibilizados pela CGARB (Coordenação Geral de Vigilância de Arboviroses - CGARB). Para análise da série histórica, são utilizadas as bases de dados congeladas em data específica para cada ano analisado. Para dados recentes (2024), a atualização é feita diariamente no painel Saúde de A a Z. \r\n\r\nPara cálculo da incidência recomenda-se utilizar locais de residência. \r\n\r\nA partir de 2020 o estado do Espírito Santo passou a utilizar o sistema e-SUS Vigilância em Saúde. Portanto, para os casos de Arboviroses urbanas do Espírito Santo foram considerados apenas os dados disponibilizados pelo Sinan online (dengue e chikungunya) e Sinan Net (zika). \r\n\r\nOutras informações podem ser consultadas nos sítios eletrônicos: \r\n\r\nPortal Sinan \r\nhttps://portalsinan.saude.gov.br \r\n\r\nZika: https://portalsinan.saude.gov.br/zika \r\n\r\nSaúde de A a Z\r\nZika: https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/z/zika-virus\r\n","formats":["PDF","API","CSV","JSON","XML"],"groups":[{"display_name":"Arboviroses","name":"arboviroses"}],"tags":[]},{"name":"sisagua-cadastro-carro-pipa-procedencia","title":"SISAGUA - Cadastro Carro Pipa Procedência","notes":"SISAGUA - Cadastro Carro Pipa Procedência","formats":["ODT","API","CSV","JSON","XML"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[]},{"name":"sisagua-cadastro-carro-pipa-populacao","title":"SISAGUA - Cadastro Carro Pipa População","notes":"Dados cadastrais sobre Carros-Pipa utilizados para abastecimento de água para consumo humano, com informações sobre a população atendida por cada um deles.","formats":["ODT","API","CSV","JSON","XML"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[]},{"name":"sisagua-pontos-de-captacao","title":"SISAGUA - Pontos de captação","notes":"Dados sobre os pontos de captação de água para consumo humano registrados nos sistemas e soluções alternativas de abastecimento de água cadastrados no Sisagua.","formats":["ODT","API","CSV","JSON","XML"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[{"display_name":"SISAGUA","name":"SISAGUA"}]},{"name":"sisagua-tratamento-de-agua","title":"SISAGUA - Tratamento de água","notes":"Dados sobre o tratamento de água empregado nos sistemas e soluções alternativas de abastecimento de água para consumo humano, informados pelo prestador de serviço em frequência anual.","formats":["ODT","API","CSV","JSON","XML"],"groups":[{"display_name":"Vigilância e 
Meio Ambiente","name":"vigilancia-e-meio-ambiente"}],"tags":[]}],"page":"1","rows":"20"},"__N_SSP":true} diff --git a/pysus/tests/api/saude/fixtures/dataset_arboviroses-dengue.json b/pysus/tests/api/saude/fixtures/dataset_arboviroses-dengue.json new file mode 100644 index 00000000..4fbbc512 --- /dev/null +++ b/pysus/tests/api/saude/fixtures/dataset_arboviroses-dengue.json @@ -0,0 +1 @@ +{"pageProps":{"author":"Coordenação-Geral de Vigilância de Arboviroses – CGARB/DEDT/SVSA/MS ; Coordenação-Geral de Informações e Análises Epidemiológicas – CGIAE/DAENT/SVSA/MS","author_email":"","creator_user_id":"8a38bdca-8601-4d5e-a3e4-f463c6f8ff09","id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","isopen":true,"license_id":"cc-by","license_title":"Creative Commons Atribuição","license_url":"http://www.opendefinition.org/licenses/cc-by","maintainer":"","maintainer_email":"","metadata_created":"2024-02-22T19:23:06.206939","metadata_modified":"2026-08-16T03:59:48.701915","name":"arboviroses-dengue","notes":"O Sistema de Informação de Agravos de Notificação (Sinan) tem como objetivo coletar, transmitir e disseminar dados gerados rotineiramente pela vigilância epidemiológica das três esferas de governo, por meio de uma rede informatizada, para apoiar o processo de investigação e dar subsídios à análise das informações das doenças e dos agravos de notificação compulsória. Atualmente, o sistema possui duas versões vigentes, Sinan Online e Sinan Net. \r\n \r\nO Sinan Online visa à inserção e disseminação dos dados de notificação e investigação de dengue e de febre de chikungunya, enquanto que o Sinan Net é alimentado pela notificação e investigação da grande maioria dos agravos e doenças, que constam na Lista Nacional de Notificação Compulsória de Doenças, Agravos e Eventos de Saúde Pública, do Anexo 1 do Anexo V da Portaria de Consolidação nº 4, de 28 de setembro de 2017, que consolida as normas sobre os sistemas e os subsistemas do Sistema Único de Saúde, mas é facultado a estados e municípios incluir outros problemas de saúde importantes para o seu contexto local. \r\n \r\nDestaca-se que a dengue é doença de notificação compulsória, ou seja, todo caso suspeito e/ou confirmado deve ser obrigatoriamente notificado ao Serviço de Vigilância Epidemiológica da Secretaria Municipal de Saúde (SMS). As notificações de casos suspeitos de dengue devem ser registradas na Ficha de Notificação/Investigação da dengue e chikungunya e inseridas no Sistema de Informação de Agravos de Notificação – Sinan Online. Os óbitos suspeitos pela infecção do vírus dengue (DENV) são de notificação compulsória imediata para todas as esferas de gestão do Sistema Único de Saúde (SUS), a ser realizada em até 24 horas a partir do seu conhecimento, pelo meio de comunicação mais rápido disponível. Posteriormente, os dados devem ser inseridos no Sistema de Informação de Agravos de Notificação (Sinan Online). \r\n\r\n__Dengue - Notas__: \r\n\r\nTodos os casos suspeitos de dengue devem ser investigados e encerrados no Sinan Online em até 60 dias da data de notificação. Para os casos prováveis de dengue são considerados todos os casos suspeitos notificados, com exceção dos casos que foram descartados. \r\n \r\nAs bases de dados de dengue aqui disponíveis podem apresentar pequenas divergências com os dados disponibilizados pela CGARB (Coordenação Geral de Vigilância de Arboviroses - CGARB). Para análise da série histórica, são utilizadas as bases de dados congeladas em data específica para cada ano analisado. Para dados recentes (2024), a atualização é feita diariamente no painel Saúde de A a Z. Dengue: https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/d/dengue/dengue. \r\n \r\nPara extração de dados sobre casos graves e óbitos de dengue (é necessário considerar as definições de classificação de casos conforme o Guia de Vigilância em Saúde, 2022. Para casos graves, deve-se aplicar filtros na classificação final, correspondentes à “dengue com complicações”, “febre hemorrágica da dengue”, “síndrome do choque da dengue”, “dengue com sinais de alarme” e “dengue grave”) e filtros no critério de confirmação, correspondentes à “laboratorial” e “clínico-epidemiológico”). Para óbitos, adicionalmente, deve-se aplicar um filtro na evolução, correspondente à “Óbito pelo agravo”.\r\n \r\nPara cálculo da incidência recomenda-se utilizar locais de residência. \r\n \r\n**A partir de 2020 o estado do Espírito Santo passou a utilizar o sistema e-SUS Vigilância em Saúde. Portanto, para os casos de Arboviroses urbanas do Espírito Santo foram considerados apenas os dados disponibilizados pelo Sinan online (dengue e chikungunya) e Sinan Net (zika)**. \r\n\r\nOutras informações podem ser consultadas nos sítios eletrônicos:\r\n###Portal Sinan \r\nhttps://portalsinan.saude.gov.br/ \r\nDengue - http://portalsinan.saude.gov.br/dengue\r\n\r\n###Saúde de A a Z \r\nDengue: https://www.gov.br/saude/pt-br/assuntos/saude-de-a-a-z/d/dengue/dengue ","num_resources":83,"num_tags":0,"organization":{"id":"47e37a4a-4c0c-4d98-8a76-351826101359","name":"ministerio-da-saude","title":"Ministério da Saúde","type":"organization","description":"","image_url":"2024-02-08-183755.7246712022-01-11-181650.855423logoms.png","created":"2022-01-11T18:11:22.588067","is_organization":true,"approval_status":"approved","state":"active"},"owner_org":"47e37a4a-4c0c-4d98-8a76-351826101359","private":false,"state":"active","title":"Sinan/Dengue","type":"dataset","url":"Sistema de Informação de Agravos de Notificação (SINAN) ","version":"","extras":[{"key":"Contato","value":"arboviroses@saude.gov.br"},{"key":"Frequência de atualização","value":"Semanal"}],"groups":[{"description":"","display_name":"Arboviroses","id":"64ed843c-b3fb-44a5-be2d-15127ad00e7d","image_display_url":"https://ckan-dadosabertos.saude.gov.br/uploads/group/2024-11-22-221435.516701arboviroses-dengue-chikungunya-zika.png","name":"arboviroses","title":"Arboviroses"}],"resources":[{"cache_last_updated":null,"cache_url":null,"created":"2024-02-22T19:24:55.932294","datastore_active":false,"description":"","format":"PDF","hash":"","id":"ccc60f03-2834-49a3-98f7-7ee76f50a316","last_modified":"2024-03-22T18:45:26.828489","metadata_modified":"2025-07-14T19:07:20.057593","mimetype":"application/pdf","mimetype_inner":null,"name":"Dicionário de dados - Dengue","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":0,"resource_type":null,"size":387274,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/dic_dados_dengue.pdf","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-22T20:30:03.390689","datastore_active":false,"description":null,"format":"API","hash":"","id":"a9b73910-f233-417b-85c9-95230c269e1c","last_modified":"2026-08-16T00:59:26.767367","metadata_modified":"2026-08-16T03:59:48.720017","mimetype":null,"mimetype_inner":null,"name":"Dengue 2000-2026","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":1,"resource_type":null,"size":null,"state":"active","url":"https://apidadosabertos.saude.gov.br/v1/#/Agravo%20Arboviroses/get_arboviroses_dengue","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2026-01-24T09:09:28.374037","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"f71ec0f9-82eb-4177-bd6a-ff76f2dfa84d","last_modified":"2026-08-15T22:56:17.097835","metadata_modified":"2026-08-16T01:56:38.911571","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2026","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":2,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR26.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2025-05-08T14:30:56.464889","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"5c9132a9-77c2-4b15-8afc-a43c58fc9ec0","last_modified":"2026-08-15T22:56:16.181592","metadata_modified":"2026-08-16T01:56:38.019364","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2025","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":3,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR25.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-22T20:04:00.630495","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"4cf2f7b8-9920-41b5-817a-1ffc13e0690c","last_modified":"2026-08-15T22:56:15.151931","metadata_modified":"2026-08-16T01:56:37.055171","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2024","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":4,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR24.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-22T20:03:35.629401","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"748149fb-d056-4473-ab08-7cdcf68c029c","last_modified":"2026-08-15T22:56:14.259990","metadata_modified":"2026-08-16T01:56:36.040480","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2023","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":5,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR23.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:30:08.939954","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"215e9595-2ea9-4f44-9bcb-f11ac4a68ad6","last_modified":"2026-08-15T22:56:13.257618","metadata_modified":"2026-08-16T01:56:35.054400","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2022","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":6,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR22.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:32:19.748550","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"8e80832d-f49f-4e55-aecf-b6a8c5c1adc5","last_modified":"2026-08-15T22:56:12.380312","metadata_modified":"2026-08-16T01:56:34.181704","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2021","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":7,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR21.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:34:57.118231","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"9d7ff036-0e01-4e4e-8a5b-dda94c88c021","last_modified":"2026-08-15T22:56:11.422219","metadata_modified":"2026-08-16T01:56:33.247624","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2020","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":8,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR20.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:38:20.821487","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"8aa088b5-dd94-47d2-9ab5-7c158ce16fc9","last_modified":"2026-08-15T22:56:10.490940","metadata_modified":"2026-08-16T01:56:32.287000","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2019","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":9,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR19.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:40:35.368184","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"c34e7b1b-bada-477c-b38f-345c9b417742","last_modified":"2026-08-15T22:56:09.560620","metadata_modified":"2026-08-16T01:56:31.360061","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2018","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":10,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR18.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:03:26.228233","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"184047c0-7af8-4550-b4a4-c1d0498adbbc","last_modified":"2026-08-15T22:56:08.666807","metadata_modified":"2026-08-16T01:56:30.460349","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2017","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":11,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR17.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:05:19.431240","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"547f3c06-07fe-4358-aa45-2faf89af9cf5","last_modified":"2026-01-09T19:03:59.958623","metadata_modified":"2026-01-09T22:01:26.240033","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2016","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":12,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR16.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:13:28.628927","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"52cd771d-1149-4458-a15e-372c13159ff8","last_modified":"2026-01-09T19:03:58.994618","metadata_modified":"2026-01-09T22:01:25.279453","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2015","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":13,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR15.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:14:40.208225","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"77c2d7f1-4732-4461-9f0e-6b4c6f897099","last_modified":"2026-01-09T19:03:58.043415","metadata_modified":"2026-01-09T22:01:24.316887","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2014","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":14,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR14.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:16:53.582283","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"3cd1e8a1-9c10-4e96-9829-e2152e8c5085","last_modified":"2026-01-09T19:03:57.049150","metadata_modified":"2026-01-09T22:01:23.327877","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2013","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":15,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR13.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:19:01.595004","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"03ce7ca0-ccb9-4107-adf1-211f12b46976","last_modified":"2026-01-09T19:03:56.063860","metadata_modified":"2026-01-09T22:01:22.347364","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2012","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":16,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR12.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:21:06.069632","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"9458088f-91ec-4561-bc07-7fe88e70d1d7","last_modified":"2026-01-09T19:03:55.119155","metadata_modified":"2026-01-09T22:01:21.387585","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2011","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":17,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR11.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:23:21.226490","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"d57d1c32-27ed-45dd-bdc6-749b1d289782","last_modified":"2026-01-09T19:03:53.913103","metadata_modified":"2026-01-09T22:01:20.267792","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2010","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":18,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR10.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:46:15.224663","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"19245ce6-215c-4014-9d13-f3df6e2546d3","last_modified":"2026-01-09T19:03:52.611819","metadata_modified":"2026-01-09T22:01:19.075338","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2009","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":19,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR09.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:51:16.073708","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"ba1145ae-f990-44cb-b00e-22b7ccfc95ab","last_modified":"2026-01-09T19:03:51.331094","metadata_modified":"2026-01-09T22:01:17.683103","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2008","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":20,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR08.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:53:47.220201","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"b9327802-f450-4809-a769-12687067ae31","last_modified":"2026-01-09T19:03:49.964100","metadata_modified":"2026-01-09T22:01:16.437951","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2007","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":21,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR07.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:57:31.465768","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"adf3ae63-8e6c-4168-a397-1f098b6b2f83","last_modified":"2026-01-09T19:03:48.575593","metadata_modified":"2026-01-09T22:01:14.976430","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2006","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":22,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR06.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:03:47.135307","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"4361bece-ad95-4993-8d00-3b6524c1064a","last_modified":"2026-01-09T19:03:47.603349","metadata_modified":"2026-01-09T22:01:13.878183","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2005","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":23,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR05.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:21:11.759307","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"1906faea-1ea3-4719-81dc-b61c3258fee7","last_modified":"2026-01-09T19:03:46.402295","metadata_modified":"2026-01-09T22:01:12.730465","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2004","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":24,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR04.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:23:14.447443","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"e9935b70-f7b9-422a-bfc1-01e64f9ad8a6","last_modified":"2026-01-09T19:03:45.188728","metadata_modified":"2026-01-09T22:01:11.554856","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2003","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":25,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR03.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:24:48.774944","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"535e402f-8714-4a0f-ac6e-4eef47c2182b","last_modified":"2026-01-09T19:03:43.954155","metadata_modified":"2026-01-09T22:01:10.299300","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2002","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":26,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR02.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:26:02.127831","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"179f3dfc-b034-4b94-813b-f2afc4b60cbe","last_modified":"2026-01-09T19:03:42.602631","metadata_modified":"2026-01-09T22:01:09.052649","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2001","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":27,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR01.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:30:23.177989","datastore_active":false,"description":null,"format":"CSV","hash":"","id":"65ed7cd0-6de6-43b6-a292-31047428e847","last_modified":"2026-01-09T19:03:40.964112","metadata_modified":"2026-01-09T22:01:07.498730","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2000","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":28,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/csv/DENGBR00.csv.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2026-01-24T09:26:36.793242","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"9175999a-1a94-4d35-9e46-2c0a14e6b7a2","last_modified":"2026-08-15T22:56:26.950800","metadata_modified":"2026-08-16T01:56:48.772107","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2026","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":29,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR26.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2025-05-08T14:32:36.502106","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"2bc51776-8698-4154-9698-11d2b36370a1","last_modified":"2026-08-15T22:56:26.074590","metadata_modified":"2026-08-16T01:56:47.857135","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2025","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":30,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR25.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-23T18:34:15.109420","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"28975ef7-da35-498e-b179-cfbde2150f90","last_modified":"2026-08-15T22:56:24.915953","metadata_modified":"2026-08-16T01:56:46.831786","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2024","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":31,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR24.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-23T18:34:53.756299","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"cc265aba-0c03-4e3c-80e8-95f7a07d610e","last_modified":"2026-08-15T22:56:23.652157","metadata_modified":"2026-08-16T01:56:45.458148","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2023","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":32,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR23.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:31:39.144057","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"0362e389-1ddd-4657-b869-102da981f980","last_modified":"2026-08-15T22:56:22.652944","metadata_modified":"2026-08-16T01:56:44.507836","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2022","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":33,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR22.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:34:37.860972","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"60cca60b-3032-477f-86ea-078b85403c2a","last_modified":"2026-08-15T22:56:21.711669","metadata_modified":"2026-08-16T01:56:43.537385","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2021","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":34,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR21.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:37:58.948721","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"be628746-4129-4567-a0eb-2f5322db348b","last_modified":"2026-08-15T22:56:20.721245","metadata_modified":"2026-08-16T01:56:42.555276","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2020","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":35,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR20.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:39:33.702406","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"db3e7442-f191-46f6-9f1e-7473f2aa56e0","last_modified":"2026-08-15T22:56:19.817746","metadata_modified":"2026-08-16T01:56:41.629691","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2019","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":36,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR19.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:41:57.056610","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"3867dc8a-b8ed-460b-b6f2-82a89124275f","last_modified":"2026-08-15T22:56:18.895445","metadata_modified":"2026-08-16T01:56:40.676119","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2018","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":37,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR18.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:04:23.896631","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"52e13f8e-afce-4eb6-8f9a-cdba2a92f7f6","last_modified":"2026-08-15T22:56:18.010600","metadata_modified":"2026-08-16T01:56:39.798388","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2017","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":38,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR17.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:06:24.851322","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"470c6641-080b-4058-bcb6-3f330aeb59ba","last_modified":"2026-01-09T19:04:27.863152","metadata_modified":"2026-01-09T22:01:54.223985","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2016","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":39,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR16.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:14:28.031062","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"fc4bf3aa-2e7e-4e1c-ac03-ea220af44212","last_modified":"2026-01-09T19:04:26.558085","metadata_modified":"2026-01-09T22:01:52.841227","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2015","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":40,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR15.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:15:35.282819","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"e26cb089-198e-4492-be53-8ca8847f8648","last_modified":"2026-01-09T19:04:25.518844","metadata_modified":"2026-01-09T22:01:51.796479","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2014","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":41,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR14.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:17:53.930764","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"bcf1cb01-0fc5-4bc1-bf5a-ae46c24b0abc","last_modified":"2026-01-09T19:04:24.612645","metadata_modified":"2026-01-09T22:01:50.884797","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2013","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":42,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR13.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:20:02.528780","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"724f663b-8223-4dd5-b67b-4a7e57266d12","last_modified":"2026-01-09T19:04:23.683615","metadata_modified":"2026-01-09T22:01:49.959963","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2012","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":43,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR12.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:22:11.346771","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"a021f78e-74c9-423c-9599-00af837ba47d","last_modified":"2026-01-09T19:04:22.781960","metadata_modified":"2026-01-09T22:01:49.043842","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2011","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":44,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR11.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:24:31.798065","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"98ae1f91-d7e8-4f1d-b53b-0d921d43db80","last_modified":"2026-01-09T19:04:21.825901","metadata_modified":"2026-01-09T22:01:48.089327","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2010","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":45,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR10.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:50:11.889587","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"dfde26ca-67c6-4cb8-9492-02ac420da91d","last_modified":"2026-01-09T19:04:20.861118","metadata_modified":"2026-01-09T22:01:47.138551","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2009","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":46,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR09.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:53:23.553966","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"33d6bc4b-f66f-400e-b955-86ad5c4727de","last_modified":"2026-01-09T19:04:19.888181","metadata_modified":"2026-01-09T22:01:46.175835","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2008","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":47,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR08.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:56:18.460636","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"07b83187-5603-4a9b-a325-823de3fb7c41","last_modified":"2026-01-09T19:04:18.880049","metadata_modified":"2026-01-09T22:01:45.159880","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2007","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":48,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR07.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:00:34.129216","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"cc13b6fa-746a-4903-8599-3abe4d13eba7","last_modified":"2026-01-09T19:04:17.637386","metadata_modified":"2026-01-09T22:01:44.044074","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2006","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":49,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR06.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:04:55.319070","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"0887e508-7b83-4de5-92fc-e760cadba764","last_modified":"2026-01-09T19:04:16.314849","metadata_modified":"2026-01-09T22:01:42.681316","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2005","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":50,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR05.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:21:57.252004","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"725205c8-ea5c-45ad-a9cb-4ccd8c4293e5","last_modified":"2026-01-09T19:04:15.096380","metadata_modified":"2026-01-09T22:01:41.407156","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2004","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":51,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR04.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:24:18.882791","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"1b4012ca-9323-4629-8e4b-59a27067fc6c","last_modified":"2026-01-09T19:04:13.865260","metadata_modified":"2026-01-09T22:01:40.179463","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2003","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":52,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR03.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:25:36.044806","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"93180c69-955a-4820-8d49-27873962681f","last_modified":"2026-01-09T19:04:12.345154","metadata_modified":"2026-01-09T22:01:38.814678","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2002","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":53,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR02.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:29:43.056566","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"5d90d025-645d-44e6-9efe-f38125e3116c","last_modified":"2026-01-09T19:04:11.171245","metadata_modified":"2026-01-09T22:01:37.504653","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2001","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":54,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR01.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:35:16.294821","datastore_active":false,"description":null,"format":"JSON","hash":"","id":"3b0e6203-32b9-49e8-8d85-a06974688622","last_modified":"2026-01-09T19:04:10.077111","metadata_modified":"2026-01-09T22:01:36.344438","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2000","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":55,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/json/DENGBR00.json.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2026-01-24T09:27:24.615354","datastore_active":false,"description":null,"format":"XML","hash":"","id":"67991b12-40f9-45e4-9fac-359d5ec56196","last_modified":"2026-08-15T22:56:37.599144","metadata_modified":"2026-08-16T01:56:59.398249","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2026","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":56,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR26.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2025-05-08T14:34:03.899374","datastore_active":false,"description":null,"format":"XML","hash":"","id":"b42d63c3-9ed2-49fd-b30c-892f88caa223","last_modified":"2026-08-15T22:56:36.646866","metadata_modified":"2026-08-16T01:56:58.454904","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2025","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":57,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR25.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-22T20:18:54.602249","datastore_active":false,"description":null,"format":"XML","hash":"","id":"7e1df5f0-cdf2-455b-ba8b-a179593b07de","last_modified":"2026-08-15T22:56:35.229821","metadata_modified":"2026-08-16T01:56:57.305214","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2024","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":58,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR24.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-22T20:18:39.726732","datastore_active":false,"description":null,"format":"XML","hash":"","id":"46c1c6a8-570a-4f93-a0d1-5fdba7a6b184","last_modified":"2026-08-15T22:56:34.071884","metadata_modified":"2026-08-16T01:56:55.897189","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2023","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":59,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR23.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:30:42.598178","datastore_active":false,"description":null,"format":"XML","hash":"","id":"94588687-8a90-47b5-94ac-5dd43f7bf8c9","last_modified":"2026-08-15T22:56:33.089229","metadata_modified":"2026-08-16T01:56:54.880808","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2022","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":60,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR22.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:33:35.808901","datastore_active":false,"description":null,"format":"XML","hash":"","id":"1abfd667-420e-4fb0-8498-937c76f3f8e1","last_modified":"2026-08-15T22:56:32.115781","metadata_modified":"2026-08-16T01:56:53.952146","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2021","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":61,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR21.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:35:25.086484","datastore_active":false,"description":null,"format":"XML","hash":"","id":"286b5544-60b2-4a0b-9333-523a724878af","last_modified":"2026-08-15T22:56:31.137221","metadata_modified":"2026-08-16T01:56:52.945449","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2020","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":62,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR20.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:39:03.362654","datastore_active":false,"description":null,"format":"XML","hash":"","id":"69c66914-ae70-4019-b9d7-a8527751524e","last_modified":"2026-08-15T22:56:29.773338","metadata_modified":"2026-08-16T01:56:51.575178","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2019","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":63,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR19.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T18:41:08.681064","datastore_active":false,"description":null,"format":"XML","hash":"","id":"7ef44d73-61d1-4f64-a94f-f4330fd9bd15","last_modified":"2026-08-15T22:56:28.840440","metadata_modified":"2026-08-16T01:56:50.663109","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2018","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":64,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR18.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:03:52.899676","datastore_active":false,"description":null,"format":"XML","hash":"","id":"5f2d2ddd-3160-4c0f-a0a5-80fd98729191","last_modified":"2026-08-15T22:56:27.910115","metadata_modified":"2026-08-16T01:56:49.719161","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2017","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":65,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR17.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:05:54.129639","datastore_active":false,"description":null,"format":"XML","hash":"","id":"21806850-bb5a-48f4-b7ed-75780faa0fc8","last_modified":"2026-01-09T19:04:57.193990","metadata_modified":"2026-01-09T22:02:23.467730","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2016","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":66,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR16.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:14:03.845072","datastore_active":false,"description":null,"format":"XML","hash":"","id":"91347312-fea5-4ecb-a535-224beef1890c","last_modified":"2026-01-09T19:04:56.231674","metadata_modified":"2026-01-09T22:02:22.520392","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2015","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":67,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR15.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:15:06.844995","datastore_active":false,"description":null,"format":"XML","hash":"","id":"f6527fe8-e1d1-4adb-b984-7a1576e57b5c","last_modified":"2026-01-09T19:04:55.329127","metadata_modified":"2026-01-09T22:02:21.596296","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2014","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":68,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR14.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:17:19.773560","datastore_active":false,"description":null,"format":"XML","hash":"","id":"db274aaf-6086-4212-80eb-4f3f79aebb18","last_modified":"2026-01-09T19:04:54.364455","metadata_modified":"2026-01-09T22:02:20.641038","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2013","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":69,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR13.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:19:34.854371","datastore_active":false,"description":null,"format":"XML","hash":"","id":"0f954385-ba61-4d27-b3df-5f3159626293","last_modified":"2026-01-09T19:04:53.311037","metadata_modified":"2026-01-09T22:02:19.607511","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2012","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":70,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR12.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:21:40.842890","datastore_active":false,"description":null,"format":"XML","hash":"","id":"4c0e90f5-8554-48bf-98d5-2f912e873a17","last_modified":"2026-01-09T19:04:52.373633","metadata_modified":"2026-01-09T22:02:18.640883","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2011","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":71,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR11.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T19:24:00.096177","datastore_active":false,"description":null,"format":"XML","hash":"","id":"1703d61a-e849-447f-a642-217995230916","last_modified":"2026-01-09T19:04:51.407339","metadata_modified":"2026-01-09T22:02:17.687313","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2010","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":72,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR10.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:46:43.228985","datastore_active":false,"description":null,"format":"XML","hash":"","id":"c42ecdcc-68ea-43f1-a197-b2f591fe34e8","last_modified":"2026-01-09T19:04:50.422743","metadata_modified":"2026-01-09T22:02:16.688833","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2009","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":73,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR09.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:51:47.477534","datastore_active":false,"description":null,"format":"XML","hash":"","id":"bfd954f9-fa8f-41e7-98e8-9ac1848b7db6","last_modified":"2026-01-09T19:04:49.520545","metadata_modified":"2026-01-09T22:02:15.777835","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2008","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":74,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR08.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:55:29.744074","datastore_active":false,"description":null,"format":"XML","hash":"","id":"52b5c4e9-1122-4734-9c2a-fa0ca2e77d53","last_modified":"2026-01-09T19:04:48.470673","metadata_modified":"2026-01-09T22:02:14.744740","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2007","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":75,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR07.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T22:57:58.533233","datastore_active":false,"description":null,"format":"XML","hash":"","id":"3fdc4019-34b6-4406-85a4-d41e5fcee9dd","last_modified":"2026-01-09T19:04:47.533168","metadata_modified":"2026-01-09T22:02:13.795943","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2006","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":76,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR06.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:04:20.577723","datastore_active":false,"description":null,"format":"XML","hash":"","id":"0159b4d5-f14a-4874-bc16-55e475530438","last_modified":"2026-01-09T19:04:46.609119","metadata_modified":"2026-01-09T22:02:12.869597","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2005","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":77,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR05.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:21:32.532649","datastore_active":false,"description":null,"format":"XML","hash":"","id":"c3a34842-5241-4460-9eaf-4d4c5b50506f","last_modified":"2026-01-09T19:04:45.691802","metadata_modified":"2026-01-09T22:02:11.972446","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2004","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":78,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR04.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:23:46.529440","datastore_active":false,"description":null,"format":"XML","hash":"","id":"f9fa2912-79af-45ad-ab96-8d3ab32e565f","last_modified":"2026-01-09T19:04:44.756485","metadata_modified":"2026-01-09T22:02:11.020914","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2003","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":79,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR03.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:25:18.033341","datastore_active":false,"description":null,"format":"XML","hash":"","id":"f928c3d1-8e4f-45c2-a14d-3d3b226a295c","last_modified":"2026-01-09T19:04:43.823464","metadata_modified":"2026-01-09T22:02:10.084338","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2002","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":80,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR02.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:26:23.054893","datastore_active":false,"description":null,"format":"XML","hash":"","id":"9f71dd7f-ab42-4b7e-b79b-ab4d1c556051","last_modified":"2026-01-09T19:04:41.977182","metadata_modified":"2026-01-09T22:02:08.857529","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2001","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":81,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR01.xml.zip","url_type":""},{"cache_last_updated":null,"cache_url":null,"created":"2024-02-28T23:33:35.276776","datastore_active":false,"description":null,"format":"XML","hash":"","id":"73d35b0f-1d9e-428b-87e6-7f393ec99a1f","last_modified":"2026-01-09T19:04:40.663461","metadata_modified":"2026-01-09T22:02:07.020628","mimetype":"application/zip","mimetype_inner":null,"name":"Dengue - 2000","package_id":"4d5e5d44-58a8-4d67-b8aa-4ef1e4b00a1c","position":82,"resource_type":null,"size":null,"state":"active","url":"https://s3.sa-east-1.amazonaws.com/ckan.saude.gov.br/SINAN/Dengue/xml/DENGBR00.xml.zip","url_type":""}],"tags":[],"relationships_as_subject":[],"relationships_as_object":[]},"__N_SSP":true} diff --git a/pysus/tests/api/saude/fixtures/dengue_2024.csv.zip b/pysus/tests/api/saude/fixtures/dengue_2024.csv.zip new file mode 100644 index 0000000000000000000000000000000000000000..815ade13cc185722af999b6751e7772cc2cdefcf GIT binary patch literal 232 zcmWIWW@Zs#U|`^2xZEum%Ta&zd>)Xu0El^j*u~Y)-6_b(L@&9xEcBEk*C7J|w}*Ql z1ae*WQ987r%Tt-TENTwRowI-Q6-yHfe^$i5T^BuFjrH4yi+xQY@$Fs{Wvybe)^%%l z?bLnGyLETxuYW>Q(#?eP7hU&WAfWyz(=UO|_~?`got(V;-DkX9qWI@x#o?+;zo*wR m1b8zti7?}K8PIWH(7*_yklh*J&B_K6V+2AgAUy-bVE_P6qe+ke literal 0 HcmV?d00001 diff --git a/pysus/tests/api/saude/fixtures/homepage.html b/pysus/tests/api/saude/fixtures/homepage.html new file mode 100644 index 00000000..e5d060be --- /dev/null +++ b/pysus/tests/api/saude/fixtures/homepage.html @@ -0,0 +1 @@ +Portal de Dados Abertos do SUS
Ilustração esquerdaIlustração esquerda

Saúde para todos,
Dados para todos

O que você deseja buscar?

Sugestões:
covid-19DPNICorona
Ilustração direita

Temas

https://ckan-dadosabertos.saude.gov.br/uploads/group/2024-11-22-221435.516701arboviroses-dengue-chikungunya-zika.png
Arboviroses
4 Conjuntos de dados
Acessar
https://ckan-dadosabertos.saude.gov.br/uploads/group/2024-11-22-222025.344474assistencia-a-saude.png
Assistência à saúde
4 Conjuntos de dados
Acessar
https://ckan-dadosabertos.saude.gov.br/uploads/group/2024-11-22-222238.458471assistencia-farmaceutica.png
Assistência Farmacêutica
1 Conjuntos de dados
Acessar
https://ckan-dadosabertos.saude.gov.br/uploads/group/2024-11-22-222144.588090atencao-primaria.png
Atenção Primária
3 Conjuntos de dados
Acessar
https://ckan-dadosabertos.saude.gov.br/uploads/group/2024-11-22-220946.224463ciencia-tecnologia.png
Ciência & Tecnologia
4 Conjuntos de dados
Acessar
https://ckan-dadosabertos.saude.gov.br/uploads/group/2024-11-22-222757.142927diagnosticos-e-tratamentos.png
Diagnósticos e Tratamentos
2 Conjuntos de dados
Acessar

Atualizações

Saiba mais

PDA 2024-2026
PDA 2024-2026
O Plano de Dados Abertos (PDA) é o instrumento norteador das ações de implementação e promoção de abertura de dados, inclusive os geoespacializados, que deverão obedecer aos padrões mínimos de qualidade, de forma a facilitar o entendimento e a reutilização das informações. É este documento que organiza o planejamento referente à implantação e racionalização dos processos de publicação de dados abertos nas organizações públicas.
Cartilha de Dados Abertos
Cartilha de Dados Abertos
A Cartilha de Dados Abertos do SUS foi elaborada para orientar as áreas técnicas do Ministério da Saúde (MS) sobre o processo de disseminação de dados em formato aberto e a sua importância para a ampliação da transparência pública.
Logotipo Ouvidoria Geral do SUSLogotipo Ouvidoria Geral do SUSLogotipo Ouvidoria Geral do SUSLogotipo Ouvidoria Geral do SUSLogotipo Ouvidoria Geral do SUS
Ouvidoria Geral do SUS
Teleatendimento: de segunda-feira a
sexta-feira, das 8h às 20h
e aos sábados, das 8h às 18h
Logotipo SUS - Ministério da SaúdeLogotipo SUS - Ministério da SaúdeLogotipo SUS - Ministério da SaúdeLogotipo SUS - Ministério da SaúdeLogotipo SUS - Ministério da Saúde
Texto destinado a exibição de informações relacionadas à licença de uso.
diff --git a/pysus/tests/api/saude/test_catalog.py b/pysus/tests/api/saude/test_catalog.py new file mode 100644 index 00000000..23695f98 --- /dev/null +++ b/pysus/tests/api/saude/test_catalog.py @@ -0,0 +1,132 @@ +"""Tests for the catalog fetcher (Next.js data layer).""" + +from __future__ import annotations + +import httpx +import pytest +from pysus.api.saude.catalog import ( + fetch_catalog_page, + fetch_dataset, + list_groups, + list_tags, +) +from pysus.api.saude.errors import DatasetNotFound, PortalChanged + + +class TestFetchCatalogPage: + @pytest.mark.asyncio + async def test_returns_catalog_page( + self, tmp_path, mocked_saude, saude_homepage_html + ): + async with httpx.AsyncClient(transport=mocked_saude) as client: + page = await fetch_catalog_page( + client, + build_id="test-build-id", + page=1, + cache_root=tmp_path, + ttl=__import__("datetime").timedelta(hours=24), + ) + assert page.number_of_packages == 138 + assert page.page == 1 + assert len(page.packages) == 20 + names = [p.name for p in page.packages] + assert "arboviroses-dengue" in names + + @pytest.mark.asyncio + async def test_raises_portal_changed_on_bad_shape(self, tmp_path): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b'{"pageProps": {}}') + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + with pytest.raises(PortalChanged): + await fetch_catalog_page( + client, + build_id="x", + page=1, + cache_root=tmp_path, + ttl=__import__("datetime").timedelta(hours=24), + ) + + +class TestFetchDataset: + @pytest.mark.asyncio + async def test_returns_package(self, tmp_path, mocked_saude): + async with httpx.AsyncClient(transport=mocked_saude) as client: + package = await fetch_dataset( + client, + build_id="test-build-id", + slug="arboviroses-dengue", + cache_root=tmp_path, + ttl=__import__("datetime").timedelta(hours=24), + ) + assert package.name == "arboviroses-dengue" + assert len(package.resources) == 83 + assert package.periodicity == "Semanal" + + @pytest.mark.asyncio + async def test_raises_dataset_not_found(self, tmp_path): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, content=b'{"pageProps": {"name": "other"}}' + ) + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + with pytest.raises(DatasetNotFound): + await fetch_dataset( + client, + build_id="x", + slug="missing", + cache_root=tmp_path, + ttl=__import__("datetime").timedelta(hours=24), + ) + + @pytest.mark.asyncio + async def test_raises_portal_changed_on_bad_payload(self, tmp_path): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, content=b'{"pageProps": {"name": "x", "bogus": true}}' + ) + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + with pytest.raises(PortalChanged): + await fetch_dataset( + client, + build_id="x", + slug="x", + cache_root=tmp_path, + ttl=__import__("datetime").timedelta(hours=24), + ) + + +class TestListGroups: + @pytest.mark.asyncio + async def test_returns_14_groups(self, tmp_path, mocked_saude): + async with httpx.AsyncClient(transport=mocked_saude) as client: + groups = await list_groups( + client, + build_id="test-build-id", + cache_root=tmp_path, + ttl=__import__("datetime").timedelta(hours=24), + ) + assert len(groups) == 14 + names = {g.name for g in groups} + assert "arboviroses" in names + assert "saude-indigena" in names + assert "vacinacao" in names + + +class TestListTags: + @pytest.mark.asyncio + async def test_returns_tag_list(self, tmp_path, mocked_saude): + async with httpx.AsyncClient(transport=mocked_saude) as client: + tags = await list_tags( + client, + build_id="test-build-id", + cache_root=tmp_path, + ttl=__import__("datetime").timedelta(hours=24), + ) + assert len(tags) > 0 + assert all(t.name for t in tags) diff --git a/pysus/tests/api/saude/test_client.py b/pysus/tests/api/saude/test_client.py new file mode 100644 index 00000000..0c580f82 --- /dev/null +++ b/pysus/tests/api/saude/test_client.py @@ -0,0 +1,94 @@ +"""Tests for the SaudeClient facade. + +Wires the captured fixtures through ``httpx.MockTransport`` (via the +``saude_client`` fixture) and drives the public API end-to-end. +""" + +from __future__ import annotations + +import pytest +from pysus.api.saude import SaudeClient + + +class TestLifecycle: + @pytest.mark.asyncio + async def test_context_manager_yields_self(self, tmp_path): + async with SaudeClient(cache_dir=tmp_path, timeout=10.0) as c: + assert isinstance(c, SaudeClient) + + +class TestListDatasets: + @pytest.mark.asyncio + async def test_returns_catalog_entries(self, saude_client: SaudeClient): + page = await saude_client.list_datasets(group="arboviroses", page=1) + assert len(page) == 20 + assert any(p.name == "arboviroses-dengue" for p in page) + + +class TestFetchDataset: + @pytest.mark.asyncio + async def test_returns_full_package(self, saude_client: SaudeClient): + package = await saude_client.fetch_dataset("arboviroses-dengue") + assert package.name == "arboviroses-dengue" + assert len(package.resources) == 83 + assert package.periodicity == "Semanal" + + @pytest.mark.asyncio + async def test_resources_are_parsed(self, saude_client: SaudeClient): + resources = await saude_client.fetch_resources("arboviroses-dengue") + formats = {r.format for r in resources} + assert {"CSV", "JSON", "XML", "PDF", "API"} <= formats + + +class TestListGroups: + @pytest.mark.asyncio + async def test_returns_14_groups(self, saude_client: SaudeClient): + groups = await saude_client.list_groups() + assert len(groups) == 14 + names = {g.name for g in groups} + assert "arboviroses" in names + assert "saude-indigena" in names + assert "vacinacao" in names + + +class TestListTags: + @pytest.mark.asyncio + async def test_returns_tags(self, saude_client: SaudeClient): + tags = await saude_client.list_tags() + assert len(tags) > 0 + + +class TestDownloadDataset: + @pytest.mark.asyncio + async def test_writes_csv_files(self, saude_client: SaudeClient, tmp_path): + dest = tmp_path / "downloads" + paths = await saude_client.download_dataset( + "arboviroses-dengue", dest_dir=dest, fmt="CSV" + ) + assert all(p.exists() for p in paths) + assert all(p.stat().st_size > 0 for p in paths) + + +class TestDownloadResource: + @pytest.mark.asyncio + async def test_writes_single_csv(self, saude_client: SaudeClient, tmp_path): + package = await saude_client.fetch_dataset("arboviroses-dengue") + target = next(r for r in package.resources if r.format == "CSV") + path = await saude_client.download_resource( + "arboviroses-dengue", + resource_id=target.id, + dest_dir=tmp_path, + ) + assert path.exists() + assert path.stat().st_size > 0 + + +class TestCaching: + @pytest.mark.asyncio + async def test_cached_build_id_is_reused( + self, saude_client: SaudeClient, tmp_path + ): + await saude_client.list_groups() + # Second call should reuse the cached buildId + await saude_client.list_groups() + assert (tmp_path / "build_id.json").exists() diff --git a/pysus/tests/api/saude/test_download.py b/pysus/tests/api/saude/test_download.py new file mode 100644 index 00000000..496ea6b4 --- /dev/null +++ b/pysus/tests/api/saude/test_download.py @@ -0,0 +1,154 @@ +"""Tests for the resource / dataset download helpers.""" + +from __future__ import annotations + +import httpx +import pytest +from pysus.api.saude.download import ( + download_dataset, + download_resource, + filename_for, +) +from pysus.api.saude.errors import ResourceNotFound +from pysus.api.saude.resources import Resource + + +class TestFilenameFor: + def test_uses_resource_name(self): + resource = Resource.model_validate( + { + "id": "abc", + "name": "Dengue - 2024", + "format": "CSV", + "url": "https://example.com/x.csv", + } + ) + assert ( + filename_for(resource, "arboviroses-dengue", 0) + == "Dengue - 2024.csv" + ) + + def test_appends_format_extension_when_missing(self): + resource = Resource.model_validate( + { + "id": "abc", + "name": "Dengue 2024", + "format": "JSON", + "url": "https://example.com/x", + } + ) + assert ( + filename_for(resource, "arboviroses-dengue", 0) + == "Dengue 2024.json" + ) + + def test_skips_extension_when_format_is_api(self): + resource = Resource.model_validate( + { + "id": "abc", + "name": "API docs", + "format": "API", + "url": "https://example.com/docs", + } + ) + assert filename_for(resource, "x", 0) == "API docs" + + def test_strips_unsafe_characters(self): + resource = Resource.model_validate( + { + "id": "abc", + "name": "weird/name?with&chars", + "format": "CSV", + "url": "https://example.com/x", + } + ) + name = filename_for(resource, "x", 0) + assert "/" not in name + assert "?" not in name + assert "&" not in name + + +class TestDownloadResource: + @pytest.mark.asyncio + async def test_downloads_single_resource( + self, saude_dataset_page_props, tmp_path, mocked_saude + ): + from pysus.api.saude.resources import CKANPackage + + package = CKANPackage.model_validate(saude_dataset_page_props) + target = next(r for r in package.resources if r.format == "CSV") + + async with httpx.AsyncClient(transport=mocked_saude) as client: + path = await download_resource( + client, package, resource_id=target.id, dest_dir=tmp_path + ) + assert path.exists() + assert path.stat().st_size > 0 + + @pytest.mark.asyncio + async def test_skips_api_resources( + self, saude_dataset_page_props, tmp_path, mocked_saude + ): + from pysus.api.saude.resources import CKANPackage + + package = CKANPackage.model_validate(saude_dataset_page_props) + api_resource = next(r for r in package.resources if r.format == "API") + async with httpx.AsyncClient(transport=mocked_saude) as client: + with pytest.raises(ResourceNotFound): + await download_resource( + client, + package, + resource_id=api_resource.id, + dest_dir=tmp_path, + ) + + @pytest.mark.asyncio + async def test_ambiguous_selector_raises( + self, saude_dataset_page_props, tmp_path, mocked_saude + ): + from pysus.api.saude.resources import CKANPackage + + package = CKANPackage.model_validate(saude_dataset_page_props) + async with httpx.AsyncClient(transport=mocked_saude) as client: + with pytest.raises(ValueError, match="exactly one"): + await download_resource(client, package, dest_dir=tmp_path) + + @pytest.mark.asyncio + async def test_no_match_raises_not_found( + self, saude_dataset_page_props, tmp_path, mocked_saude + ): + from pysus.api.saude.resources import CKANPackage + + package = CKANPackage.model_validate(saude_dataset_page_props) + async with httpx.AsyncClient(transport=mocked_saude) as client: + with pytest.raises(ResourceNotFound): + await download_resource( + client, + package, + resource_id="nonexistent-id", + dest_dir=tmp_path, + ) + + +class TestDownloadDataset: + @pytest.mark.asyncio + async def test_downloads_only_csv_resources( + self, saude_dataset_page_props, tmp_path, mocked_saude + ): + from pysus.api.saude.resources import CKANPackage + + package = CKANPackage.model_validate(saude_dataset_page_props) + expected_csvs = sum(1 for r in package.resources if r.format == "CSV") + assert expected_csvs > 0 + + async with httpx.AsyncClient(transport=mocked_saude) as client: + paths = await download_dataset( + client, + package, + dest_dir=tmp_path, + fmt="CSV", + ) + assert len(paths) == expected_csvs + for path in paths: + assert path.exists() + assert path.stat().st_size > 0 diff --git a/pysus/tests/api/saude/test_next_data.py b/pysus/tests/api/saude/test_next_data.py new file mode 100644 index 00000000..9d5435a7 --- /dev/null +++ b/pysus/tests/api/saude/test_next_data.py @@ -0,0 +1,158 @@ +"""Tests for the Next.js buildId extractor.""" + +from __future__ import annotations + +import json +from datetime import datetime, timedelta +from pathlib import Path + +import httpx +import pytest +from pysus.api.saude.errors import BuildIdMissing, NoUsableBuildId +from pysus.api.saude.next_data import ( + _is_fresh, + _parse_build_id, + _read_cache, + _write_cache, + fetch_build_id, +) + + +class TestParseBuildId: + def test_extracts_build_id_from_homepage(self, saude_homepage_html: str): + build_id = _parse_build_id(saude_homepage_html) + assert isinstance(build_id, str) + assert len(build_id) >= 8 + + def test_raises_when_script_tag_missing(self): + html = "no script tag here" + with pytest.raises(BuildIdMissing, match="__NEXT_DATA__"): + _parse_build_id(html) + + def test_raises_when_payload_lacks_build_id(self): + html = ( + '' + ) + with pytest.raises(BuildIdMissing, match="buildId"): + _parse_build_id(html) + + def test_raises_when_payload_is_invalid_json(self): + html = ( + '" + ) + with pytest.raises(BuildIdMissing): + _parse_build_id(html) + + +class TestCache: + def test_roundtrip(self, tmp_path: Path): + path = tmp_path / "build_id.json" + _write_cache(path, "abc123", datetime.now()) + loaded = _read_cache(path) + assert loaded is not None + assert loaded["buildId"] == "abc123" + assert "saved_at" in loaded + + def test_read_missing_returns_none(self, tmp_path: Path): + assert _read_cache(tmp_path / "missing.json") is None + + def test_is_fresh_true_for_recent(self, tmp_path: Path): + path = tmp_path / "build_id.json" + _write_cache(path, "abc", datetime.now()) + assert _is_fresh(path, timedelta(hours=1), datetime.now()) + + def test_is_fresh_false_for_old(self, tmp_path: Path): + path = tmp_path / "build_id.json" + _write_cache(path, "abc", datetime.now() - timedelta(hours=2)) + assert not _is_fresh(path, timedelta(hours=1), datetime.now()) + + def test_is_fresh_false_for_missing(self, tmp_path: Path): + assert not _is_fresh( + tmp_path / "missing.json", timedelta(hours=1), datetime.now() + ) + + def test_is_fresh_false_for_garbage(self, tmp_path: Path): + path = tmp_path / "build_id.json" + path.write_text("{not json") + assert not _is_fresh(path, timedelta(hours=1), datetime.now()) + + +class TestFetchBuildId: + @pytest.mark.asyncio + async def test_uses_cached_when_fresh( + self, tmp_path: Path, saude_homepage_html: str + ): + transport = httpx.MockTransport( + lambda req: httpx.Response( + 200, content=saude_homepage_html.encode() + ) + ) + async with httpx.AsyncClient(transport=transport) as client: + cache_path = tmp_path / "build_id.json" + _write_cache(cache_path, "cached-id", datetime.now()) + build_id = await fetch_build_id( + client, + cache_path=cache_path, + homepage_url="https://dadosabertos.saude.gov.br/", + ) + assert build_id == "cached-id" + + @pytest.mark.asyncio + async def test_refetches_when_stale( + self, tmp_path: Path, saude_homepage_html: str + ): + transport = httpx.MockTransport( + lambda req: httpx.Response( + 200, content=saude_homepage_html.encode() + ) + ) + async with httpx.AsyncClient(transport=transport) as client: + cache_path = tmp_path / "build_id.json" + _write_cache( + cache_path, "stale-id", datetime.now() - timedelta(hours=48) + ) + build_id = await fetch_build_id( + client, + cache_path=cache_path, + homepage_url="https://dadosabertos.saude.gov.br/", + ) + assert build_id != "stale-id" + # The cache was overwritten with the fresh value + loaded = json.loads(cache_path.read_text()) + assert loaded["buildId"] == build_id + + @pytest.mark.asyncio + async def test_falls_back_to_stale_cache_on_homepage_failure( + self, tmp_path: Path + ): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503, content=b"service unavailable") + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + cache_path = tmp_path / "build_id.json" + _write_cache(cache_path, "stale-id", datetime.now()) + build_id = await fetch_build_id( + client, + cache_path=cache_path, + homepage_url="https://dadosabertos.saude.gov.br/", + ) + assert build_id == "stale-id" + + @pytest.mark.asyncio + async def test_raises_when_no_cache_and_homepage_fails( + self, tmp_path: Path + ): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503, content=b"service unavailable") + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + with pytest.raises(NoUsableBuildId): + await fetch_build_id( + client, + cache_path=tmp_path / "build_id.json", + homepage_url="https://dadosabertos.saude.gov.br/", + ) diff --git a/pysus/tests/api/saude/test_resources.py b/pysus/tests/api/saude/test_resources.py new file mode 100644 index 00000000..99c957bd --- /dev/null +++ b/pysus/tests/api/saude/test_resources.py @@ -0,0 +1,139 @@ +"""Tests for the pydantic resource / package models.""" + +from __future__ import annotations + +from pysus.api.saude.resources import ( + CatalogEntry, + CatalogPage, + CKANPackage, + Extra, + GroupRef, + Organization, + Resource, + TagRef, +) + + +class TestResourceModel: + def test_parses_full_resource(self, saude_dataset_page_props): + first = saude_dataset_page_props["resources"][0] + resource = Resource.model_validate(first) + assert resource.id == first["id"] + assert resource.name == first["name"] + assert resource.format == first["format"].upper() + assert resource.url == first["url"] + assert resource.position == first["position"] + assert resource.created is not None + assert resource.last_modified is not None + + def test_normalises_format_to_uppercase(self): + resource = Resource.model_validate( + { + "id": "abc", + "name": "x", + "format": "csv", + "url": "https://example.com/x.csv", + } + ) + assert resource.format == "CSV" + + def test_ignores_unknown_keys(self): + resource = Resource.model_validate( + { + "id": "abc", + "name": "x", + "format": "CSV", + "url": "https://example.com/x.csv", + "future_field_added_by_ckan": "noise", + } + ) + assert resource.id == "abc" + + def test_handles_null_dates(self): + resource = Resource.model_validate( + { + "id": "abc", + "name": "x", + "format": "CSV", + "url": "https://example.com/x.csv", + "created": None, + "last_modified": None, + } + ) + assert resource.created is None + assert resource.last_modified is None + + +class TestCKANPackageModel: + def test_parses_full_package(self, saude_dataset_page_props): + package = CKANPackage.model_validate(saude_dataset_page_props) + assert package.id == saude_dataset_page_props["id"] + assert package.name == "arboviroses-dengue" + assert package.num_resources == 83 + assert package.organization is not None + assert package.organization.id is not None + assert len(package.resources) == 83 + + def test_periodicity_property(self, saude_dataset_page_props): + package = CKANPackage.model_validate(saude_dataset_page_props) + assert package.periodicity == "Semanal" + + def test_contact_property(self, saude_dataset_page_props): + package = CKANPackage.model_validate(saude_dataset_page_props) + assert package.contact == "arboviroses@saude.gov.br" + + def test_ckan_id_alias(self, saude_dataset_page_props): + package = CKANPackage.model_validate(saude_dataset_page_props) + assert package.ckan_id == package.id + + def test_organization_populated(self, saude_dataset_page_props): + package = CKANPackage.model_validate(saude_dataset_page_props) + assert isinstance(package.organization, Organization) + + def test_groups_have_display_name(self, saude_dataset_page_props): + package = CKANPackage.model_validate(saude_dataset_page_props) + assert package.groups[0].display_name is not None + + def test_periodicity_missing_returns_none(self): + package = CKANPackage.model_validate( + { + "id": "abc", + "name": "x", + "title": "X", + "metadata_created": "2024-01-01T00:00:00", + "metadata_modified": "2024-01-01T00:00:00", + "num_resources": 0, + "extras": [], + } + ) + assert package.periodicity is None + assert package.contact is None + + +class TestCatalogEntryModel: + def test_parses_listing_entry(self, saude_catalog_payload): + first = saude_catalog_payload["pageProps"]["packages"][0] + entry = CatalogEntry.model_validate(first) + assert entry.name == first["name"] + assert entry.title == first["title"] + assert "CSV" in entry.formats + assert isinstance(entry.groups[0], GroupRef) + assert isinstance(entry.tags[0], TagRef) + + +class TestCatalogPageModel: + def test_parses_catalog_page(self, saude_catalog_payload): + page = CatalogPage.model_validate(saude_catalog_payload["pageProps"]) + assert page.number_of_packages == 138 + assert page.page == 1 + assert page.rows == 20 + assert len(page.packages) == 20 + assert "groups" in page.available_filters + assert "tags" in page.available_filters + + +class TestExtraModel: + def test_basic(self): + extra = Extra(key="Frequência de atualização", value="Semanal") + assert extra.key == "Frequência de atualização" + assert extra.value == "Semanal" From 9895b48454ae0c6eaf6a68797c23cdf663c7d150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 17 Aug 2026 22:13:42 -0300 Subject: [PATCH 2/4] feat(saude): add source-scoped dataset registry for OpenDataSUS --- pysus/api/saude/client.py | 82 ++++- pysus/api/saude/databases.py | 422 ++++++++++++++++++++++++ pysus/api/saude/metadata.py | 84 ++++- pysus/api/saude/models.py | 245 ++++++++++++++ pysus/api/types.py | 63 +++- pysus/tests/api/saude/test_databases.py | 134 ++++++++ pysus/tests/api/saude/test_models.py | 147 +++++++++ 7 files changed, 1162 insertions(+), 15 deletions(-) create mode 100644 pysus/api/saude/databases.py create mode 100644 pysus/api/saude/models.py create mode 100644 pysus/tests/api/saude/test_databases.py create mode 100644 pysus/tests/api/saude/test_models.py diff --git a/pysus/api/saude/client.py b/pysus/api/saude/client.py index cc2a8347..58a28408 100644 --- a/pysus/api/saude/client.py +++ b/pysus/api/saude/client.py @@ -5,9 +5,12 @@ from collections.abc import AsyncIterator, Callable from datetime import timedelta from pathlib import Path +from typing import TYPE_CHECKING, ClassVar import httpx +from pydantic import PrivateAttr from pysus import CACHEPATH +from pysus.api.models import BaseRemoteClient, BaseRemoteFile from .catalog import ( _DEFAULT_TTL, @@ -22,8 +25,11 @@ from .download import download_resource as _download_resource from .resources import CatalogEntry, CKANPackage, GroupRef, Resource, TagRef +if TYPE_CHECKING: # pragma: no cover + from .models import SaudeDataset -class SaudeClient: + +class SaudeClient(BaseRemoteClient): """Async client for the OpenDataSUS portal. The portal is a Next.js frontend over a CKAN backend. ``SaudeClient`` @@ -41,7 +47,12 @@ class SaudeClient: >>> asyncio.run(main()) """ - BASE_URL = "https://dadosabertos.saude.gov.br" + BASE_URL: ClassVar[str] = "https://dadosabertos.saude.gov.br" + + _cache_dir: Path = PrivateAttr() + _cache_ttl: timedelta = PrivateAttr(default=_DEFAULT_TTL) + _client: httpx.AsyncClient = PrivateAttr() + _build_id: str | None = PrivateAttr(default=None) def __init__( self, @@ -51,16 +62,51 @@ def __init__( timeout: float = 30.0, user_agent: str | None = None, ) -> None: - self.cache_dir = ( + super().__init__() + self._cache_dir = ( Path(cache_dir) if cache_dir else Path(CACHEPATH) / "saude" ) - self.cache_ttl = cache_ttl - self.cache_dir.mkdir(parents=True, exist_ok=True) + self._cache_ttl = cache_ttl + self._cache_dir.mkdir(parents=True, exist_ok=True) headers = {"User-Agent": user_agent or "pysus-saude/0.1 (research)"} self._client = httpx.AsyncClient( headers=headers, timeout=timeout, follow_redirects=True ) - self._build_id: str | None = None + self._build_id = None + + @property + def name(self) -> str: + """Return the short client name.""" + return "Saude" + + @property + def long_name(self) -> str: + """Return the human-readable client name.""" + return "Portal de Dados Abertos do SUS" + + @property + def description(self) -> str: + """Return a description of the client.""" + return ( + "Interface de acesso ao Portal de Dados Abertos do " + "Ministério da Saúde" + ) + + @property + def cache_dir(self) -> Path: + """Return the on-disk cache directory.""" + return self._cache_dir + + @property + def cache_ttl(self) -> timedelta: + """Return the cache TTL.""" + return self._cache_ttl + + async def connect(self) -> None: + """Ensure the underlying HTTP client is ready (no-op).""" + + async def login(self, **kwargs) -> None: + """Authenticate — the portal is public, so this is a no-op.""" async def __aenter__(self) -> SaudeClient: return self @@ -70,7 +116,17 @@ async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: async def close(self) -> None: """Close the underlying HTTP client.""" - await self._client.aclose() + if not self._client.is_closed: + await self._client.aclose() + + async def download( + self, + file: BaseRemoteFile, + output: Path, + callback: Callable[[int, int], None] | None = None, + ) -> Path: + """Download a remote Saude file to *output*.""" + return await file._download(output=output, callback=callback) async def _ensure_build_id(self, use_cache: bool = True) -> str: if self._build_id and not use_cache: @@ -181,6 +237,18 @@ async def fetch_resources( package = await self.fetch_dataset(slug, use_cache=use_cache) return package.resources + async def datasets(self, **kwargs) -> list[SaudeDataset]: + """Return the pre-configured Saude theme datasets. + + Each dataset is a :class:`SaudeDataset` built from the static + registry in :mod:`pysus.api.saude.databases`; ``content`` + resolves the CKAN packages belonging to the theme. + """ + from .databases import DATASET_SPECS + from .models import SaudeDataset + + return [SaudeDataset(spec=spec, client=self) for spec in DATASET_SPECS] + async def download_resource( self, slug: str, diff --git a/pysus/api/saude/databases.py b/pysus/api/saude/databases.py new file mode 100644 index 00000000..e60927de --- /dev/null +++ b/pysus/api/saude/databases.py @@ -0,0 +1,422 @@ +"""Pre-configured dataset registry for the OpenDataSUS portal. + +Each :class:`DatasetSpec` describes one theme of the SUS open-data +portal. The spec carries: + +- the canonical dataset ``name`` (uppercase, source-scoped); +- the CKAN ``group`` slug used to select packages from the catalog + (the portal groups its 138 datasets into 14 themes); +- ``slug_patterns`` / ``exclude_patterns`` — extra regex filters + applied over the full catalog listing (used when a theme is spread + across groups or when one group hosts several themes, e.g. SISAGUA + inside ``vigilancia-e-meio-ambiente``); +- the DEMAS ``tags`` and their REST ``endpoints`` (the structured + query API — consumed from Stage 3 onwards). + +Source differentiation: datasets that also exist on dados.gov.br or +DATASUS FTP (CNES, PNI, SIM, SINASC, Arboviroses) are declared **here +too**, as Saude-source declarations. The same logical dataset in +another source keeps its own declaration there; both are linked by +``identity.cross_origin_id`` (the shared CKAN UUID) at merge time, +never collapsed at declaration time. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +_YEAR_RE = re.compile(r"(20\d{2}|19\d{2})") + + +@dataclass(frozen=True) +class DatasetSpec: + """Static description of one Saude theme dataset.""" + + name: str + long_name: str + description: str + ckan_group: str | None = None + slug_patterns: tuple[str, ...] = () + exclude_patterns: tuple[str, ...] = () + demas_tags: tuple[str, ...] = () + endpoints: tuple[str, ...] = () + + def matches(self, slug: str) -> bool: + """Return True when *slug* belongs to this spec.""" + if self.exclude_patterns and any( + re.search(pat, slug, re.I) for pat in self.exclude_patterns + ): + return False + if not self.slug_patterns: + return True + return any(re.search(pat, slug, re.I) for pat in self.slug_patterns) + + +def parse_year(name: str) -> int | None: + """Extract a 4-digit year from a resource/endpoint name, if any.""" + match = _YEAR_RE.search(name or "") + if not match: + return None + year = int(match.group(1)) + return year if 1970 <= year <= 2100 else None + + +DATASET_SPECS: tuple[DatasetSpec, ...] = ( + DatasetSpec( + name="ARBOVIROSES", + long_name="Arboviroses", + description=( + "Notificações de arboviroses (dengue, chikungunya, zika e " + "febre amarela) registradas no Sinan, por ano e município." + ), + ckan_group="arboviroses", + demas_tags=("Agravo Arboviroses",), + endpoints=( + "/arboviroses/dengue", + "/arboviroses/chikungunya", + "/arboviroses/zikavirus", + "/arboviroses/febre-amarela-humanos-primatas-nao-humanos", + "/arboviroses/febre-amarela-epzootias", + ), + ), + DatasetSpec( + name="ASSISTENCIASAUDE", + long_name="Assistência à Saúde", + description=( + "Estabelecimentos hospitalares, leitos, Unidades Básicas " + "de Saúde e registros de ocupação hospitalar." + ), + ckan_group="assistencia-a-saude", + demas_tags=("Assistência à Saúde",), + endpoints=( + "/assistencia-a-saude/hospitais-e-leitos", + "/assistencia-a-saude/unidade-basicas-de-saude", + "/assistencia-a-saude/registro-de-ocupacao-hospitalar-covid-19", + ), + ), + DatasetSpec( + name="ATENCAOPRIMARIA", + long_name="Atenção Primária", + description=( + "Programa Mais Médicos (PMMB), Previne Brasil, SISAB e " + "ENANI-2019." + ), + ckan_group="atencao-primaria", + demas_tags=("Atenção Primária",), + endpoints=( + "/atencao-primaria/enani-2019", + "/atencao-primaria/pmmb-consolidado", + "/atencao-primaria/pmmb-serie-historica", + "/atencao-primaria/pmmb-relacao-nominal-coparticipacao", + "/atencao-primaria/pmmb-relatorio-historico-cadastro-cnes", + "/atencao-primaria/pmmb-especialista-consolidado", + "/atencao-primaria/pmmb-especialista-serie-historica", + "/atencao-primaria/pmmb-relacao-nominal-ativo", + "/atencao-primaria/cadastro-vinculado-programa-previne-brasil", + "/atencao-primaria/indicador-desempenho-programa-previne-brasil", + "/atencao-primaria/pmmb-especialista-relacao-nominal-ativo", + ), + ), + DatasetSpec( + name="BNAFAR", + long_name=( + "Base Nacional de Dados de Ações e Serviços da " + "Assistência Farmacêutica" + ), + description=( + "Estoque de medicamentos do Sistema Nacional de Gestão da " + "Assistência Farmacêutica (Hórus)." + ), + ckan_group="assistencia-farmaceutica", + demas_tags=("BNAFAR",), + endpoints=("/daf/estoque-medicamentos-bnafar-horus",), + ), + DatasetSpec( + name="CNES", + long_name="Cadastro Nacional de Estabelecimentos de Saúde", + description=( + "Estabelecimentos de saúde e tipos de unidade " + "cadastrados no CNES." + ), + slug_patterns=("cnes",), + demas_tags=("CNES",), + endpoints=( + "/cnes/tipounidades", + "/cnes/tipounidades/{codigo_tipo_unidade}", + "/cnes/estabelecimentos", + "/cnes/estabelecimentos/{codigo_cnes}", + ), + ), + DatasetSpec( + name="CIENCIATECNOLOGIA", + long_name="Ciência & Tecnologia", + description=( + "Conitec (PCDT, demandas, consultas públicas), Plataforma " + "Brasil, pesquisas em saúde Decit e indicadores RIPSA." + ), + ckan_group="ciencia-tecnologia", + slug_patterns=( + "^ripsa", + "^pesquisa_saude", + "^contribuicoes-de-consultas-publicas", + "^tecnologias-e-diretrizes", + "^pcdt", + ), + demas_tags=("Ciência & Tecnologia",), + endpoints=( + "/ciencia-tecnologia/dgits-contribuicoes-consultas-publicas", + "/ciencia-tecnologia/dgits-controle-demandas-conitec", + "/ciencia-tecnologia/dgits-controle-pcdt", + "/ciencia-tecnologia/dgits-tecnologias-diretrizes", + "/ciencia-tecnologia/plataformabr-pesquisa-saude", + "/ciencia-tecnologia/plataformabr-projeto-aprovado", + ), + ), + DatasetSpec( + name="DIAGNOSTICOSTRATAMENTOS", + long_name="Diagnósticos e Tratamentos", + description=( + "Protocolos Clínicos e Diretrizes Terapêuticas (PCDT) e " + "tecnologias para tratamento e prevenção." + ), + ckan_group="diagnosticos-e-tratamentos", + ), + DatasetSpec( + name="ECONOMIASAUDE", + long_name="Economia da Saúde", + description=("Banco de Preços em Saúde (BPS), ApuraSUS e SIOPS."), + ckan_group="economia-da-saude", + demas_tags=("Economia da Saúde",), + endpoints=( + "/economia-da-saude/bps", + "/economia-da-saude/sistema-de-apuracao-e-gestao-de-custos-" + "do-sus-apurasus", + ), + ), + DatasetSpec( + name="EDUCACAOSAUDE", + long_name="Educação em Saúde", + description=( + "Programa De Volta Para Casa (PVC) — desinstitucionalização " + "de pessoas com transtornos mentais." + ), + ckan_group="educacao-em-saude", + demas_tags=("Educação em Saúde",), + endpoints=("/educacao-em-saude/pvc",), + ), + DatasetSpec( + name="MACROSAUDE", + long_name="Macrorregião e Região de Saúde", + description=( + "Municípios com as informações de macrorregião e região " + "de saúde, e indicadores de gestão municipal (MGDI)." + ), + ckan_group="indicadores-de-saude", + demas_tags=("Macrorregião e Região de Saúde",), + endpoints=("/macrorregiao-e-regiao-de-saude/municipio",), + ), + DatasetSpec( + name="OUVIDORIA", + long_name="Ouvidoria", + description=( + "Manifestações registradas na Ouvidoria do SUS, por UF, " + "assunto e problema." + ), + slug_patterns=("ouvidor",), + demas_tags=("Ouvidoria",), + endpoints=("/ouvidoria/ouvidor2", "/ouvidoria/ouvidor3"), + ), + DatasetSpec( + name="OUTROSTEMAS", + long_name="Outros Temas", + description=( + "Coordenação de Estratégia de Dados (CED) — demandas " + "abertas para a equipe de banco de dados." + ), + slug_patterns=("^ced-coordenacao",), + demas_tags=("Outros Temas",), + endpoints=("/outros-temas/ced",), + ), + DatasetSpec( + name="PDA", + long_name="Saúde Digital", + description=( + "Plano de Dados Abertos e ações de saúde digital do " + "Ministério da Saúde." + ), + ckan_group="pda", + ), + DatasetSpec( + name="PREVENCAOPROMOCAO", + long_name="Prevenção e Promoção da Saúde", + description=( + "Distribuição de equipamentos de proteção individual e insumos." + ), + ckan_group="prevencao-e-promocao-da-saude", + demas_tags=("Prevenção e Promoção",), + endpoints=("/prevencao-e-promocao/distribuicao-epi-insumo",), + ), + DatasetSpec( + name="SISAGUA", + long_name=( + "Sistema de Informação da Vigilância da Qualidade da Água " + "para Consumo Humano" + ), + description=( + "Vigilância e controle mensal da qualidade da água para " + "consumo humano, captação, tratamento e abastecimento." + ), + ckan_group="vigilancia-e-meio-ambiente", + slug_patterns=("sisagua",), + demas_tags=("SISAGUA",), + endpoints=( + "/sisagua/vigilancia-parametros-basicos", + "/sisagua/controle-semestral", + "/sisagua/controle-mensal-parametros-basicos", + "/sisagua/pontos-de-captacao", + "/sisagua/cadastro-carro-pipa-populacao", + "/sisagua/cadastro-carro-pipa-procedencia", + "/sisagua/controle-mensal-amostras-fora-do-padrao", + "/sisagua/controle-mensal-demais-parametros", + "/sisagua/controle-mensal-infraestrutura-operacional", + "/sisagua/controle-mensal-plano-amostragem", + "/sisagua/populacao-abastecida", + "/sisagua/tratamento-de-agua", + "/sisagua/vigilancia-cianobacterias-e-cianotoxinas", + "/sisagua/vigilancia-demais-parametros", + ), + ), + DatasetSpec( + name="SISVAN", + long_name="Sistema de Vigilância Alimentar e Nutricional", + description=("Acompanhamento de estado nutricional da população."), + slug_patterns=("sisvan",), + demas_tags=("SISVAN",), + endpoints=("/sisvan/estado-nutricional",), + ), + DatasetSpec( + name="SAUDEINDIGENA", + long_name="Saúde Indígena", + description=( + "Siasi/SasiSUS e Sesai: morbidades, imunização, saúde " + "bucal, saneamento, óbitos e demografia da população " + "indígena assistida." + ), + ckan_group="saude-indigena", + demas_tags=("Saúde Indígena",), + endpoints=( + "/saude-indigena/sasisus-esgotamento-sanitario", + "/saude-indigena/sasi-sus-gerenciamento-de-residuos-solidos", + "/saude-indigena/acompanhamento-obra-infraestrutura-saude", + "/saude-indigena/" + "planilha-de-fornecimento-e-monitoramento-da-qualidade-da-" + "agua-acesso-a-agua", + "/saude-indigena/" + "planilha-registros-habilitacao-recebimento-incentivo", + "/saude-indigena/" + "indicadores-enfrentamento-monitoramento-covid19-indigenas", + "/saude-indigena/" + "sistema-de-atencao-a-saude-indigena-modulo-de-vigilancia-" + "alimentar-e-nutricional", + "/saude-indigena/siasi-acompanhamento-gestacional", + "/saude-indigena/siasi-modulo-morbidades", + "/saude-indigena/sesai-atendimentos", + "/saude-indigena/sesai-recursos-humanos", + "/saude-indigena/siasi-modulo-saude-bucal-ficha3", + "/saude-indigena/siasi-modulo-saude-bucal-ficha4", + "/saude-indigena/siasi-modulo-saude-bucal-ficha7", + ), + ), + DatasetSpec( + name="VACINACAO", + long_name="Vacinação", + description=( + "Doses aplicadas pelo PNI por ano, ESAVI e insumos " + "estratégicos (SIES)." + ), + ckan_group="vacinacao", + demas_tags=("Vacinação",), + endpoints=( + "/vacinacao/doses-aplicadas-pni-2020", + "/vacinacao/doses-aplicadas-pni-2021", + "/vacinacao/doses-aplicadas-pni-2022", + "/vacinacao/doses-aplicadas-pni-2023", + "/vacinacao/doses-aplicadas-pni-2024", + "/vacinacao/doses-aplicadas-pni-2025", + "/vacinacao/doses-aplicadas-pni-2026", + "/vacinacao/esavi", + "/vacinacao/sistema-de-informacao-de-insumos-estrategicos", + ), + ), + DatasetSpec( + name="VIGILANCIAMEIOAMBIENTE", + long_name="Vigilância e Meio Ambiente", + description=( + "SRAG, síndrome gripal, SIM, Sinasc e mpox — dados de " + "vigilância epidemiológica e ambiental." + ), + ckan_group="vigilancia-e-meio-ambiente", + exclude_patterns=("sisagua",), + demas_tags=("Vigilância e Meio Ambiente",), + endpoints=( + "/vigilancia-e-meio-ambiente/" + "notificacoes-de-sindrome-gripal-leve-2020", + "/vigilancia-e-meio-ambiente/" + "notificacoes-de-sindrome-gripal-leve-2021", + "/vigilancia-e-meio-ambiente/" + "notificacoes-de-sindrome-gripal-leve-2022", + "/vigilancia-e-meio-ambiente/" + "notificacoes-de-sindrome-gripal-leve-2023", + "/vigilancia-e-meio-ambiente/" + "notificacoes-de-sindrome-gripal-leve-2024", + "/vigilancia-e-meio-ambiente/" + "sistema-de-informacao-sobre-mortalidade", + "/vigilancia-e-meio-ambiente/" + "sistema-de-informacao-sobre-nascidos-vivos", + "/vigilancia-e-meio-ambiente/srag-2009-2012", + "/vigilancia-e-meio-ambiente/srag-2013-2018", + "/vigilancia-e-meio-ambiente/srag-2019-2026", + "/vigilancia-e-meio-ambiente/mpox", + ), + ), +) + +#: Lookup by canonical name. +SPECS_BY_NAME: dict[str, DatasetSpec] = { + spec.name: spec for spec in DATASET_SPECS +} + + +def spec_for(slug: str, groups: tuple[str, ...] = ()) -> DatasetSpec | None: + """Return the most specific spec whose filters match *slug*. + + ``groups`` is the package's CKAN group membership (from the + catalog listing); specs that declare a ``ckan_group`` only match + packages that actually belong to that group. Specs with explicit + ``slug_patterns`` take precedence over group-only specs. + """ + candidates = [ + spec + for spec in DATASET_SPECS + if spec.matches(slug) + and (not spec.ckan_group or spec.ckan_group in groups) + ] + if not candidates: + return None + return max( + candidates, + key=lambda spec: ( + bool(spec.slug_patterns), + len(spec.slug_patterns), + ), + ) + + +__all__ = [ + "DATASET_SPECS", + "DatasetSpec", + "SPECS_BY_NAME", + "parse_year", + "spec_for", +] diff --git a/pysus/api/saude/metadata.py b/pysus/api/saude/metadata.py index 66ec1b9e..721731a9 100644 --- a/pysus/api/saude/metadata.py +++ b/pysus/api/saude/metadata.py @@ -32,7 +32,17 @@ class SaudeDatasetExtractor(MetadataExtractor): - """Build a dataset-level bag from a CKAN package.""" + """Build a dataset-level bag from a CKAN package or a theme spec. + + Two input shapes are accepted: + + - a :class:`~pysus.api.saude.resources.CKANPackage` (a concrete + catalog dataset) — full facets incl. license, periodicity and + the cross-origin CKAN UUID; + - a spec-backed :class:`~pysus.api.saude.models.SaudeDataset` + (a theme grouping several packages) — identity/description only, + since a theme is not a single CKAN record. + """ origin = "saude" @@ -47,8 +57,11 @@ def supported_facets(self) -> set[str]: } def _extract(self, obj: Any) -> MetadataBag: - package = obj + if hasattr(obj, "metadata_created"): + return self._from_package(obj) + return self._from_spec(obj) + def _from_package(self, package: Any) -> MetadataBag: organization = "" if package.organization is not None: organization = ( @@ -101,6 +114,32 @@ def _extract(self, obj: Any) -> MetadataBag: ), ) + def _from_spec(self, dataset: Any) -> MetadataBag: + spec = dataset.spec + return MetadataBag( + identity=IdentityFacet( + name=spec.name, + slug=spec.name.lower(), + aliases=list(spec.demas_tags), + ), + description=DescriptionFacet( + title=spec.long_name, + long_name=spec.long_name, + description=spec.description, + themes=[ + spec.ckan_group.replace("-", " ") if spec.ckan_group else "" + ], + ), + provenance=ProvenanceFacet( + origin=self.origin, + organization="Ministério da Saúde", + ), + structure=StructureFacet( + file_count=len(spec.endpoints), + ), + access=AccessFacet(policy="active/public/open"), + ) + # ---------------------------------------------------------------------- # Group (theme) @@ -108,7 +147,7 @@ def _extract(self, obj: Any) -> MetadataBag: class SaudeGroupExtractor(MetadataExtractor): - """Build a group-level bag from a CKAN ``GroupRef``.""" + """Build a group-level bag from a ``GroupRef`` or a ``CatalogEntry``.""" origin = "saude" @@ -117,12 +156,40 @@ def supported_facets(self) -> set[str]: def _extract(self, obj: Any) -> MetadataBag: group = obj - display = group.display_name or group.name or "" + # SaudeGroup wraps a CatalogEntry + entry = getattr(group, "entry", None) + if entry is not None: + return MetadataBag( + identity=IdentityFacet(name=entry.name, slug=entry.name), + description=DescriptionFacet( + title=entry.title, + long_name=entry.title, + description=entry.notes or "", + tags=[tag.name for tag in entry.tags], + themes=[g.name for g in entry.groups], + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + # GroupRef (theme) — has display_name + if hasattr(group, "display_name"): + display = group.display_name or group.name or "" + return MetadataBag( + identity=IdentityFacet(name=group.name, slug=group.name), + description=DescriptionFacet( + title=display, + themes=[group.name], + ), + provenance=ProvenanceFacet(origin=self.origin), + ) + # CatalogEntry — has title/notes return MetadataBag( identity=IdentityFacet(name=group.name, slug=group.name), description=DescriptionFacet( - title=display, - themes=[group.name], + title=group.title, + long_name=group.title, + description=group.notes or "", + tags=[tag.name for tag in group.tags], + themes=[g.name for g in group.groups], ), provenance=ProvenanceFacet(origin=self.origin), ) @@ -150,7 +217,10 @@ def supported_facets(self) -> set[str]: } def _extract(self, obj: Any) -> MetadataBag: - resource = obj + # SaudeFile wraps a Resource in ``record`` + resource = getattr(obj, "record", None) + if resource is None: + resource = obj return MetadataBag( identity=IdentityFacet( name=resource.name, diff --git a/pysus/api/saude/models.py b/pysus/api/saude/models.py new file mode 100644 index 00000000..edd4df9f --- /dev/null +++ b/pysus/api/saude/models.py @@ -0,0 +1,245 @@ +"""Remote models for the Saude (dadosabertos.saude.gov.br) client. + +Maps the portal's CKAN catalog into the PySUS remote hierarchy: + +- :class:`SaudeDataset` — one theme dataset (from + :mod:`~pysus.api.saude.databases` specs); its content is the set of + CKAN packages matched by the spec's group/patterns; +- :class:`SaudeGroup` — one CKAN package (e.g. + ``arboviroses-dengue``); its files are the package resources; +- :class:`SaudeFile` — one downloadable resource (CSV/JSON/XML/PDF). + +The same logical dataset also exists on other sources (dados.gov.br, +DATASUS FTP) — those keep their own declarations; linkage across +sources happens at merge time via ``identity.cross_origin_id`` (the +shared CKAN UUID), never by collapsing declarations here. +""" + +from __future__ import annotations + +from collections.abc import Callable +from datetime import datetime +from pathlib import Path +from typing import ClassVar, cast + +from pydantic import Field, PrivateAttr +from pysus.api.models import BaseRemoteDataset, BaseRemoteFile, BaseRemoteGroup +from pysus.api.types import State + +from .databases import DatasetSpec, parse_year +from .download import download_resource +from .errors import ResourceNotFound +from .metadata import ( + SaudeDatasetExtractor, + SaudeFileExtractor, + SaudeGroupExtractor, +) +from .resources import CatalogEntry, CKANPackage, Resource + +__all__ = ["SaudeDataset", "SaudeFile", "SaudeGroup"] + + +class SaudeFile(BaseRemoteFile): + """A downloadable resource of a CKAN package on OpenDataSUS.""" + + record: Resource + type: str = "File" + extractor_types: ClassVar[list] = [SaudeFileExtractor] + + @property + def extension(self) -> str: + """Return the resource format lower-cased (e.g. ``.csv.zip``).""" + fmt = (self.record.format or "").lower() + return f".{fmt}" if fmt else Path(self.record.url).suffix + + @property + def size(self) -> int: + """Return the resource size in bytes (0 when unknown).""" + return self.record.size or 0 + + @property + def modify(self) -> datetime: + """Return the last modification timestamp of the resource. + + Raises + ------ + ValueError + If the resource carries no modification date. + """ + m = self.record.last_modified or self.record.metadata_modified + if not m: + raise ValueError("File requires a modify date") + return m + + @property + def year(self) -> int | None: + """Return the year parsed from the resource name, if any.""" + return parse_year(self.record.name) + + @property + def month(self) -> int | None: + """Return the month — resource names carry no month info.""" + return None + + @property + def state(self) -> State | None: + """Return the state — resources are national scope.""" + return None + + async def _download( + self, + output: Path | None = None, + callback: Callable[[int, int], None] | None = None, + ) -> Path: + """Download the resource to *output*.""" + if output is None: + output = Path(f"./{self.basename}") + group = cast("SaudeGroup", self.group) + package = await group.fetch_package() + client = cast("SaudeClient", self.client) + return await download_resource( + client._client, + package, + resource_id=self.record.id, + dest_dir=output.parent, + progress=callback, + ) + + async def fetch_size(self) -> int: + """Fetch the resource size from the remote server.""" + return self.record.size or 0 + + +class SaudeGroup(BaseRemoteGroup): + """One CKAN package (dataset) inside a Saude theme dataset.""" + + entry: CatalogEntry + extractor_types: ClassVar[list] = [SaudeGroupExtractor] + _package: CKANPackage | None = PrivateAttr(default=None) + + @property + def name(self) -> str: + """Return the package slug.""" + return self.entry.name + + @property + def long_name(self) -> str: + """Return the package title.""" + return self.entry.title + + @property + def description(self) -> str: + """Return the package notes.""" + return self.entry.notes or "" + + async def fetch_package(self, use_cache: bool = True) -> CKANPackage: + """Fetch (and cache) the full CKAN package for this group.""" + if self._package is None: + client = cast("SaudeClient", self.dataset.client) + self._package = await client.fetch_dataset( + self.entry.name, use_cache=use_cache + ) + return self._package + + @property + async def package(self) -> CKANPackage: + """The full CKAN package (fetched lazily).""" + return await self.fetch_package() + + async def _fetch_files(self) -> list[BaseRemoteFile]: + """Build SaudeFile objects from the package resources. + + Resources with format ``API`` are documentation links and are + skipped; PDF dictionaries are kept (they document the columns). + """ + package = await self.fetch_package() + files: list[BaseRemoteFile] = [] + for resource in package.resources: + if (resource.format or "").upper() == "API": + continue + files.append( + SaudeFile( + record=resource, + dataset=self.dataset, + group=self, + path=Path(resource.url), + ) + ) + return files + + async def resource(self, resource_id: str) -> Resource: + """Return the resource with the given id, if it exists.""" + package = await self.fetch_package() + for resource in package.resources: + if resource.id == resource_id: + return resource + raise ResourceNotFound( + f"Resource '{resource_id}' not found in '{self.name}'." + ) + + +class SaudeDataset(BaseRemoteDataset): + """A theme dataset of the OpenDataSUS portal. + + Instances are created from a :class:`DatasetSpec` and expose the + portal's packages (via ``content`` → :class:`SaudeGroup`) and the + DEMAS endpoints (via ``spec.endpoints`` — Stage 3 turns these + into queryable files). + """ + + spec: DatasetSpec = Field(exclude=True) + client: SaudeClient = Field(exclude=True) + extractor_types: ClassVar[list] = [SaudeDatasetExtractor] + + @property + def name(self) -> str: + """Return the canonical dataset name (e.g. ``SISAGUA``).""" + return self.spec.name + + @property + def long_name(self) -> str: + """Return the human-readable dataset name.""" + return self.spec.long_name + + @property + def description(self) -> str: + """Return the dataset description.""" + return self.spec.description + + @property + def endpoints(self) -> tuple[str, ...]: + """Return the DEMAS REST endpoints for this dataset.""" + return self.spec.endpoints + + async def _fetch_content(self) -> list[SaudeGroup]: + """Return the SaudeGroups (CKAN packages) of this dataset.""" + spec = self.spec + groups: list[SaudeGroup] = [] + if spec.ckan_group: + async for entry in self.client.iter_datasets(group=spec.ckan_group): + if spec.matches(entry.name): + groups.append(SaudeGroup(entry=entry, dataset=self)) + else: + async for entry in self.client.iter_datasets(): + if spec.matches(entry.name): + groups.append(SaudeGroup(entry=entry, dataset=self)) + return groups + + +# Rebuild pydantic models with postponed annotations so their fields +# are fully defined (the ``client: SaudeClient`` forward reference +# cannot resolve at class-definition time). +import pydantic # noqa: E402 + +from .client import SaudeClient # noqa: E402 + +for _model in list(globals().values()): + if ( + isinstance(_model, type) + and issubclass(_model, pydantic.BaseModel) + and _model.__module__ == __name__ + ): + try: + _model.model_rebuild(_types_namespace={"SaudeClient": SaudeClient}) + except Exception: # noqa: B902 — rebuild best effort + pass diff --git a/pysus/api/types.py b/pysus/api/types.py index ce940d73..e32652e9 100644 --- a/pysus/api/types.py +++ b/pysus/api/types.py @@ -19,7 +19,7 @@ def _validate_s3_bucket(v: str) -> str: def _validate_origin(v: str) -> str: - valid = (FTP, DADOSGOV, DUCKLAKE) + valid = (FTP, DADOSGOV, DUCKLAKE, SAUDE) assert v in valid, f"Invalid origin: {v!r}" return v @@ -67,6 +67,24 @@ def _validate_dataset_name(v: str) -> str: "IBGE", "CNES", "CIHA", + "ARBOVIROSES", + "ASSISTENCIASAUDE", + "ATENCAOPRIMARIA", + "BNAFAR", + "CIENCIATECNOLOGIA", + "DIAGNOSTICOSTRATAMENTOS", + "ECONOMIASAUDE", + "EDUCACAOSAUDE", + "MACROSAUDE", + "OUVIDORIA", + "OUTROSTEMAS", + "PDA", + "PREVENCAOPROMOCAO", + "SISAGUA", + "SISVAN", + "SAUDEINDIGENA", + "VACINACAO", + "VIGILANCIAMEIOAMBIENTE", ) assert v in valid, f"Invalid dataset name: {v!r}" return v @@ -109,6 +127,7 @@ def _validate_state(v: str) -> str: FTP: Annotated[str, AfterValidator(_validate_origin)] = "FTP" DADOSGOV: Annotated[str, AfterValidator(_validate_origin)] = "DadosGov" DUCKLAKE: Annotated[str, AfterValidator(_validate_origin)] = "DuckLake" +SAUDE: Annotated[str, AfterValidator(_validate_origin)] = "Saude" S3_ENDPOINT: Annotated[str, AfterValidator(_validate_s3_endpoint)] = ( "nbg1.your-objectstorage.com" @@ -143,6 +162,48 @@ def _validate_state(v: str) -> str: IBGE: Annotated[str, AfterValidator(_validate_dataset_name)] = "IBGE" CNES: Annotated[str, AfterValidator(_validate_dataset_name)] = "CNES" CIHA: Annotated[str, AfterValidator(_validate_dataset_name)] = "CIHA" +ARBOVIROSES: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "ARBOVIROSES" +) +ASSISTENCIASAUDE: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "ASSISTENCIASAUDE" +) +ATENCAOPRIMARIA: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "ATENCAOPRIMARIA" +) +BNAFAR: Annotated[str, AfterValidator(_validate_dataset_name)] = "BNAFAR" +CIENCIATECNOLOGIA: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "CIENCIATECNOLOGIA" +) +DIAGNOSTICOSTRATAMENTOS: Annotated[ + str, AfterValidator(_validate_dataset_name) +] = "DIAGNOSTICOSTRATAMENTOS" +ECONOMIASAUDE: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "ECONOMIASAUDE" +) +EDUCACAOSAUDE: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "EDUCACAOSAUDE" +) +MACROSAUDE: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "MACROSAUDE" +) +OUVIDORIA: Annotated[str, AfterValidator(_validate_dataset_name)] = "OUVIDORIA" +OUTROSTEMAS: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "OUTROSTEMAS" +) +PDA: Annotated[str, AfterValidator(_validate_dataset_name)] = "PDA" +PREVENCAOPROMOCAO: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "PREVENCAOPROMOCAO" +) +SISAGUA: Annotated[str, AfterValidator(_validate_dataset_name)] = "SISAGUA" +SISVAN: Annotated[str, AfterValidator(_validate_dataset_name)] = "SISVAN" +SAUDEINDIGENA: Annotated[str, AfterValidator(_validate_dataset_name)] = ( + "SAUDEINDIGENA" +) +VACINACAO: Annotated[str, AfterValidator(_validate_dataset_name)] = "VACINACAO" +VIGILANCIAMEIOAMBIENTE: Annotated[ + str, AfterValidator(_validate_dataset_name) +] = "VIGILANCIAMEIOAMBIENTE" Origin: TypeAlias = Annotated[str, AfterValidator(_validate_origin)] ColumnType: TypeAlias = Annotated[str, AfterValidator(_validate_column_type)] diff --git a/pysus/tests/api/saude/test_databases.py b/pysus/tests/api/saude/test_databases.py new file mode 100644 index 00000000..257ff4ed --- /dev/null +++ b/pysus/tests/api/saude/test_databases.py @@ -0,0 +1,134 @@ +"""Tests for the Saude dataset registry (databases.py).""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError + +import pytest +from pysus.api.saude.databases import ( + DATASET_SPECS, + SPECS_BY_NAME, + parse_year, + spec_for, +) + + +class TestDatasetSpecs: + def test_19_specs_registered(self): + assert len(DATASET_SPECS) == 19 + + def test_names_are_unique_and_uppercase(self): + names = [s.name for s in DATASET_SPECS] + assert len(names) == len(set(names)) + assert all(n == n.upper() for n in names) + + def test_every_spec_has_description(self): + for spec in DATASET_SPECS: + assert spec.long_name + assert spec.description + + def test_specs_by_name_lookup(self): + assert SPECS_BY_NAME["SISAGUA"] is DATASET_SPECS[14] + + def test_endpoints_are_absolute_paths(self): + for spec in DATASET_SPECS: + for endpoint in spec.endpoints: + assert endpoint.startswith("/") + + def test_all_specs_have_source_scope(self): + # CNES/VACINACAO etc. are Saude-source declarations even when + # the same logical dataset exists on dados.gov.br / DATASUS FTP + for spec in DATASET_SPECS: + assert spec.name in { + "ARBOVIROSES", + "ASSISTENCIASAUDE", + "ATENCAOPRIMARIA", + "BNAFAR", + "CNES", + "CIENCIATECNOLOGIA", + "DIAGNOSTICOSTRATAMENTOS", + "ECONOMIASAUDE", + "EDUCACAOSAUDE", + "MACROSAUDE", + "OUVIDORIA", + "OUTROSTEMAS", + "PDA", + "PREVENCAOPROMOCAO", + "SISAGUA", + "SISVAN", + "SAUDEINDIGENA", + "VACINACAO", + "VIGILANCIAMEIOAMBIENTE", + } + + +class TestSpecMatches: + def test_group_only_spec_matches_within_group(self): + spec = SPECS_BY_NAME["ARBOVIROSES"] + assert spec.matches("arboviroses-dengue") + assert spec.matches("any-slug") # group filter is external + + def test_pattern_spec(self): + spec = SPECS_BY_NAME["SISVAN"] + assert spec.matches("sisvan-estado-nutricional") + assert not spec.matches("mpox") + + def test_exclude_pattern(self): + spec = SPECS_BY_NAME["VIGILANCIAMEIOAMBIENTE"] + assert spec.matches("mpox") + assert not spec.matches("sisagua-controle-semestral") + + +class TestSpecFor: + def test_pattern_spec_wins_over_group_spec(self): + spec = spec_for( + "cnes-cadastro-nacional-de-estabelecimentos-de-saude", + ("assistencia-a-saude",), + ) + assert spec is not None + assert spec.name == "CNES" + + def test_sisagua_in_vigilancia_group(self): + spec = spec_for( + "sisagua-controle-semestral", + ("vigilancia-e-meio-ambiente",), + ) + assert spec is not None + assert spec.name == "SISAGUA" + + def test_mpox_in_vigilancia_group(self): + spec = spec_for("mpox", ("vigilancia-e-meio-ambiente",)) + assert spec is not None + assert spec.name == "VIGILANCIAMEIOAMBIENTE" + + def test_no_match(self): + assert spec_for("nao-existe", ("arboviroses",)) is not None + assert spec_for("nao-existe", ("grupo-inexistente",)) is None + + def test_group_only_spec(self): + spec = spec_for("arboviroses-dengue", ("arboviroses",)) + assert spec is not None + assert spec.name == "ARBOVIROSES" + + +class TestParseYear: + def test_year_from_resource_name(self): + assert parse_year("Dengue - 2024") == 2024 + assert parse_year("Dengue - 2000") == 2000 + + def test_year_from_endpoint(self): + assert parse_year("/vacinacao/doses-aplicadas-pni-2024") == 2024 + + def test_no_year(self): + assert parse_year("Dicionário de dados") is None + + def test_invalid_year(self): + assert parse_year("Dengue - 1899") is None + assert parse_year("Dengue - 2101") is None + + +class TestDatasetSpecIsFrozen: + def test_frozen(self): + spec = SPECS_BY_NAME["BNAFAR"] + with pytest.raises(FrozenInstanceError): + spec.name = "OTHER" # type: ignore[misc] diff --git a/pysus/tests/api/saude/test_models.py b/pysus/tests/api/saude/test_models.py new file mode 100644 index 00000000..b859779a --- /dev/null +++ b/pysus/tests/api/saude/test_models.py @@ -0,0 +1,147 @@ +"""Tests for the Saude remote models (Dataset/Group/File).""" + +from __future__ import annotations + +import pytest +from pysus.api.metadata.models import MetadataBag +from pysus.api.saude.client import SaudeClient +from pysus.api.saude.databases import SPECS_BY_NAME +from pysus.api.saude.models import SaudeDataset, SaudeFile, SaudeGroup +from pysus.api.saude.resources import CatalogEntry, CKANPackage + + +@pytest.fixture +def catalog_entry() -> CatalogEntry: + from pysus.api.saude.resources import GroupRef, TagRef + + return CatalogEntry( + name="arboviroses-dengue", + title="Sinan/Dengue", + notes="Notificações de dengue.", + formats=["PDF", "CSV", "JSON", "XML"], + groups=[GroupRef(name="arboviroses", display_name="Arboviroses")], + tags=[TagRef(name="dengue", display_name="Dengue")], + ) + + +@pytest.fixture +def saude_dataset(saude_client: SaudeClient) -> SaudeDataset: + return SaudeDataset(spec=SPECS_BY_NAME["ARBOVIROSES"], client=saude_client) + + +class TestSaudeDataset: + def test_name_from_spec(self, saude_dataset): + assert saude_dataset.name == "ARBOVIROSES" + assert saude_dataset.long_name == "Arboviroses" + + def test_endpoints_from_spec(self, saude_dataset): + assert len(saude_dataset.endpoints) == 5 + assert "/arboviroses/dengue" in saude_dataset.endpoints + + def test_metadata_from_spec(self, saude_dataset): + bag = saude_dataset.metadata + assert isinstance(bag, MetadataBag) + assert bag.identity.name == "ARBOVIROSES" + assert bag.provenance.origin == "saude" + assert bag.description.title == "Arboviroses" + + @pytest.mark.asyncio + async def test_content_returns_groups(self, saude_dataset): + # catalog filtered by the spec's ckan_group against the mock + content = await saude_dataset.content + assert len(content) >= 1 + names = [g.name for g in content] + assert "arboviroses-dengue" in names + + +class TestSaudeGroup: + def test_properties(self, saude_dataset, catalog_entry): + group = SaudeGroup(entry=catalog_entry, dataset=saude_dataset) + assert group.name == "arboviroses-dengue" + assert group.long_name == "Sinan/Dengue" + assert group.description.startswith("Notificações") + + def test_metadata(self, saude_dataset, catalog_entry): + group = SaudeGroup(entry=catalog_entry, dataset=saude_dataset) + bag = group.metadata + assert bag.identity.name == "arboviroses-dengue" + assert bag.description.title == "Sinan/Dengue" + assert bag.provenance.origin == "saude" + + @pytest.mark.asyncio + async def test_files_from_package(self, saude_dataset, catalog_entry): + group = SaudeGroup(entry=catalog_entry, dataset=saude_dataset) + files = await group.files + formats = {f.record.format for f in files} + assert "API" not in formats + assert "CSV" in formats + assert all(isinstance(f, SaudeFile) for f in files) + + @pytest.mark.asyncio + async def test_package_cached(self, saude_dataset, catalog_entry): + group = SaudeGroup(entry=catalog_entry, dataset=saude_dataset) + first = await group.fetch_package() + second = await group.fetch_package() + assert isinstance(first, CKANPackage) + assert first is second + + +class TestSaudeFile: + @pytest.mark.asyncio + async def test_file_from_package(self, saude_dataset, catalog_entry): + group = SaudeGroup(entry=catalog_entry, dataset=saude_dataset) + files = await group.files + csv_file = next(f for f in files if f.record.format == "CSV") + assert csv_file.extension == ".csv" + assert csv_file.year is not None + assert csv_file.state is None + assert csv_file.month is None + + def test_metadata_from_resource(self, saude_dataset, catalog_entry): + from pysus.api.saude.resources import Resource + + resource = Resource( + id="abc", + name="Dengue - 2024", + format="CSV", + url="https://example.com/x.csv.zip", + ) + file = SaudeFile( + record=resource, + dataset=saude_dataset, + group=SaudeGroup(entry=catalog_entry, dataset=saude_dataset), + path=resource.url, + ) + bag = file.metadata + assert bag.identity.name == "Dengue - 2024" + assert bag.access.format == "CSV" + assert bag.provenance.origin == "saude" + + @pytest.mark.asyncio + async def test_download_writes_file( + self, saude_dataset, catalog_entry, tmp_path + ): + group = SaudeGroup(entry=catalog_entry, dataset=saude_dataset) + files = await group.files + csv_file = next(f for f in files if f.record.format == "CSV") + path = await csv_file._download(output=tmp_path / "out.csv.zip") + assert path.exists() + assert path.stat().st_size > 0 + + +class TestSourceDifferentiation: + """Same logical dataset on another source keeps its own declaration.""" + + def test_cnes_is_declared_as_saude_source(self): + # CNES exists on DadosGov and FTP too — the Saude spec is a + # separate, source-scoped declaration, not a reference to them. + spec = SPECS_BY_NAME["CNES"] + assert spec.ckan_group is None + assert spec.slug_patterns == ("cnes",) + assert len(spec.endpoints) == 4 + + def test_vacinacao_overlaps_pni(self): + # PNI on DadosGov ↔ VACINACAO theme on Saude: different + # declarations, linked later via identity.cross_origin_id. + spec = SPECS_BY_NAME["VACINACAO"] + assert any("doses-aplicadas-pni" in e for e in spec.endpoints) From e05e2767d3ef8f62c243b82c4226b4aa61bc3184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Tue, 18 Aug 2026 12:26:20 -0300 Subject: [PATCH 3/4] chore: ignore waypoints and roadmaps from repo --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1b762be6..40cc3879 100644 --- a/.gitignore +++ b/.gitignore @@ -191,3 +191,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ pyrightconfig.json +roadmap.md From 9ff14fd8f79799b4d90de9a003617b694cec4bba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Tue, 18 Aug 2026 14:16:12 -0300 Subject: [PATCH 4/4] chore: include JSONL parsing --- pysus/api/extensions.py | 96 ++++++++++++++++++++++++++++++++++++++++- pysus/api/types.py | 2 + pysus/cli/__init__.py | 3 ++ 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/pysus/api/extensions.py b/pysus/api/extensions.py index b561b0e8..447a8a92 100644 --- a/pysus/api/extensions.py +++ b/pysus/api/extensions.py @@ -7,7 +7,7 @@ import shutil import tarfile import zipfile -from collections.abc import AsyncGenerator, Callable +from collections.abc import AsyncGenerator, Callable, Iterator from datetime import datetime from pathlib import Path @@ -770,6 +770,100 @@ async def stream( yield await self.load() +class JSONL(BaseTabularFile): + """Represents a JSON Lines file — one JSON object per line. + + Used by the Saude client to persist paginated DEMAS REST rows. + """ + + type: FileType = Field("JSONL") + _columns_cache: list["Column"] | None = PrivateAttr(default=None) + _rows_cache: int | None = PrivateAttr(default=None) + + def _read_lines(self) -> Iterator[dict]: + """Yield decoded JSON objects from the file, line by line.""" + + def _gen(): + with open(self.path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + yield json.loads(line) + + return _gen() + + @property + def columns(self) -> list["Column"]: + """Return the column metadata from the first record.""" + if self._columns_cache is not None: + return self._columns_cache + if self.path.stat().st_size == 0: + self._columns_cache = [] + return self._columns_cache + sample: dict = {} + with open(self.path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + sample = json.loads(line) + break + if not sample: + self._columns_cache = [] + return self._columns_cache + df = pd.DataFrame([sample]) + self._columns_cache = [ + Column.from_schema(name=col, dtype=_map_dtype(str(dt))) + for col, dt in zip(df.columns, df.dtypes) + ] + return self._columns_cache + + @property + def rows(self) -> int: + """Return the number of JSON records in the file.""" + if self._rows_cache is not None: + return self._rows_cache + count = 0 + with open(self.path, encoding="utf-8") as fh: + for line in fh: + if line.strip(): + count += 1 + self._rows_cache = count + return self._rows_cache + + async def load(self) -> pd.DataFrame: + """Read the entire JSONL file into a DataFrame.""" + + def _load(): + return pd.read_json(self.path, lines=True) + + return await to_thread.run_sync(_load) + + async def stream( + self, + chunk_size: int = 10000, + ) -> AsyncGenerator[pd.DataFrame, None]: + """Yield the JSONL records in batches of the given size.""" + + def _chunks(): + batch: list[dict] = [] + with open(self.path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + batch.append(json.loads(line)) + if len(batch) >= chunk_size: + yield pd.DataFrame(batch) + batch = [] + if batch: + yield pd.DataFrame(batch) + + for chunk in _chunks(): + yield chunk + await asyncio.sleep(0) + + class PDF(BaseLocalFile): """Represents a PDF file.""" diff --git a/pysus/api/types.py b/pysus/api/types.py index e32652e9..2aeca372 100644 --- a/pysus/api/types.py +++ b/pysus/api/types.py @@ -45,6 +45,7 @@ def _validate_file_type(v: str) -> str: "PARQUET", "CSV", "JSON", + "JSONL", "PDF", "DBC", "DBF", @@ -148,6 +149,7 @@ def _validate_state(v: str) -> str: PARQUET: Annotated[str, AfterValidator(_validate_file_type)] = "PARQUET" CSV: Annotated[str, AfterValidator(_validate_file_type)] = "CSV" JSON: Annotated[str, AfterValidator(_validate_file_type)] = "JSON" +JSONL: Annotated[str, AfterValidator(_validate_file_type)] = "JSONL" PDF: Annotated[str, AfterValidator(_validate_file_type)] = "PDF" DBC: Annotated[str, AfterValidator(_validate_file_type)] = "DBC" DBF: Annotated[str, AfterValidator(_validate_file_type)] = "DBF" diff --git a/pysus/cli/__init__.py b/pysus/cli/__init__.py index e6b514d9..f882ddff 100644 --- a/pysus/cli/__init__.py +++ b/pysus/cli/__init__.py @@ -5,7 +5,10 @@ import typer from pysus import __version__ +from .saude import app as saude_app + app = typer.Typer(help="PySUS CLI") +app.add_typer(saude_app, name="saude") def _is_colab() -> bool: