pyproject.toml

[build-system]
requires = ["hatchling>=1.21"]
build-backend = "hatchling.build"

[project]
# Distribution name on PyPI. The import package stays `freshdata`, but the
# distribution MUST remain `freshdata-cleaner`: PyPI rejects `freshdata` as
# too similar to the existing `fresh-data` project (see commit 88aa495).
name = "freshdata-cleaner"
version = "2.1.0"
description = "Fast, safe, automatic data cleaning for real-world tabular data."
readme = "README.md"
requires-python = ">=3.9"
license = { text = "MIT" }
authors = [
    { name = "Johnny Wilson Dougherty", email = "johnnydougherty09@gmail.com" },
]
keywords = [
    "data-cleaning",
    "data-quality",
    "pandas",
    "preprocessing",
    "data-preprocessing",
    "machine-learning",
    "feature-engineering",
    "missing-values",
    "outlier-detection",
    "data-profiling",
    "eda",
    "automation",
    "data-science",
    "etl",
    "tabular-data",
    "dataframe",
]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Intended Audience :: Information Technology",
    "License :: OSI Approved :: MIT License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3 :: Only",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Topic :: Scientific/Engineering :: Information Analysis",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Typing :: Typed",
]
dependencies = [
    "pandas>=1.5,<3",
    "numpy>=1.22",
]

[project.optional-dependencies]
ml = [
    "scikit-learn>=1.0",
]
# pyarrow is required by fd.clean(polars_df): the polars→pandas interchange
# (DataFrame.to_pandas) needs it, so the extra must ship it.
polars = [
    "polars>=1.0",
    "pyarrow>=10",
]
pyarrow = [
    "pyarrow>=10",
]
duckdb = [
    "duckdb>=0.10",
]
# Distributed execution backend. PySpark also needs a JVM available at runtime.
spark = [
    "pyspark>=3.4",
]
# Benchmark harness (synthetic Parquet generation + memory measurement).
bench = [
    "pyarrow>=10",
    "psutil>=5.9",
]
# Everything for the out-of-core / Arrow-native execution engine.
outofcore = [
    "polars>=1.0",
    "duckdb>=0.10",
    "pyarrow>=10",
]
cli = [
    "pyyaml>=5.1",
]
# Styled terminal output (Peel). Everything renders as plain text without it.
rich = [
    "rich>=13",
]
# Interactive output layer. The renderers produce self-contained HTML with zero
# of these installed; these libraries *upgrade* tables/charts when present and
# are never imported by `import freshdata`.
viz = [
    "itables>=1.6",
    "plotly>=5.0",
    "great-tables>=0.9",
]
# Composed notebook widgets on top of viz.
notebook = [
    "itables>=1.6",
    "plotly>=5.0",
    "great-tables>=0.9",
    "anywidget>=0.9",
    "ipython>=7.0",
]
# Domain validator packs (finance/retail/transport) read their rules from YAML.
domains = [
    "pyyaml>=5.1",
]
cleanlab = [
    "cleanlab>=2.0",
    "scikit-learn>=1.0",
]
# Optional local semantic model runtime (embedding backend). Model weights are
# NOT bundled and are never downloaded automatically: install this extra, then
# run `freshdata models pull fd-col-encoder-v1` (or drop files into
# FRESHDATA_MODEL_DIR) to enable the embedding backend. Core install stays
# model-free and never imports these packages.
semantic = [
    "onnxruntime>=1.15",
    "tokenizers>=0.13",
]
# Everything needed for the enterprise layer (clustering, masking, lineage, CLI),
# excluding the heavier ML label-noise extra (`cleanlab`).
enterprise = [
    "polars>=1.0",
    "pyarrow>=10",
    "scikit-learn>=1.0",
    "pyyaml>=5.1",
]
all = [
    "polars>=1.0",
    "pyarrow>=10",
    "scikit-learn>=1.0",
    "pyyaml>=5.1",
    "cleanlab>=2.0",
    "duckdb>=0.10",
    "presidio-analyzer>=2.2",
    "presidio-anonymizer>=2.2",
    "pyffx>=0.3.0",
    # Python 3.9 wheel caps for the privacy stack; see the `privacy` extra.
    "spacy<3.8.8; python_version<'3.10'",
    "thinc<8.3.5; python_version<'3.10'",
    "blis<1.2.1; python_version<'3.10'",
    "itables>=1.6",
    "plotly>=5.0",
    "great-tables>=0.9",
    "anywidget>=0.9",
    "ipython>=7.0",
    "openpyxl>=3.0.7",
]
# fd.clean_excel reads and writes .xlsx through pandas, which needs openpyxl.
excel = [
    "openpyxl>=3.0.7",
]
# Stronger PII detection (NER) + cryptographic format-preserving encryption.
# The base regex/context detector and surrogate FPE need none of these.
# On Python 3.9 the presidio -> spacy -> thinc -> blis chain must stop at the
# last releases that ship cp39 wheels (spacy 3.8.7, thinc 8.3.4, blis 1.2.0).
# Newer ones still declare Requires-Python >=3.9 but publish only an sdist for
# 3.9, and spacy >=3.8.13 cannot even build there (it needs thinc>=8.3.12, which
# is Python >=3.10). Keep these caps in sync with the `all` extra.
# Even at these caps, thinc 8.3.4 and blis 1.2.0 have no cp39 Linux aarch64
# wheel, so that platform compiles them from source (see docs/installation.md).
# No pin avoids it: spacy 3.8.7 requires thinc>=8.3.4,<8.4.
privacy = [
    "presidio-analyzer>=2.2",
    "presidio-anonymizer>=2.2",
    "pyffx>=0.3.0",
    "spacy<3.8.8; python_version<'3.10'",
    "thinc<8.3.5; python_version<'3.10'",
    "blis<1.2.1; python_version<'3.10'",
]
# DuckDB-backed probabilistic entity resolution at scale. The pandas fallback
# backend needs none of these.
entity-resolution = [
    "duckdb>=0.10",
]
dev = [
    "pytest>=7.0",
    "pytest-cov>=4.0",
    "jsonschema>=4.0",
    "ruff>=0.4",
    "mypy>=1.8",
    "build>=1.0",
    "twine>=5.0",
    "pre-commit>=3.0",
    "polars>=1.0",
    "pyarrow>=10",
    "pyyaml>=5.1",
    "sqlalchemy>=1.4",
    "duckdb>=0.10",
    "psutil>=5.9",
    "openpyxl>=3.0.7",
]
docs = [
    "mkdocs-material>=9.5",
    "mkdocstrings[python]>=0.25",
    "pymdown-extensions>=10.0",
]
dagster = ["dagster>=1.5.0"]
airflow = ["apache-airflow>=2.6.0"]
dbt = [
    "dbt-core>=1.5.0",
    "sqlalchemy>=1.4",
]
# Convenience superset of the orchestration extras. References them instead of
# re-pinning so the versions stay single-sourced (a bump to `dagster`/`airflow`/
# `dbt` can't silently drift from `integrations`). Same install closure.
integrations = ["freshdata-cleaner[dagster,airflow,dbt]"]
# Optional streaming source connectors for micro-batch cleaning. The base
# StreamingCleaner needs none of these; they are only required for the matching
# `clean_kafka` / `clean_arrow_flight` helpers.
kafka = [
    "kafka-python>=2.0",
]
flight = [
    "pyarrow>=10",
]
freshcore = [
    "maturin>=1.7",
]

[project.scripts]
freshdata = "freshdata.enterprise.cli:main"
dbt-gate = "freshdata.integrations.dbt.cli:main"

[project.urls]
Homepage = "https://freshcode-org.github.io/freshdata/"
Documentation = "https://freshcode-org.github.io/freshdata/"
Repository = "https://github.com/FreshCode-Org/freshdata"
Issues = "https://github.com/FreshCode-Org/freshdata/issues"
Changelog = "https://github.com/FreshCode-Org/freshdata/blob/main/CHANGELOG.md"
"Release Notes" = "https://github.com/FreshCode-Org/freshdata/releases"

[tool.hatch.build.targets.wheel]
packages = ["src/freshdata"]
# Ship the dbt Jinja macro (a non-Python data file) inside the wheel.
# The semantic calibration table is a small JSON default; model weights
# (*.onnx) are intentionally never shipped in the wheel.
artifacts = [
    "src/freshdata/integrations/dbt/macros/*.sql",
    "src/freshdata/schemas/*.json",
    "src/freshdata/semantic/data/*.json",
]

[tool.hatch.build.targets.sdist]
# Patterns are anchored with a leading "/": hatchling matches unanchored
# patterns at any depth (a bare "examples" would also ship docs/examples).
include = [
    "/src",
    "/crates/freshcore",
    "/tests",
    "/examples",
    "/README.md",
    "/CHANGELOG.md",
    "/LICENSE",
    "/SECURITY.md",
    "/RELEASE.md",
    "/CONTRIBUTING.md",
    "/CONTRIBUTING_DOMAINS.md",
    "/CODE_OF_CONDUCT.md",
]

[tool.pytest.ini_options]
testpaths = ["tests"]
# training/ is a dev-only top-level package outside src/freshdata (see
# training/README.md); tests/training/* imports it directly, which only
# resolves if the repo root is on sys.path. `python -m pytest` adds this
# automatically, but CI (and most editors) invoke the bare `pytest` console
# script, which does not — hence pinning it here instead.
pythonpath = ["."]
addopts = "-q --strict-markers --cov=freshdata --cov-report=term-missing --cov-fail-under=93"
xfail_strict = true
markers = [
    "large: optional large-file benchmarks (set FRESHDATA_AQI_PATH)",
    "large_smoke: bounded ~1M-row deterministic smoke that runs in the PR lane",
    "online: live download from manifest URLs (requires network)",
    "tier1: anchor online datasets with full golden expectations",
]
filterwarnings = [
    # Treat freshdata-emitted warnings as test failures (plan: zero-warning bar).
    "error::DeprecationWarning:freshdata",
    "error::UserWarning:freshdata",
    # sklearn KNNImputer can warn on collinear numeric partners; freshdata
    # suppresses these during imputation, but keep pytest output clean too.
    "ignore:divide by zero encountered in matmul:RuntimeWarning:sklearn.utils.extmath",
    "ignore:overflow encountered in matmul:RuntimeWarning:sklearn.utils.extmath",
    "ignore:invalid value encountered in matmul:RuntimeWarning:sklearn.utils.extmath",
]

[tool.coverage.run]
source = ["freshdata"]
branch = false
omit = [
    # The Spark backend needs a running JVM/SparkSession that CI may not provide;
    # it is exercised by tests/test_execution/test_spark_engine.py only when a
    # SparkSession can start. Excluded from the coverage gate so an absent JVM
    # does not fail the build (the other backends carry the shared-contract tests).
    "src/freshdata/execution/backends/_spark.py",
]

[tool.coverage.report]
fail_under = 93
show_missing = true
skip_empty = true

[tool.ruff]
line-length = 99
target-version = "py39"
src = ["src", "tests"]
# ASV-managed benchmark sub-project: tool-generated layout and per-machine
# result artifacts; not shipped and not held to the package lint bar.
extend-exclude = ["freshdata-benchmarks"]

[tool.ruff.lint]
select = ["E", "F", "W", "I", "UP", "B", "SIM", "C4", "RET", "PL"]
ignore = [
    "PLR0911", # too many returns — dtype heuristics are naturally branchy
    "PLR0912", # too many branches — cleaning steps are naturally branchy
    "PLR0913", # many keyword options is the point of the config surface
    "PLR0917", # positional-arg count — internal helpers, sibling of PLR0913
    "PLR2004", # magic-value comparisons in heuristics are documented inline
]

[tool.ruff.lint.per-file-ignores]
"tests/*" = ["PLR2004", "SIM117"]
"src/freshdata/engine/model_select.py" = ["PLR0915"]
"src/freshdata/engine/outliers.py" = ["PLR0915"]
"src/freshdata/imputation/*" = ["PLC0415"]
"src/freshdata/adapters/polars.py" = ["PLC0415", "PLW0603"]
"scripts/debug_datasets.py" = ["PLR0915"]
# Enterprise layer: lazy optional imports (PLC0415) keep `import freshdata` cheap;
# the orchestration functions are intentionally long (PLR0915).
"src/freshdata/enterprise/cleaner.py" = ["PLC0415", "PLR0915", "PLW2901"]
"src/freshdata/enterprise/interface.py" = ["PLC0415"]
# privacy: Presidio/collections imported lazily (PLC0415), analyzer cached
# (PLW0603); entity_resolution: duckdb/collections lazy (PLC0415). Both have
# intentionally long orchestration functions (PLR0915).
"src/freshdata/enterprise/privacy.py" = ["PLC0415", "PLW0603", "PLR0915"]
# privacy_policy: PyYAML imported lazily (PLC0415); apply_privacy_policy is a
# single intentionally long action-dispatch orchestrator (PLR0915).
"src/freshdata/enterprise/privacy_policy.py" = ["PLC0415", "PLR0915"]
"src/freshdata/enterprise/entity_resolution.py" = ["PLC0415", "PLR0915"]
"src/freshdata/enterprise/cli.py" = ["PLC0415", "PLR0915"]
# api: fd.clean is the single top-level orchestration entry point (context,
# memory, learned profiles, engines, domains) and is intentionally long.
"src/freshdata/api.py" = ["PLR0915"]
"src/freshdata/__init__.py" = ["PLC0415"]
# Domain packs defer the PyYAML import so the base infra stays dependency-free.
"src/freshdata/domains/base.py" = ["PLC0415"]
# Integrations: orchestration frameworks are optional, so each adapter imports its
# framework lazily (PLC0415) and caches the lazily-built class (PLW0603).
"src/freshdata/integrations/_core.py" = ["PLC0415"]
"src/freshdata/integrations/dagster/__init__.py" = ["PLC0415", "PLW0603"]
"src/freshdata/integrations/airflow/__init__.py" = ["PLC0415", "PLW0603"]
"src/freshdata/integrations/dbt/__init__.py" = ["PLC0415"]
# Integration tests import adapters in-function (after fixtures patch sys.modules).
"tests/test_integrations/*" = ["PLC0415"]
# Out-of-core engine: every backend imports its optional dependency (polars,
# duckdb, pyarrow) lazily (PLC0415) so `import freshdata` stays light, and the
# backend modules are imported lazily by the selector to avoid hard deps.
"src/freshdata/execution/*" = ["PLC0415"]
"src/freshdata/benchmarks/*" = ["PLC0415"]
# Local model runtime: onnxruntime/tokenizers ([semantic] extra) import lazily
# (PLC0415) so `import freshdata` stays model-free; the process-wide encoder
# factory hook is module state (PLW0603).
"src/freshdata/models/*" = ["PLC0415", "PLW0603"]
"tests/test_execution/*" = ["PLC0415"]
# Streaming: optional connectors (kafka, pyarrow.flight) and the CSV/Parquet
# readers + memory probes import lazily (PLC0415) so the base streaming path and
# `import freshdata` stay dependency-free.
"src/freshdata/streaming/*" = ["PLC0415"]
"benchmarks/*" = ["PLC0415"]
"benchmarks/bench_freshcore.py" = ["E402", "I001"]
# Fixture modules encode defect tables as one row per line; wrapping those
# rows destroys the tabular readability the fixtures exist for. The gold
# builder is one linear data-construction pass; splitting it adds nothing.
"benchmarks/fixtures/*" = ["E501"]
"benchmarks/fixtures/gold.py" = ["E501", "PLR0915"]
"benchmarks/harness_metrics.py" = ["PLR0915"]
"tests/test_import_time.py" = ["PLC0415"]
# Render layer + ReportFrame: optional viz deps and the renderer registry are
# imported lazily (PLC0415) so `import freshdata` never pulls plotly/itables.
"src/freshdata/render/*" = ["PLC0415"]
"src/freshdata/_reportframe.py" = ["PLC0415"]
"tests/test_render_html.py" = ["PLC0415"]
# Phase-3 features lazily import the enterprise/cleaner layers (PLC0415) so the
# light core and `import freshdata` stay dependency-free.
"src/freshdata/memory.py" = ["PLC0415"]
"src/freshdata/quality.py" = ["PLC0415"]
"src/freshdata/enterprise/join_assist.py" = ["PLC0415"]
"tests/test_streaming_cleaner.py" = ["PLC0415"]
"tests/test_streaming_state.py" = ["PLC0415"]
"tests/test_streaming_reports.py" = ["PLC0415"]
"tests/test_streaming_cli.py" = ["PLC0415"]
# Phase 5: the training/ package is a dev-only tool outside the runtime
# import path (see training/README.md); its trainers/teacher/CLI modules
# lazily import numpy/onnx/freshdata internals (PLC0415) to keep each
# `python -m training.*` entry point cheap, and a couple of long dataclass-
# heavy functions exceed the statement-count heuristic (PLR0915).
"training/*" = ["PLC0415", "PLR0915"]
"tests/training/*" = ["PLC0415"]
"tests/test_cleanbench_full.py" = ["PLC0415"]
"tests/test_cleanbench_public_release.py" = ["PLC0415"]
"tests/test_phase5_backcompat.py" = ["PLC0415"]
"tests/test_plugins.py" = ["PLC0415"]
"benchmarks/baselines/great_expectations_baseline.py" = ["PLC0415"]
"benchmarks/baselines/llm_agent_baseline.py" = ["PLC0415"]
"benchmarks/cleanbench/*" = ["PLC0415"]

[tool.mypy]
python_version = "3.10"
strict = false
warn_unused_ignores = true
warn_redundant_casts = true
no_implicit_optional = true
follow_imports = "silent"
files = ["src/freshdata"]

[[tool.mypy.overrides]]
module = ["pandas.*", "numpy.*", "sklearn.*", "polars.*", "requests.*", "cleanlab.*", "yaml.*", "dagster.*", "airflow.*", "sqlalchemy.*", "duckdb.*", "pyarrow.*", "psutil.*", "kafka.*", "presidio_analyzer.*", "presidio_anonymizer.*", "pyffx.*", "pyspark", "pyspark.*", "onnxruntime", "onnxruntime.*", "tokenizers", "tokenizers.*"]
ignore_missing_imports = true