From 45c11737f8f63d3a39918f3f08456473d637a0ed Mon Sep 17 00:00:00 2001 From: Aman-Cool Date: Thu, 26 Feb 2026 03:44:57 +0530 Subject: [PATCH 1/5] fix: replace HDF5 fill values with NaN before DataFrame construction --- aidrin/file_handling/readers/hdf5_reader.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aidrin/file_handling/readers/hdf5_reader.py b/aidrin/file_handling/readers/hdf5_reader.py index 2d56a6e2..97731e91 100644 --- a/aidrin/file_handling/readers/hdf5_reader.py +++ b/aidrin/file_handling/readers/hdf5_reader.py @@ -2,6 +2,7 @@ import uuid import h5py +import numpy as np import pandas as pd from flask import current_app, session @@ -46,6 +47,20 @@ def recurse(name, obj, path=[]): try: if isinstance(obj, h5py.Dataset): data = obj[()] + # Replace the dataset's fill value with NaN so that + # pd.isnull()-based metrics (completeness, outliers, etc.) + # correctly detect missing data. CSV ingestion performs this + # translation automatically via pd.read_csv's na_values list; + # without it, fill-value-encoded missingness (e.g. -9999.0) + # is invisible to every downstream metric and completeness is + # silently reported as 100% for incomplete scientific datasets. + if hasattr(data, "dtype") and data.dtype.kind in ("f", "i", "u"): + fill_val = obj.fillvalue + if fill_val is not None: + mask = data == fill_val + if mask.any(): + data = data.astype(np.float64) + data[mask] = np.nan # If it's a 1D or structured dataset, load it into dicts if isinstance(data, (list, tuple)) or hasattr(data, "dtype"): try: From b7488214d154c4f6c699857800d2a6f3b4b16bba Mon Sep 17 00:00:00 2001 From: Aman-Cool Date: Thu, 26 Feb 2026 08:10:22 +0530 Subject: [PATCH 2/5] feat: multi-source fill value detection for HDF5 datasets --- aidrin/file_handling/readers/hdf5_reader.py | 63 +++++++++++++++++---- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/aidrin/file_handling/readers/hdf5_reader.py b/aidrin/file_handling/readers/hdf5_reader.py index 97731e91..133b3485 100644 --- a/aidrin/file_handling/readers/hdf5_reader.py +++ b/aidrin/file_handling/readers/hdf5_reader.py @@ -10,6 +10,48 @@ class hdf5Reader(BaseFileReader): + def __init__(self, file_path: str, logger, fill_values=None): + super().__init__(file_path, logger) + # Optional user-supplied fill values merged with auto-detected ones. + # Accepts any iterable of scalars, e.g. fill_values=[-9999, -1]. + self.fill_values = set(fill_values) if fill_values is not None else set() + + def _collect_fill_values(self, dataset): + """Return a set of numeric sentinels that represent missing data for + this dataset, drawn from four sources in priority order: + + 1. User-supplied values passed at construction time. + 2. ``_FillValue`` attribute — NetCDF/CF convention, present in + virtually all climate, oceanography, and atmospheric HDF5 files. + 3. ``missing_value`` attribute — older NetCDF convention; may be a + scalar or a 1-D array listing multiple sentinels. + 4. ``dataset.fillvalue`` — the HDF5-native fill value written into + the file at dataset-creation time (always present; defaults to + 0 / 0.0 when the producer did not set one explicitly). + + Only numeric values are collected; non-numeric sentinels (e.g. byte + strings in variable-length string datasets) are silently skipped + because the dtype guard in read() filters those datasets out before + this method is called. + """ + collected = set(self.fill_values) + + for attr_name in ("_FillValue", "missing_value"): + if attr_name in dataset.attrs: + raw = dataset.attrs[attr_name] + for v in np.atleast_1d(raw).ravel(): + try: + collected.add(float(v)) + except (TypeError, ValueError): + pass + + try: + collected.add(float(dataset.fillvalue)) + except (TypeError, ValueError): + pass + + return collected + def read(self): try: rows = [] @@ -47,17 +89,18 @@ def recurse(name, obj, path=[]): try: if isinstance(obj, h5py.Dataset): data = obj[()] - # Replace the dataset's fill value with NaN so that - # pd.isnull()-based metrics (completeness, outliers, etc.) - # correctly detect missing data. CSV ingestion performs this - # translation automatically via pd.read_csv's na_values list; - # without it, fill-value-encoded missingness (e.g. -9999.0) - # is invisible to every downstream metric and completeness is - # silently reported as 100% for incomplete scientific datasets. + # Translate all fill-value sentinels to NaN so that + # pd.isnull()-based metrics (completeness, outliers, …) + # correctly detect missing data. Sentinels are collected + # from the HDF5 native fillvalue, the _FillValue and + # missing_value dataset attributes (NetCDF/CF convention), + # and any values supplied by the caller at construction time. if hasattr(data, "dtype") and data.dtype.kind in ("f", "i", "u"): - fill_val = obj.fillvalue - if fill_val is not None: - mask = data == fill_val + fill_vals = self._collect_fill_values(obj) + if fill_vals: + mask = np.zeros(data.shape, dtype=bool) + for fv in fill_vals: + mask |= data == fv if mask.any(): data = data.astype(np.float64) data[mask] = np.nan From d080e7e1b0b69d24b00921b5261acf30f4b1f5b9 Mon Sep 17 00:00:00 2001 From: Aman-Cool Date: Thu, 26 Feb 2026 08:14:15 +0530 Subject: [PATCH 3/5] feat: distinguish explicit vs uncertain fill values with targeted warnings --- aidrin/file_handling/readers/hdf5_reader.py | 105 ++++++++++++++------ 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/aidrin/file_handling/readers/hdf5_reader.py b/aidrin/file_handling/readers/hdf5_reader.py index 133b3485..e10cbb88 100644 --- a/aidrin/file_handling/readers/hdf5_reader.py +++ b/aidrin/file_handling/readers/hdf5_reader.py @@ -17,40 +17,55 @@ def __init__(self, file_path: str, logger, fill_values=None): self.fill_values = set(fill_values) if fill_values is not None else set() def _collect_fill_values(self, dataset): - """Return a set of numeric sentinels that represent missing data for - this dataset, drawn from four sources in priority order: - - 1. User-supplied values passed at construction time. - 2. ``_FillValue`` attribute — NetCDF/CF convention, present in - virtually all climate, oceanography, and atmospheric HDF5 files. - 3. ``missing_value`` attribute — older NetCDF convention; may be a - scalar or a 1-D array listing multiple sentinels. - 4. ``dataset.fillvalue`` — the HDF5-native fill value written into - the file at dataset-creation time (always present; defaults to - 0 / 0.0 when the producer did not set one explicitly). - - Only numeric values are collected; non-numeric sentinels (e.g. byte - strings in variable-length string datasets) are silently skipped - because the dtype guard in read() filters those datasets out before - this method is called. + """Return (explicit, uncertain) sets of numeric missing-data sentinels. + + explicit — safe to replace silently: + • User-supplied values passed at construction time. + • ``_FillValue`` attribute (NetCDF/CF convention). + • ``missing_value`` attribute (older NetCDF convention; may be a + scalar or a 1-D array of multiple sentinels). + • The HDF5 native ``dataset.fillvalue`` when it is non-zero *or* + when fill-value attributes are present (the producer clearly + cared about missingness, so the native value is intentional). + + uncertain — producer intent is ambiguous, warn before replacing: + • The HDF5 native ``dataset.fillvalue`` when it equals the dtype + default (0 / 0.0) *and* no fill-value attributes are present. + HDF5 always stores a fill value; without an explicit assignment + it defaults to zero, which is a valid measurement in counts, + indices, and many physical quantities. + + Only numeric values are collected; non-numeric sentinels are skipped + because the dtype guard in read() excludes string/compound datasets + before this method is called. """ - collected = set(self.fill_values) + explicit = set(self.fill_values) + has_fill_attrs = False for attr_name in ("_FillValue", "missing_value"): if attr_name in dataset.attrs: + has_fill_attrs = True raw = dataset.attrs[attr_name] for v in np.atleast_1d(raw).ravel(): try: - collected.add(float(v)) + explicit.add(float(v)) except (TypeError, ValueError): pass + uncertain = set() try: - collected.add(float(dataset.fillvalue)) + native = float(dataset.fillvalue) + if native not in explicit: + dtype_default = float(np.zeros(1, dtype=dataset.dtype)[0]) + if not has_fill_attrs and native == dtype_default: + # Zero is the HDF5 default, not a deliberate sentinel. + uncertain.add(native) + else: + explicit.add(native) except (TypeError, ValueError): pass - return collected + return explicit, uncertain def read(self): try: @@ -89,19 +104,47 @@ def recurse(name, obj, path=[]): try: if isinstance(obj, h5py.Dataset): data = obj[()] - # Translate all fill-value sentinels to NaN so that + # Translate fill-value sentinels to NaN so that # pd.isnull()-based metrics (completeness, outliers, …) - # correctly detect missing data. Sentinels are collected - # from the HDF5 native fillvalue, the _FillValue and - # missing_value dataset attributes (NetCDF/CF convention), - # and any values supplied by the caller at construction time. + # correctly detect missing data. if hasattr(data, "dtype") and data.dtype.kind in ("f", "i", "u"): - fill_vals = self._collect_fill_values(obj) - if fill_vals: - mask = np.zeros(data.shape, dtype=bool) - for fv in fill_vals: - mask |= data == fv - if mask.any(): + explicit_fills, uncertain_fills = self._collect_fill_values(obj) + all_fills = explicit_fills | uncertain_fills + if all_fills: + # Only iterate sentinels that actually appear in + # the data, so we can report accurately and avoid + # unnecessary dtype promotion. + matched = {fv for fv in all_fills if (data == fv).any()} + if matched: + mask = np.zeros(data.shape, dtype=bool) + for fv in matched: + mask |= data == fv + n_replaced = int(mask.sum()) + + uncertain_matched = matched & uncertain_fills + if uncertain_matched: + # The only sentinel that matched is the + # HDF5 default zero — it may be valid data. + self.logger.warning( + f"Dataset '{name}': {n_replaced}/" + f"{data.size} value(s) match the HDF5 " + f"default fill value {uncertain_matched} " + f"and will be replaced with NaN. " + f"If zero is a valid measurement here " + f"(e.g. counts, indices), set a " + f"'_FillValue' attribute in the file to " + f"an unambiguous sentinel, or pass " + f"fill_values=[] at construction time to " + f"suppress native fill value replacement." + ) + else: + self.logger.info( + f"Dataset '{name}': replaced " + f"{n_replaced}/{data.size} value(s) " + f"matching explicit fill sentinel(s) " + f"{matched} with NaN." + ) + data = data.astype(np.float64) data[mask] = np.nan # If it's a 1D or structured dataset, load it into dicts From ae779064b7645ff1c1c8b2e51dbba2f28a70c206 Mon Sep 17 00:00:00 2001 From: Aman-Cool Date: Thu, 26 Feb 2026 08:30:31 +0530 Subject: [PATCH 4/5] test: add HDF5 fill value normalization test suite --- aidrin/file_handling/readers/hdf5_reader.py | 11 +- test_hdf5_reader.py | 260 ++++++++++++++++++++ 2 files changed, 269 insertions(+), 2 deletions(-) create mode 100644 test_hdf5_reader.py diff --git a/aidrin/file_handling/readers/hdf5_reader.py b/aidrin/file_handling/readers/hdf5_reader.py index e10cbb88..e717df8e 100644 --- a/aidrin/file_handling/readers/hdf5_reader.py +++ b/aidrin/file_handling/readers/hdf5_reader.py @@ -57,10 +57,17 @@ def _collect_fill_values(self, dataset): native = float(dataset.fillvalue) if native not in explicit: dtype_default = float(np.zeros(1, dtype=dataset.dtype)[0]) - if not has_fill_attrs and native == dtype_default: - # Zero is the HDF5 default, not a deliberate sentinel. + if native == dtype_default: + # Native fill equals the dtype default (0 / 0.0). HDF5 + # always stores a fill value; without an explicit producer + # assignment it lands here. Even when fill-value attributes + # are present (e.g. _FillValue=-9999), the producer's chosen + # sentinel is already in `explicit` — the default zero is + # still ambiguous and must not be silently replaced. uncertain.add(native) else: + # Non-default native fill: the producer explicitly chose + # this value, so it is intentional. explicit.add(native) except (TypeError, ValueError): pass diff --git a/test_hdf5_reader.py b/test_hdf5_reader.py new file mode 100644 index 00000000..b9ad1e23 --- /dev/null +++ b/test_hdf5_reader.py @@ -0,0 +1,260 @@ +""" +Tests for HDF5 fill-value normalization in hdf5Reader. + +Verifies that every source of fill-value information (HDF5 native fillvalue, +_FillValue attribute, missing_value attribute, and user-supplied fill_values) +is correctly translated to NaN so that pd.isnull()-based metrics report +accurate completeness scores rather than the 100% that was returned before +this fix when data contained fill-value-encoded missing entries. +""" + +import logging +import math +import sys +import types + +# --------------------------------------------------------------------------- +# Compatibility shim: pkg_resources was removed from the stdlib in Python 3.12+ +# and is only available when setuptools is installed. dython imports it at +# module level, which prevents the whole aidrin package from loading on clean +# Python 3.13 environments. Inject a minimal stub before any aidrin import so +# that the test suite works without requiring a full project venv. +# --------------------------------------------------------------------------- +if "pkg_resources" not in sys.modules: + _pkg_resources = types.ModuleType("pkg_resources") + + class _Dist: + def __init__(self): + self.version = "0.0.0" + + _pkg_resources.get_distribution = lambda _name: _Dist() + sys.modules["pkg_resources"] = _pkg_resources + +import h5py +import numpy as np +import pytest + +from aidrin.file_handling.readers.hdf5_reader import hdf5Reader + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +@pytest.fixture +def logger(): + return logging.getLogger("test_hdf5_reader") + + +def _make_hdf5(path, data, fillvalue=None, attrs=None): + """Write a minimal single-dataset HDF5 file for testing.""" + with h5py.File(path, "w") as f: + kwargs = {} if fillvalue is None else {"fillvalue": fillvalue} + ds = f.create_dataset("measurements", data=data, **kwargs) + if attrs: + for k, v in attrs.items(): + ds.attrs[k] = v + + +def _read_col(tmp_path, logger, data, fillvalue=None, attrs=None, fill_values=None): + """Write a file, read it, and return the first DataFrame column as a Series.""" + fpath = str(tmp_path / "test.h5") + _make_hdf5(fpath, data, fillvalue=fillvalue, attrs=attrs) + kwargs = {} if fill_values is None else {"fill_values": fill_values} + df = hdf5Reader(fpath, logger, **kwargs).read() + assert df is not None, "hdf5Reader.read() returned None" + return df.iloc[:, 0] + + +# --------------------------------------------------------------------------- +# Explicit fill-value sources (replaced silently, no WARNING) +# --------------------------------------------------------------------------- + +class TestExplicitFillValues: + + def test_netcdf_fillvalue_attr_replaced(self, tmp_path, logger, caplog): + """_FillValue attribute sentinel is replaced with NaN without a WARNING.""" + data = np.array([1.0, -9999.0, 3.0, -9999.0, 5.0], dtype=np.float64) + + with caplog.at_level(logging.WARNING): + col = _read_col(tmp_path, logger, data, attrs={"_FillValue": -9999.0}) + + assert col.isna().sum() == 2 + assert list(col.dropna()) == [1.0, 3.0, 5.0] + assert not any("default fill value" in r.message for r in caplog.records) + + def test_missing_value_attr_replaced(self, tmp_path, logger, caplog): + """missing_value attribute sentinel (NetCDF legacy) is replaced with NaN.""" + data = np.array([10, -1, 20, -1, 30], dtype=np.int32) + + with caplog.at_level(logging.WARNING): + col = _read_col(tmp_path, logger, data, attrs={"missing_value": np.int32(-1)}) + + assert col.isna().sum() == 2 + assert not any("default fill value" in r.message for r in caplog.records) + + def test_missing_value_array_attr_all_sentinels_replaced(self, tmp_path, logger): + """missing_value may be a 1-D array listing multiple sentinels — all replaced.""" + data = np.array([1.0, -9999.0, 3.0, -1.0, 5.0], dtype=np.float64) + col = _read_col(tmp_path, logger, data, + attrs={"missing_value": np.array([-9999.0, -1.0])}) + + assert col.isna().sum() == 2 + assert list(col.dropna()) == [1.0, 3.0, 5.0] + + def test_nonzero_native_fillvalue_replaced_silently(self, tmp_path, logger, caplog): + """A non-zero HDF5 native fillvalue (no attrs) is explicit — replaced without WARNING.""" + data = np.array([1.0, -9999.0, 3.0], dtype=np.float64) + + with caplog.at_level(logging.WARNING): + col = _read_col(tmp_path, logger, data, fillvalue=-9999.0) + + assert col.isna().sum() == 1 + assert list(col.dropna()) == [1.0, 3.0] + assert not any("default fill value" in r.message for r in caplog.records) + + def test_user_supplied_fill_values_replace_sentinel(self, tmp_path, logger, caplog): + """fill_values constructor parameter marks an arbitrary value as explicit.""" + data = np.array([1.0, 42.0, 3.0, 42.0, 5.0], dtype=np.float64) + + with caplog.at_level(logging.WARNING): + col = _read_col(tmp_path, logger, data, fill_values=[42.0]) + + assert col.isna().sum() == 2 + assert list(col.dropna()) == [1.0, 3.0, 5.0] + assert not any("default fill value" in r.message for r in caplog.records) + + def test_user_supplied_overrides_uncertain_classification(self, tmp_path, logger, caplog): + """Passing fill_values=[0] moves zero from uncertain to explicit — no WARNING.""" + data = np.array([0, 1, 2, 0, 4], dtype=np.int32) + + with caplog.at_level(logging.WARNING): + col = _read_col(tmp_path, logger, data, fill_values=[0]) + + assert col.isna().sum() == 2 + assert not any("default fill value" in r.message for r in caplog.records) + + +# --------------------------------------------------------------------------- +# Uncertain fill-value source: HDF5 default zero (WARNING expected) +# --------------------------------------------------------------------------- + +class TestUncertainFillValues: + + def test_zero_default_fillvalue_emits_warning(self, tmp_path, logger, caplog): + """HDF5 default zero fillvalue (no attrs) triggers a WARNING before replacement.""" + data = np.array([0, 1, 2, 0, 4], dtype=np.int32) + # No explicit fillvalue → h5py defaults to 0 for int32 + + with caplog.at_level(logging.WARNING): + _read_col(tmp_path, logger, data) + + assert any("default fill value" in r.message for r in caplog.records) + + def test_zero_default_fillvalue_still_replaced(self, tmp_path, logger): + """Zeros are replaced with NaN even when a WARNING is emitted.""" + data = np.array([0, 1, 2, 0, 4], dtype=np.int32) + col = _read_col(tmp_path, logger, data) + + assert col.isna().sum() == 2 + + def test_warning_includes_replacement_count(self, tmp_path, logger, caplog): + """The WARNING message reports how many values will be replaced.""" + data = np.array([0.0, 1.0, 0.0], dtype=np.float64) + + with caplog.at_level(logging.WARNING): + _read_col(tmp_path, logger, data) + + warning_msgs = [r.message for r in caplog.records if "default fill value" in r.message] + assert len(warning_msgs) == 1 + # Message should contain the count "2" and total "3" + assert "2" in warning_msgs[0] + assert "3" in warning_msgs[0] + + def test_zero_fillvalue_with_fill_attr_warns_zero_keeps_valid(self, tmp_path, logger, caplog): + """ + When _FillValue attr is -9999.0 and HDF5 native fillvalue defaults to 0.0, + zeros are still uncertain and emit a WARNING — they are NOT silently collapsed + into explicit alongside the -9999.0 sentinel. This guards against the bug + where has_fill_attrs=True caused default zeros to be added to explicit, + silently replacing legitimate zero measurements. + """ + data = np.array([0.0, 1.0, -9999.0, 3.0], dtype=np.float64) + + with caplog.at_level(logging.WARNING): + col = _read_col(tmp_path, logger, data, attrs={"_FillValue": -9999.0}) + + # The -9999.0 sentinel is replaced (explicit) + assert math.isnan(float(col[col.index[2]])) + # 0.0 is replaced too (uncertain), but a WARNING was emitted + assert any("default fill value" in r.message for r in caplog.records) + # Valid measurements 1.0 and 3.0 survive + non_nan = col.dropna().tolist() + assert 1.0 in non_nan + assert 3.0 in non_nan + + +# --------------------------------------------------------------------------- +# No-op cases: nothing matched, nothing logged +# --------------------------------------------------------------------------- + +class TestNoReplacementNeeded: + + def test_no_matching_fill_values_no_nan(self, tmp_path, logger): + """When no data values match any sentinel, the column is unchanged.""" + data = np.array([1.0, 2.0, 3.0], dtype=np.float64) + col = _read_col(tmp_path, logger, data, fillvalue=-9999.0) + + assert col.isna().sum() == 0 + + def test_no_matching_fill_values_no_log_noise(self, tmp_path, logger, caplog): + """No 'replaced' or 'default fill value' messages emitted when nothing matched.""" + data = np.array([1.0, 2.0, 3.0], dtype=np.float64) + + with caplog.at_level(logging.INFO): + _read_col(tmp_path, logger, data, fillvalue=-9999.0) + + assert not any( + "replaced" in r.message or "default fill value" in r.message + for r in caplog.records + ) + + +# --------------------------------------------------------------------------- +# End-to-end: completeness metric reflects true missingness +# --------------------------------------------------------------------------- + +class TestCompletenessAccuracy: + + def test_completeness_is_correct_not_100_percent(self, tmp_path, logger): + """ + Before this fix, hdf5Reader returned raw fill values and pd.isnull() + saw no NaN, so completeness was always reported as 1.0 (100%) even for + datasets with extensive missingness. After the fix, completeness + reflects the true fraction of present values. + """ + # 3 valid values, 2 fill-value-encoded missing → true completeness = 0.6 + data = np.array([1.0, -9999.0, 3.0, -9999.0, 5.0], dtype=np.float64) + col = _read_col(tmp_path, logger, data, attrs={"_FillValue": -9999.0}) + + completeness = 1 - col.isnull().mean() + assert abs(completeness - 0.6) < 1e-9, ( + f"Expected completeness 0.6, got {completeness}. " + "Fill values were not translated to NaN." + ) + + def test_fully_present_dataset_still_reports_100_percent(self, tmp_path, logger): + """A genuinely complete dataset still scores 1.0 after the fix.""" + data = np.array([1.0, 2.0, 3.0, 4.0, 5.0], dtype=np.float64) + col = _read_col(tmp_path, logger, data, fillvalue=-9999.0) + + completeness = 1 - col.isnull().mean() + assert completeness == 1.0 + + def test_integer_dataset_completeness(self, tmp_path, logger): + """Fill-value NaN replacement works for integer dtypes (promoted to float64).""" + data = np.array([10, 32767, 20, 32767, 30], dtype=np.int16) + col = _read_col(tmp_path, logger, data, attrs={"_FillValue": np.int16(32767)}) + + completeness = 1 - col.isnull().mean() + assert abs(completeness - 0.6) < 1e-9 From a606344199920e5ef3f713562e239663ff273818 Mon Sep 17 00:00:00 2001 From: Aman-Cool Date: Thu, 26 Feb 2026 08:32:57 +0530 Subject: [PATCH 5/5] docs: document HDF5 fill value normalisation in completeness and limitations --- docs/source/limitations.rst | 6 +++++- docs/source/usage.rst | 31 ++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/source/limitations.rst b/docs/source/limitations.rst index 28cfcf70..06626724 100644 --- a/docs/source/limitations.rst +++ b/docs/source/limitations.rst @@ -21,7 +21,11 @@ What We Can Do - Provide **quantitative metrics** for dataset readiness and visualizations of results. - Analyze **DCAT and DataCite JSON metadata** for FAIR compliance. - Identify **missing or incomplete metadata elements**. -- Work with **structured tabular datasets** (e.g., CSV, Excel) for basic data readiness checks. +- Work with **structured tabular datasets** (CSV, Excel, JSON, NumPy ``.npz``, and + HDF5 ``.h5``) for data readiness checks. For HDF5 files, format-native missing + data sentinels (``_FillValue``, ``missing_value``, and the dataset's HDF5 fill + value) are automatically normalised to ``NaN`` before any metric is computed, + ensuring accurate completeness, outlier, and privacy scores. What We Cannot Do ~~~~~~~~~~~~~~~~~ diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 3ad6ab36..4f5148d3 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -65,6 +65,29 @@ Evaluates dataset completeness by checking for missing values. **Returns**: A dictionary with an overall completeness score (1 for no missing values, 0 for all missing) and a histogram of missing value proportions per column. +.. note:: + + **HDF5 fill value handling**: HDF5 datasets encode missing data as a numeric + sentinel (the *fill value*) rather than as a blank cell. When reading an + ``.h5`` file AIDRIN automatically translates these sentinels to ``NaN`` + before computing completeness, so the score reflects true data availability + rather than always reporting 100%. + + Sentinels are collected from the following sources, in priority order: + + 1. **User-supplied** – pass ``fill_values=[v1, v2, …]`` to ``hdf5Reader`` + at construction time to declare domain-specific sentinels explicitly. + 2. **_FillValue attribute** – the NetCDF/CF convention used by virtually + all climate, oceanography, and atmospheric HDF5 files. + 3. **missing_value attribute** – the older NetCDF convention; may be a + scalar or an array of multiple sentinels. + 4. **HDF5 native fill value** – the value stored in the dataset's own + metadata (``dataset.fillvalue``). When this equals the dtype default + (``0`` / ``0.0``) and no fill-value attributes are present, a warning + is logged before replacement because zero is a legitimate measurement in + many scientific datasets (e.g. counts, indices). Set a ``_FillValue`` + attribute in the file to an unambiguous sentinel to suppress this warning. + calculate_correlations ^^^^^^^^^^^^^^^^^^^^^^ @@ -421,6 +444,12 @@ Notes - The local installation requires setting up Redis, Celery, and Flask (see `Installation <./installation.html>`_). The web application at `aidrin.io `_ handles these server-side, offering a no-setup alternative. - Both use the same codebase, ensuring identical functionality. The web application is ideal for users who prefer a browser-based interface. -- **File Formats**: The web application supports CSV files for data uploads and DCAT/DataCite JSON for metadata in the Understandability and Usability dimension. +- **File Formats**: The web application supports CSV, Excel, JSON, NumPy (``.npz``), + and HDF5 (``.h5``) files for data uploads, and DCAT/DataCite JSON for metadata + in the Understandability and Usability dimension. For HDF5 files, fill-value + sentinels (``_FillValue``, ``missing_value``, and the HDF5 native fill value) are + automatically converted to ``NaN`` so that all metrics — completeness, outliers, + feature relevance, and privacy — operate on accurately marked missing data. See + the ``calculate_completeness`` note above for the full sentinel-resolution order. - **Visualizations**: Generated downloadable plots (e.g., histograms, bar charts, heatmaps) are displayed in the web interface. - **JSON Reports**: Each dimension’s analysis generates a downloadable JSON report containing all metrics, statistics, and visualization data (where applicable).