fix: replace HDF5 fill values with NaN before DataFrame construction

aidrin/file_handling/readers/hdf5_reader.py

@@ -2,13 +2,78 @@
 import uuid
 
 import h5py
+import numpy as np
 import pandas as pd
 from flask import current_app, session
 
 from aidrin.file_handling.readers.base_reader import BaseFileReader
 
 
 class hdf5Reader(BaseFileReader):
+    def __init__(self, file_path: str, logger, fill_values=None):
+        super().__init__(file_path, logger)
+        # Optional user-supplied fill values merged with auto-detected ones.
+        # Accepts any iterable of scalars, e.g. fill_values=[-9999, -1].
+        self.fill_values = set(fill_values) if fill_values is not None else set()
+
+    def _collect_fill_values(self, dataset):
+        """Return (explicit, uncertain) sets of numeric missing-data sentinels.
+
+        explicit — safe to replace silently:
+            • User-supplied values passed at construction time.
+            • ``_FillValue`` attribute (NetCDF/CF convention).
+            • ``missing_value`` attribute (older NetCDF convention; may be a
+              scalar or a 1-D array of multiple sentinels).
+            • The HDF5 native ``dataset.fillvalue`` when it is non-zero *or*
+              when fill-value attributes are present (the producer clearly
+              cared about missingness, so the native value is intentional).
+
+        uncertain — producer intent is ambiguous, warn before replacing:
+            • The HDF5 native ``dataset.fillvalue`` when it equals the dtype
+              default (0 / 0.0) *and* no fill-value attributes are present.
+              HDF5 always stores a fill value; without an explicit assignment
+              it defaults to zero, which is a valid measurement in counts,
+              indices, and many physical quantities.
+
+        Only numeric values are collected; non-numeric sentinels are skipped
+        because the dtype guard in read() excludes string/compound datasets
+        before this method is called.
+        """
+        explicit = set(self.fill_values)
+        has_fill_attrs = False
+
+        for attr_name in ("_FillValue", "missing_value"):
+            if attr_name in dataset.attrs:
+                has_fill_attrs = True
+                raw = dataset.attrs[attr_name]
+                for v in np.atleast_1d(raw).ravel():
+                    try:
+                        explicit.add(float(v))
+                    except (TypeError, ValueError):
+                        pass
+
+        uncertain = set()
+        try:
+            native = float(dataset.fillvalue)
+            if native not in explicit:
+                dtype_default = float(np.zeros(1, dtype=dataset.dtype)[0])
+                if native == dtype_default:
+                    # Native fill equals the dtype default (0 / 0.0).  HDF5
+                    # always stores a fill value; without an explicit producer
+                    # assignment it lands here.  Even when fill-value attributes
+                    # are present (e.g. _FillValue=-9999), the producer's chosen
+                    # sentinel is already in `explicit` — the default zero is
+                    # still ambiguous and must not be silently replaced.
+                    uncertain.add(native)
+                else:
+                    # Non-default native fill: the producer explicitly chose
+                    # this value, so it is intentional.
+                    explicit.add(native)
+        except (TypeError, ValueError):
+            pass
+
+        return explicit, uncertain
+
     def read(self):
         try:
             rows = []
@@ -46,6 +111,49 @@ def recurse(name, obj, path=[]):
                 try:
                     if isinstance(obj, h5py.Dataset):
                         data = obj[()]
+                        # Translate fill-value sentinels to NaN so that
+                        # pd.isnull()-based metrics (completeness, outliers, …)
+                        # correctly detect missing data.
+                        if hasattr(data, "dtype") and data.dtype.kind in ("f", "i", "u"):
+                            explicit_fills, uncertain_fills = self._collect_fill_values(obj)
+                            all_fills = explicit_fills | uncertain_fills
+                            if all_fills:
+                                # Only iterate sentinels that actually appear in
+                                # the data, so we can report accurately and avoid
+                                # unnecessary dtype promotion.
+                                matched = {fv for fv in all_fills if (data == fv).any()}
+                                if matched:
+                                    mask = np.zeros(data.shape, dtype=bool)
+                                    for fv in matched:
+                                        mask |= data == fv
+                                    n_replaced = int(mask.sum())
+
+                                    uncertain_matched = matched & uncertain_fills
+                                    if uncertain_matched:
+                                        # The only sentinel that matched is the
+                                        # HDF5 default zero — it may be valid data.
+                                        self.logger.warning(
+                                            f"Dataset '{name}': {n_replaced}/"
+                                            f"{data.size} value(s) match the HDF5 "
+                                            f"default fill value {uncertain_matched} "
+                                            f"and will be replaced with NaN. "
+                                            f"If zero is a valid measurement here "
+                                            f"(e.g. counts, indices), set a "
+                                            f"'_FillValue' attribute in the file to "
+                                            f"an unambiguous sentinel, or pass "
+                                            f"fill_values=[] at construction time to "
+                                            f"suppress native fill value replacement."
+                                        )
+                                    else:
+                                        self.logger.info(
+                                            f"Dataset '{name}': replaced "
+                                            f"{n_replaced}/{data.size} value(s) "
+                                            f"matching explicit fill sentinel(s) "
+                                            f"{matched} with NaN."
+                                        )
+
+                                    data = data.astype(np.float64)
+                                    data[mask] = np.nan
                         # If it's a 1D or structured dataset, load it into dicts
                         if isinstance(data, (list, tuple)) or hasattr(data, "dtype"):
                             try:

docs/source/limitations.rst

@@ -21,7 +21,11 @@ What We Can Do
 - Provide **quantitative metrics** for dataset readiness and visualizations of results.
 - Analyze **DCAT and DataCite JSON metadata** for FAIR compliance.
 - Identify **missing or incomplete metadata elements**.
-- Work with **structured tabular datasets** (e.g., CSV, Excel) for basic data readiness checks.
+- Work with **structured tabular datasets** (CSV, Excel, JSON, NumPy ``.npz``, and
+  HDF5 ``.h5``) for data readiness checks.  For HDF5 files, format-native missing
+  data sentinels (``_FillValue``, ``missing_value``, and the dataset's HDF5 fill
+  value) are automatically normalised to ``NaN`` before any metric is computed,
+  ensuring accurate completeness, outlier, and privacy scores.
 
 What We Cannot Do
 ~~~~~~~~~~~~~~~~~

docs/source/usage.rst

@@ -65,6 +65,29 @@ Evaluates dataset completeness by checking for missing values.
 
 **Returns**: A dictionary with an overall completeness score (1 for no missing values, 0 for all missing) and a histogram of missing value proportions per column.
 
+.. note::
+
+   **HDF5 fill value handling**: HDF5 datasets encode missing data as a numeric
+   sentinel (the *fill value*) rather than as a blank cell.  When reading an
+   ``.h5`` file AIDRIN automatically translates these sentinels to ``NaN``
+   before computing completeness, so the score reflects true data availability
+   rather than always reporting 100%.
+
+   Sentinels are collected from the following sources, in priority order:
+
+   1. **User-supplied** – pass ``fill_values=[v1, v2, …]`` to ``hdf5Reader``
+      at construction time to declare domain-specific sentinels explicitly.
+   2. **_FillValue attribute** – the NetCDF/CF convention used by virtually
+      all climate, oceanography, and atmospheric HDF5 files.
+   3. **missing_value attribute** – the older NetCDF convention; may be a
+      scalar or an array of multiple sentinels.
+   4. **HDF5 native fill value** – the value stored in the dataset's own
+      metadata (``dataset.fillvalue``).  When this equals the dtype default
+      (``0`` / ``0.0``) and no fill-value attributes are present, a warning
+      is logged before replacement because zero is a legitimate measurement in
+      many scientific datasets (e.g. counts, indices).  Set a ``_FillValue``
+      attribute in the file to an unambiguous sentinel to suppress this warning.
+
 calculate_correlations
 ^^^^^^^^^^^^^^^^^^^^^^
 
@@ -421,6 +444,12 @@ Notes
   - The local installation requires setting up Redis, Celery, and Flask (see `Installation <./installation.html>`_). The web application at `aidrin.io <https://aidrin.io>`_ handles these server-side, offering a no-setup alternative.
   - Both use the same codebase, ensuring identical functionality. The web application is ideal for users who prefer a browser-based interface.
 
-- **File Formats**: The web application supports CSV files for data uploads and DCAT/DataCite JSON for metadata in the Understandability and Usability dimension.
+- **File Formats**: The web application supports CSV, Excel, JSON, NumPy (``.npz``),
+  and HDF5 (``.h5``) files for data uploads, and DCAT/DataCite JSON for metadata
+  in the Understandability and Usability dimension.  For HDF5 files, fill-value
+  sentinels (``_FillValue``, ``missing_value``, and the HDF5 native fill value) are
+  automatically converted to ``NaN`` so that all metrics — completeness, outliers,
+  feature relevance, and privacy — operate on accurately marked missing data.  See
+  the ``calculate_completeness`` note above for the full sentinel-resolution order.
 - **Visualizations**: Generated downloadable plots (e.g., histograms, bar charts, heatmaps) are displayed in the web interface.
 - **JSON Reports**: Each dimension’s analysis generates a downloadable JSON report containing all metrics, statistics, and visualization data (where applicable).