fix: replace fill-value sentinels in HDF5 compound datasets

aidrin/file_handling/readers/hdf5_reader.py

@@ -156,6 +156,44 @@ def recurse(name, obj, path=[]):
                         if isinstance(data, (list, tuple)) or hasattr(data, "dtype"):
                             try:
                                 df = pd.DataFrame(data)
+                                if hasattr(data, "dtype") and data.dtype.kind == "V":
+                                    explicit, _ = self._collect_fill_values(obj)
+                                    for col in df.select_dtypes(include="number").columns:
+                                        # _collect_fill_values cannot unpack per-field fill
+                                        # values from a compound dataset.fillvalue (it is a
+                                        # void tuple and float() on it raises TypeError).
+                                        # Derive uncertain fills per field instead.
+                                        col_explicit = set(explicit)
+                                        col_uncertain = set()
+                                        try:
+                                            field_dtype = data.dtype.fields[col][0]
+                                            native = float(obj.fillvalue[col])
+                                            default = float(np.zeros(1, dtype=field_dtype)[0])
+                                            if native not in col_explicit:
+                                                if native == default:
+                                                    col_uncertain.add(native)
+                                                else:
+                                                    col_explicit.add(native)
+                                        except (TypeError, ValueError, KeyError):
+                                            pass
+                                        matched = (col_explicit | col_uncertain) & set(df[col].dropna().unique())
+                                        if matched:
+                                            uncertain_matched = matched & col_uncertain
+                                            if uncertain_matched:
+                                                self.logger.warning(
+                                                    f"Dataset '{name}', field '{col}': "
+                                                    f"{df[col].isin(uncertain_matched).sum()}/"
+                                                    f"{len(df[col])} value(s) match the HDF5 "
+                                                    f"default fill value {uncertain_matched} "
+                                                    f"and will be replaced with NaN. "
+                                                    f"If zero is a valid measurement here "
+                                                    f"(e.g. counts, indices), set a "
+                                                    f"'_FillValue' attribute in the file to "
+                                                    f"an unambiguous sentinel, or pass "
+                                                    f"fill_values=[] at construction time to "
+                                                    f"suppress native fill value replacement."
+                                                )
+                                            df[col] = df[col].where(~df[col].isin(matched))
                             except Exception:
                                 df = pd.DataFrame(data.tolist())  # base
                             for _, row in df.iterrows():

test_hdf5_reader.py

@@ -258,3 +258,107 @@ def test_integer_dataset_completeness(self, tmp_path, logger):
 
         completeness = 1 - col.isnull().mean()
         assert abs(completeness - 0.6) < 1e-9
+
+
+# ---------------------------------------------------------------------------
+# Compound/structured datasets: fill-value replacement across named fields
+# ---------------------------------------------------------------------------
+
+class TestCompoundDatasets:
+
+    def _make_compound_hdf5(self, path, data, fill_attr=None):
+        """Write a compound dataset HDF5 file for testing."""
+        with h5py.File(path, "w") as f:
+            ds = f.create_dataset("observations", data=data)
+            if fill_attr is not None:
+                ds.attrs["_FillValue"] = fill_attr
+
+    def test_compound_sentinel_replaced_with_nan(self, tmp_path, logger):
+        """Sentinel in a numeric field of a compound dataset is replaced with NaN."""
+        dt = np.dtype([("depth", "<f4"), ("temp", "<f4")])
+        data = np.array(
+            [(100.0, 20.5), (200.0, 25.1), (300.0, -9999.0), (400.0, 18.7)],
+            dtype=dt,
+        )
+        fpath = str(tmp_path / "compound.h5")
+        self._make_compound_hdf5(fpath, data, fill_attr=np.float32(-9999.0))
+
+        df = hdf5Reader(fpath, logger).read()
+        assert df is not None
+        assert df["temp"].isna().sum() == 1
+
+    def test_compound_non_sentinel_values_unchanged(self, tmp_path, logger):
+        """Real measurements in a compound dataset are not touched."""
+        dt = np.dtype([("depth", "<f4"), ("temp", "<f4")])
+        data = np.array(
+            [(100.0, 20.5), (200.0, 25.1), (300.0, -9999.0), (400.0, 18.7)],
+            dtype=dt,
+        )
+        fpath = str(tmp_path / "compound.h5")
+        self._make_compound_hdf5(fpath, data, fill_attr=np.float32(-9999.0))
+
+        df = hdf5Reader(fpath, logger).read()
+        assert df is not None
+        valid = df["temp"].dropna().tolist()
+        assert sorted(valid) == pytest.approx(sorted([20.5, 25.1, 18.7]), rel=1e-4)
+
+    def test_compound_completeness_reflects_true_missingness(self, tmp_path, logger):
+        """Completeness score for a compound dataset reflects actual missing entries."""
+        dt = np.dtype([("depth", "<f4"), ("temp", "<f4")])
+        data = np.array(
+            [(100.0, 20.5), (200.0, 25.1), (300.0, -9999.0), (400.0, 18.7)],
+            dtype=dt,
+        )
+        fpath = str(tmp_path / "compound.h5")
+        self._make_compound_hdf5(fpath, data, fill_attr=np.float32(-9999.0))
+
+        df = hdf5Reader(fpath, logger).read()
+        assert df is not None
+        completeness = 1 - df["temp"].isnull().mean()
+        assert abs(completeness - 0.75) < 1e-6, (
+            f"Expected completeness 0.75, got {completeness}. "
+            "Sentinel was not replaced with NaN in compound dataset."
+        )
+
+    def test_compound_no_fill_attr_no_nan(self, tmp_path, logger):
+        """Compound dataset with no fill attribute comes through unchanged."""
+        dt = np.dtype([("depth", "<f4"), ("temp", "<f4")])
+        data = np.array(
+            [(100.0, 20.5), (200.0, 25.1), (300.0, 18.7)],
+            dtype=dt,
+        )
+        fpath = str(tmp_path / "compound_nofill.h5")
+        self._make_compound_hdf5(fpath, data, fill_attr=None)
+
+        df = hdf5Reader(fpath, logger).read()
+        assert df is not None
+        assert df["temp"].isna().sum() == 0
+
+    def test_compound_uncertain_zero_emits_warning(self, tmp_path, logger, caplog):
+        """Uncertain zero fill in a compound field emits a WARNING, consistent with flat datasets."""
+        dt = np.dtype([("depth", "<f4"), ("temp", "<f4")])
+        data = np.array(
+            [(100.0, 0.0), (200.0, 25.1), (300.0, 18.7)],
+            dtype=dt,
+        )
+        fpath = str(tmp_path / "compound_uncertain.h5")
+        self._make_compound_hdf5(fpath, data, fill_attr=None)
+
+        with caplog.at_level(logging.WARNING):
+            hdf5Reader(fpath, logger).read()
+
+        assert any("default fill value" in r.message for r in caplog.records)
+
+    def test_compound_uncertain_zero_still_replaced(self, tmp_path, logger):
+        """Uncertain zero in a compound field is replaced with NaN even though it emits a WARNING."""
+        dt = np.dtype([("depth", "<f4"), ("temp", "<f4")])
+        data = np.array(
+            [(100.0, 0.0), (200.0, 25.1), (300.0, 18.7)],
+            dtype=dt,
+        )
+        fpath = str(tmp_path / "compound_uncertain2.h5")
+        self._make_compound_hdf5(fpath, data, fill_attr=None)
+
+        df = hdf5Reader(fpath, logger).read()
+        assert df is not None
+        assert df["temp"].isna().sum() == 1