From 021d0a6d1ec7bd2ff2e74947c1b4a75343c482a1 Mon Sep 17 00:00:00 2001 From: Thibaud Toullier Date: Wed, 29 Jan 2025 14:45:34 +0100 Subject: [PATCH 1/4] :sparkles: First working version with `zarr` --- h5grove/content.py | 127 ++++++++++++++++++++--------------- h5grove/custom_types.py | 14 ++++ h5grove/utils.py | 118 ++++++++++++++++++++++---------- h5grove/zarr_type_wrapper.py | 115 +++++++++++++++++++++++++++++++ setup.cfg | 1 + 5 files changed, 283 insertions(+), 92 deletions(-) create mode 100644 h5grove/custom_types.py create mode 100644 h5grove/zarr_type_wrapper.py diff --git a/h5grove/content.py b/h5grove/content.py index eaa9e4d..e231be0 100644 --- a/h5grove/content.py +++ b/h5grove/content.py @@ -3,12 +3,12 @@ from typing import ( Any, Generic, - TypeVar, cast, ) - +import os import contextlib from pathlib import Path +import zarr import h5py import numpy as np @@ -17,6 +17,13 @@ except ImportError: pass +from .custom_types import ( + T, + TDataset, + TGroup, + TFile, + TDatatype +) from .models import ( LinkResolution, Selection, @@ -44,6 +51,8 @@ hdf_path_join, get_dataset_slice, sorted_dict, + is_h5py_file, + close_file, ) @@ -117,51 +126,47 @@ def target_path(self) -> str: return self._target_path -T = TypeVar("T", h5py.Dataset, h5py.Datatype, h5py.Group) - - class ResolvedEntityContent(EntityContent, Generic[T]): """Content for a link that can be resolved into a h5py entity""" - def __init__(self, path: str, h5py_entity: T): + def __init__(self, path: str, entity: T): super().__init__(path) - self._h5py_entity = h5py_entity - """Resolved h5py entity""" + self._entity = entity + """Resolved h5py or zarr entity""" def attributes( self, attr_keys: Sequence[str] | None = None ) -> dict[str, AttributeMetadata]: """Attributes of the h5py entity. Can be filtered by keys.""" if attr_keys is None: - return dict((*self._h5py_entity.attrs.items(),)) + return dict((*self._entity.attrs.items(),)) - return dict((key, self._h5py_entity.attrs[key]) for key in attr_keys) + return dict((key, self._entity.attrs[key]) for key in attr_keys) def metadata(self, depth=None) -> ResolvedEntityMetadata: """Resolved entity metadata""" - attribute_names = sorted(self._h5py_entity.attrs.keys()) + attribute_names = sorted(self._entity.attrs.keys()) return sorted_dict( ( "attributes", [ - attr_metadata(self._h5py_entity.attrs, name) + attr_metadata(self._entity.attrs, name) for name in attribute_names ], ), *super().metadata().items(), ) - -class DatasetContent(ResolvedEntityContent[h5py.Dataset]): +class DatasetContent(ResolvedEntityContent[TDataset]): kind = "dataset" def metadata(self, depth=None) -> DatasetMetadata: """Dataset metadata""" return sorted_dict( - ("chunks", self._h5py_entity.chunks), - ("filters", get_filters(self._h5py_entity)), - ("shape", self._h5py_entity.shape), - ("type", get_type_metadata(self._h5py_entity.id.get_type())), + ("chunks", self._entity.chunks), + ("filters", get_filters(self._entity)), + ("shape", self._entity.shape), + ("type", get_type_metadata(self._entity)), *super().metadata().items(), ) @@ -179,7 +184,7 @@ def data( - `origin` (default): No conversion - `safe`: Convert to a type supported by JS typedarray (https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Global_Objects/TypedArray) """ - result = convert(get_dataset_slice(self._h5py_entity, selection), dtype) + result = convert(get_dataset_slice(self._entity, selection), dtype) # Do not flatten scalars nor h5py.Empty if flatten and isinstance(result, np.ndarray): @@ -209,20 +214,20 @@ def _get_finite_data(self, selection: Selection) -> np.ndarray: return data[mask] -class GroupContent(ResolvedEntityContent[h5py.Group]): +class GroupContent(ResolvedEntityContent[TGroup]): kind = "group" - def __init__(self, path: str, h5py_entity: h5py.Group, h5file: h5py.File): - super().__init__(path, h5py_entity) - self._h5file = h5file + def __init__(self, path: str, entity: TGroup, file_entity: TFile): + super().__init__(path, entity) + self._file_entity = file_entity """File in which the entity was resolved. This is needed to resolve child entity.""" def _get_child_metadata_content(self, depth=0): return [ create_content( - self._h5file, hdf_path_join(self._path, child_path) + self._file_entity, hdf_path_join(self._path, child_path) ).metadata(depth) - for child_path in self._h5py_entity.keys() + for child_path in self._entity.keys() ] def metadata(self, depth: int = 1) -> GroupMetadata: @@ -239,19 +244,19 @@ def metadata(self, depth: int = 1) -> GroupMetadata: ) -class DatatypeContent(ResolvedEntityContent[h5py.Datatype]): +class DatatypeContent(ResolvedEntityContent[TDatatype]): kind = "datatype" def metadata(self, depth=None) -> DatatypeMetadata: """Datatype metadata""" return sorted_dict( - ("type", get_type_metadata(self._h5py_entity.id)), + ("type", get_type_metadata(self._entity)), *super().metadata().items(), ) def create_content( - h5file: h5py.File, + file_entity: TFile, path: str | None, resolve_links: LinkResolution = LinkResolution.ONLY_VALID, ): @@ -259,7 +264,7 @@ def create_content( Factory function to get entity content from a HDF5 file. This handles external/soft link resolution and dataset decompression. - :param h5file: An open HDF5 file containing the entity + :param file_entity: An open HDF5 file containing the entity :param path: Path to the entity in the file. :param resolve_links: Tells which external and soft links should be resolved. Defaults to resolving only valid links. :raises h5grove.utils.PathError: If the path cannot be found in the file @@ -269,7 +274,7 @@ def create_content( if path is None: path = "/" - entity = get_entity_from_file(h5file, path, resolve_links) + entity = get_entity_from_file(file_entity, path, resolve_links) if isinstance(entity, h5py.ExternalLink): return ExternalLinkContent(path, entity) @@ -277,16 +282,18 @@ def create_content( if isinstance(entity, h5py.SoftLink): return SoftLinkContent(path, entity) - if isinstance(entity, h5py.Dataset): + if isinstance(entity, (h5py.Dataset, zarr.Array)): return DatasetContent(path, entity) - if isinstance(entity, h5py.Group): - return GroupContent(path, entity, h5file) + if isinstance(entity, (h5py.Group, zarr.Group)): + return GroupContent(path, entity, file_entity) - if isinstance(entity, h5py.Datatype): + if isinstance(entity, (h5py.Datatype, np.dtype)): return DatatypeContent(path, entity) - raise TypeError(f"h5py entity {type(entity)} not supported") + + + raise TypeError(f"Entity {type(entity)} not supported") @contextlib.contextmanager @@ -305,7 +312,7 @@ def get_content_from_file( fallback=LinkResolution.ONLY_VALID, ) except QueryArgumentError as e: - f.close() + close_file(filepath, f) raise create_error(422, str(e)) try: @@ -315,7 +322,7 @@ def get_content_from_file( except QueryArgumentError as e: raise create_error(422, str(e)) finally: - f.close() + close_file(filepath, f) @contextlib.contextmanager @@ -324,9 +331,9 @@ def get_list_of_paths( base_path: str | None, create_error: Callable[[int, str], Exception], resolve_links_arg: str | None = LinkResolution.ONLY_VALID, - h5py_options: dict[str, Any] = {}, + open_options: dict[str, Any] = {}, ): - f = open_file_with_error_fallback(filepath, create_error, h5py_options) + f = open_file_with_error_fallback(filepath, create_error, open_options) try: resolve_links = parse_link_resolution_arg( @@ -338,20 +345,28 @@ def get_list_of_paths( names = [] - def get_path(name: bytes): - full_path = hdf_path_join(base_path, name.decode()) - content = create_content(f, full_path, resolve_links) - names.append(content.path) - - try: - base_content = create_content(f, base_path, resolve_links) - assert isinstance(base_content, GroupContent) - names.append(base_content.path) - base_content._h5py_entity.id.links.visit(get_path) - yield names - except NotFoundError as e: - raise create_error(404, str(e)) - except QueryArgumentError as e: - raise create_error(422, str(e)) - finally: - f.close() + if is_h5py_file(filepath): + def get_path(name: bytes): + full_path = hdf_path_join(base_path, name.decode()) + content = create_content(f, full_path, resolve_links) + names.append(content.path) + + + try: + base_content = create_content(f, base_path, resolve_links) + assert isinstance(base_content, GroupContent) + names.append(base_content.path) + base_content._entity.id.links.visit(get_path) + yield names + except NotFoundError as e: + raise create_error(404, str(e)) + except QueryArgumentError as e: + raise create_error(422, str(e)) + finally: + close_file(filepath, f) + else: + entity = get_entity_from_file(f, base_path, resolve_links) + if isinstance(entity, zarr.Group): + yield [m[0] for m in entity.members(max_depth=None)] + else: + yield entity.path diff --git a/h5grove/custom_types.py b/h5grove/custom_types.py new file mode 100644 index 0000000..bc078e4 --- /dev/null +++ b/h5grove/custom_types.py @@ -0,0 +1,14 @@ +from typing import TypeVar, Union +import zarr +import h5py +import numpy as np + +TH5py = TypeVar("TH5py", h5py.Dataset, h5py.Datatype, h5py.Group) +TZarr = TypeVar("TZarr", zarr.Array, zarr.Group) +T = TypeVar("T", h5py.Dataset, h5py.Datatype, h5py.Group, zarr.Array, zarr.Group) +TDataset = TypeVar("TDataset", h5py.Dataset, zarr.Array) +TDatatype = TypeVar("TDatatype", h5py.Datatype, np.dtype) +TGroup = TypeVar("TGroup", h5py.Group, zarr.Group) +TFile = TypeVar("TFile", h5py.File, zarr.Group) +TAttributes = TypeVar("TAttributes", h5py.AttributeManager, zarr.core.attributes.Attributes) +TNp = TypeVar("TNp", np.ndarray, np.number, np.bool_) diff --git a/h5grove/utils.py b/h5grove/utils.py index c49f256..1edf3de 100644 --- a/h5grove/utils.py +++ b/h5grove/utils.py @@ -3,11 +3,21 @@ from typing import Any, TypeVar from pathlib import Path +import zarr import h5py from h5py.version import version_tuple as h5py_version from os.path import basename import numpy as np +from .custom_types import ( + T, + TH5py, + TAttributes, + TFile, + TDataset, + TNp +) +from .zarr_type_wrapper import ZarrTypeWrapper from .models import ( H5pyEntity, LinkResolution, @@ -43,7 +53,7 @@ def _legacy_get_attr_id(entity_attrs: h5py.AttributeManager, attr_name: str): return h5py.h5a.open(entity_attrs._id, entity_attrs._e(attr_name)) -get_attr_id = ( +get_attr_id_hdf5 = ( _legacy_get_attr_id if h5py_version.major <= 2 and h5py_version.minor <= 9 else _get_attr_id @@ -51,45 +61,59 @@ def _legacy_get_attr_id(entity_attrs: h5py.AttributeManager, attr_name: str): def attr_metadata( - entity_attrs: h5py.AttributeManager, attr_name: str + entity_attrs: TAttributes, attr_name: str ) -> AttributeMetadata: - attrId = get_attr_id(entity_attrs, attr_name) - - return { - "name": attr_name, - "shape": attrId.shape, - "type": get_type_metadata(attrId.get_type()), - } + if isinstance(entity_attrs, h5py.AttributeManager): + attrId = get_attr_id_hdf5(entity_attrs, attr_name) + return { + "name": attr_name, + "shape": attrId.shape, + "type": get_type_metadata_h5py(attrId.get_type()), + } + else: + cast = np.array(entity_attrs[attr_name]) + return { + "name": attr_name, + "shape": cast.shape, + "type": ZarrTypeWrapper(cast.dtype, cast.shape).get_metadata() + } def get_entity_from_file( - h5file: h5py.File, + file_entity: TFile, path: str, resolve_links: LinkResolution = LinkResolution.ONLY_VALID, ) -> H5pyEntity: if path == "/": - return h5file[path] + return file_entity[path] - link = h5file.get(path, getlink=True) + if isinstance(file_entity, h5py.File): + link = file_entity.get(path, getlink=True) + else: + link = file_entity.get(path) if link is None: - raise PathError(f"{path} is not a valid path in {basename(h5file.filename)}") + if isinstance(file_entity, h5py.File): + raise PathError(f"{path} is not a valid path in {basename(file_entity.filename)}") + else: + raise PathError(f"{path} is not a valid path in {file_entity.store}") + if isinstance(link, h5py.ExternalLink) or isinstance(link, h5py.SoftLink): if resolve_links == LinkResolution.NONE: return link try: - return h5file[path] + return file_entity[path] except (OSError, KeyError): if resolve_links == LinkResolution.ONLY_VALID: return link raise LinkError( - f"Cannot resolve {link} at {path} of {basename(h5file.filename)}" + f"Cannot resolve {link} at {path} of {basename(file_entity.filename)}" ) - return h5file[path] + return file_entity[path] def parse_slice(slice_str: str) -> tuple[slice | int, ...]: @@ -138,8 +162,15 @@ def parse_slice_member(slice_member: str) -> slice | int: def sorted_dict(*args: tuple[str, Any]): return dict(sorted(args, key=lambda entry: entry[0])) +def get_type_metadata(entity: T) -> TypeMetadata: + if isinstance(entity, (h5py.Dataset, h5py.Datatype, h5py.Group)): + return get_type_metadata_h5py(entity.id.get_type()) + else: + return ZarrTypeWrapper(entity.dtype, entity.shape).get_metadata() + -def get_type_metadata(type_id: h5py.h5t.TypeID) -> TypeMetadata: + +def get_type_metadata_h5py(type_id: h5py.h5t.TypeID) -> TypeMetadata: base_metadata: TypeMetadata = { "class": type_id.get_class(), "dtype": stringify_dtype(type_id.dtype), @@ -176,7 +207,7 @@ def get_type_metadata(type_id: h5py.h5t.TypeID) -> TypeMetadata: if isinstance(type_id, h5py.h5t.TypeCompoundID): for i in range(0, type_id.get_nmembers()): - members[type_id.get_member_name(i).decode("utf-8")] = get_type_metadata( + members[type_id.get_member_name(i).decode("utf-8")] = get_type_metadata_h5py( type_id.get_member_type(i) ) @@ -191,17 +222,17 @@ def get_type_metadata(type_id: h5py.h5t.TypeID) -> TypeMetadata: return { **base_metadata, "members": members, - "base": get_type_metadata(type_id.get_super()), + "base": get_type_metadata_h5py(type_id.get_super()), } if isinstance(type_id, h5py.h5t.TypeVlenID): - return {**base_metadata, "base": get_type_metadata(type_id.get_super())} + return {**base_metadata, "base": get_type_metadata_h5py(type_id.get_super())} if isinstance(type_id, h5py.h5t.TypeArrayID): return { **base_metadata, "dims": type_id.get_array_dims(), - "base": get_type_metadata(type_id.get_super()), + "base": get_type_metadata_h5py(type_id.get_super()), } return base_metadata @@ -225,10 +256,7 @@ def _sanitize_dtype(dtype: np.dtype) -> np.dtype: return dtype -T = TypeVar("T", np.ndarray, np.number, np.bool_) - - -def convert(data: T, dtype: str | None = "origin") -> T: +def convert(data: TNp, dtype: str | None = "origin") -> TNp: """Convert array or numpy scalar to given dtype query param :param data: nD array or scalar to convert @@ -324,7 +352,7 @@ def parse_link_resolution_arg( ) -def get_dataset_slice(dataset: h5py.Dataset, selection: Selection): +def get_dataset_slice(dataset: TDataset, selection: Selection): if selection is None: return dataset[()] @@ -340,16 +368,18 @@ def get_dataset_slice(dataset: h5py.Dataset, selection: Selection): def get_filters( - dataset: h5py.Dataset, + dataset: TDataset, ) -> list[dict[str, int | str]] | None: - property_list = dataset.id.get_create_plist() - - n_filters = property_list.get_nfilters() - if n_filters <= 0: - return None + if isinstance(dataset, h5py.Dataset): + property_list = dataset.id.get_create_plist() - return [get_filter_info(property_list.get_filter(i)) for i in range(n_filters)] + n_filters = property_list.get_nfilters() + if n_filters <= 0: + return None + return [get_filter_info(property_list.get_filter(i)) for i in range(n_filters)] + else: + return [codec.to_dict() for codec in dataset.compressors] def get_filter_info( filter: tuple[int, int, tuple[int, ...], str] @@ -368,14 +398,30 @@ def stringify_dtype(dtype: np.dtype) -> StrDtype: k: stringify_dtype(dtype_tuple[0]) for k, dtype_tuple in dtype.fields.items() } +def is_h5py_file(filepath): + return filepath.endswith(('.h5', '.hdf5')) + +def is_zarr_file(filepath): + return filepath.endswith('.zarr') + +def close_file(filepath, f): + if is_h5py_file(filepath): + f.close() + else: + f.store.close() def open_file_with_error_fallback( filepath: str | Path, create_error: Callable[[int, str], Exception], - h5py_options: dict[str, Any] = {}, -) -> h5py.File: + options: dict[str, Any] = {}, +) -> TFile: try: - f = h5py.File(filepath, "r", **h5py_options) + if is_h5py_file(filepath): + f = h5py.File(filepath, "r", **options) + elif is_zarr_file(filepath): + f = zarr.open(filepath, mode="r", **options) + else: + raise create_error(406, "File extension not recognized, or unsuppported!") except OSError as e: if isinstance(e, FileNotFoundError) or "No such file or directory" in str(e): raise create_error(404, "File not found!") diff --git a/h5grove/zarr_type_wrapper.py b/h5grove/zarr_type_wrapper.py new file mode 100644 index 0000000..bb2947b --- /dev/null +++ b/h5grove/zarr_type_wrapper.py @@ -0,0 +1,115 @@ +import numpy as np + +from .models import TypeMetadata + + +class ZarrTypeWrapper(): + def __init__(self, type_id: np.dtype, shape: tuple): + + self.type_id = type_id + self.shape = shape + + for key in dir(type_id): + if not key.startswith('__'): + try: + setattr(self, key, getattr(self.type_id, key)) + except: + pass + + def is_integer(self) -> bool: + return len(self.shape) == 0 and (self.type_id.kind == 'u' or self.type_id.kind == 'i') + + def is_float(self) -> bool: + return len(self.shape) == 0 and self.type_id.kind == 'f' + + def is_string(self) -> bool: + return self.type_id.kind == 'U' + + def is_byte(self) -> bool: + return self.type_id.kind == 'b' or self.type_id.kind == 'B' + + def is_opaque(self) -> bool: + return self.type_id.kind == 'O' + + def is_compound(self) -> bool: + return self.type_id.kind == 'V' + + def is_array(self) -> bool: + return len(self.shape) > 0 + + + def get_byte_order(self) -> int: + match self.type_id.byteorder: + case '=': + return 0 + case '<': + return 0 + case '>': + return 1 + case '|': + return 4 + + def get_charset(self) -> int: + """Returns the charset of given np.dtype str. + + Actually the str are unicode objects. + We return h5py.h5t.CSET_UTF8 by default. + """ + return 1 + + def get_strpad(self) -> int: + return 0 + + def is_variable_str(self) -> bool: + return True + + def get_metadata(self) -> TypeMetadata: + base_metadata : TypeMetadata = { + "class": type(self.type_id).__name__, + "dtype": str(self), + "size": self.itemsize if hasattr(self, "itemsize") else (), + + } + members = {} + if self.is_integer(): + return { + **base_metadata, + "order": self.get_byte_order(), + "sign": 0 if self.kind == 'u' else 1, + } + + if self.is_float(): + return { + **base_metadata, + "order": self.get_byte_order(), + } + + if self.is_string(): + return { + **base_metadata, + "cset": self.get_charset(), + "strpad": self.get_strpad(), + "vlen": self.is_variable_str(), + } + + if self.is_byte(): + return {**base_metadata, "order": self.get_byte_order()} + + if self.is_opaque(): + return {**base_metadata, "tag": "object"} + + if self.is_compound(): + for name, dt in self.fields.items(): + members[name] = ZarrTypeWrapper(type_id=dt[0], shape=dt[1]).get_metadata() + + return {**base_metadata, "members": members} + + if self.is_array(): + return { + **base_metadata, + "dims": self.shape, + "base": ZarrTypeWrapper(type_id=self.type_id, shape=()) + } + + def __str__(self): + return str(self.type_id) diff --git a/setup.cfg b/setup.cfg index f2926d8..2a7f4ce 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ install_requires = numpy orjson h5py >= 3 + zarr >= 2 tifffile typing-extensions From caf20341d4ba211e7a6ae100d91b4d6572d3213e Mon Sep 17 00:00:00 2001 From: Thibaud Toullier Date: Mon, 3 Feb 2025 11:30:52 +0100 Subject: [PATCH 2/4] :wrench: fix issues with path between Zarr and HDF --- h5grove/content.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/h5grove/content.py b/h5grove/content.py index e231be0..d1c42d7 100644 --- a/h5grove/content.py +++ b/h5grove/content.py @@ -52,6 +52,7 @@ get_dataset_slice, sorted_dict, is_h5py_file, + is_zarr_file, close_file, ) @@ -225,7 +226,7 @@ def __init__(self, path: str, entity: TGroup, file_entity: TFile): def _get_child_metadata_content(self, depth=0): return [ create_content( - self._file_entity, hdf_path_join(self._path, child_path) + self._file_entity, os.path.join(self._path, child_path) ).metadata(depth) for child_path in self._entity.keys() ] @@ -306,6 +307,9 @@ def get_content_from_file( ): f = open_file_with_error_fallback(filepath, create_error, h5py_options) + if is_zarr_file(filepath): + path = path.lstrip('/') + try: resolve_links = parse_link_resolution_arg( resolve_links_arg, @@ -314,7 +318,6 @@ def get_content_from_file( except QueryArgumentError as e: close_file(filepath, f) raise create_error(422, str(e)) - try: yield create_content(f, path, resolve_links) except NotFoundError as e: From e5c04d1c16006c5ed825118d168af43b728b352d Mon Sep 17 00:00:00 2001 From: Thibaud Toullier Date: Tue, 11 Mar 2025 10:36:52 +0100 Subject: [PATCH 3/4] :sparkles: Allow s3fs.File to be opened --- h5grove/content.py | 8 ++++---- h5grove/utils.py | 29 ++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/h5grove/content.py b/h5grove/content.py index d1c42d7..3e083e1 100644 --- a/h5grove/content.py +++ b/h5grove/content.py @@ -51,8 +51,8 @@ hdf_path_join, get_dataset_slice, sorted_dict, - is_h5py_file, - is_zarr_file, + is_file_type, + FileTypeEnum, close_file, ) @@ -307,7 +307,7 @@ def get_content_from_file( ): f = open_file_with_error_fallback(filepath, create_error, h5py_options) - if is_zarr_file(filepath): + if is_file_type(filepath, FileTypeEnum.ZARR): path = path.lstrip('/') try: @@ -348,7 +348,7 @@ def get_list_of_paths( names = [] - if is_h5py_file(filepath): + if is_file_type(filepath, FileTypeEnum.H5PY): def get_path(name: bytes): full_path = hdf_path_join(base_path, name.decode()) content = create_content(f, full_path, resolve_links) diff --git a/h5grove/utils.py b/h5grove/utils.py index 1edf3de..e119218 100644 --- a/h5grove/utils.py +++ b/h5grove/utils.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable -from typing import Any, TypeVar +from enum import Enum, auto +from typing import Any from pathlib import Path import zarr @@ -11,7 +12,6 @@ from .custom_types import ( T, - TH5py, TAttributes, TFile, TDataset, @@ -29,6 +29,11 @@ ) +class FileTypeEnum(Enum): + H5PY = auto() + ZARR = auto() + + class NotFoundError(Exception): pass @@ -404,8 +409,22 @@ def is_h5py_file(filepath): def is_zarr_file(filepath): return filepath.endswith('.zarr') +def is_file_type(filepath, file_type = FileTypeEnum.H5PY): + if isinstance(filepath, str): + filepath_str = filepath + elif getattr(filepath, 'path', None) is not None: + filepath_str = getattr(filepath, 'path', None) + else: + filepath_str = str(filepath) + + match file_type: + case FileTypeEnum.H5PY: + return is_h5py_file(filepath_str) + case FileTypeEnum.ZARR: + return is_zarr_file(filepath_str) + def close_file(filepath, f): - if is_h5py_file(filepath): + if is_file_type(filepath, FileTypeEnum.H5PY): f.close() else: f.store.close() @@ -416,9 +435,9 @@ def open_file_with_error_fallback( options: dict[str, Any] = {}, ) -> TFile: try: - if is_h5py_file(filepath): + if is_file_type(filepath, file_type=FileTypeEnum.H5PY): f = h5py.File(filepath, "r", **options) - elif is_zarr_file(filepath): + elif is_file_type(filepath, file_type=FileTypeEnum.ZARR): f = zarr.open(filepath, mode="r", **options) else: raise create_error(406, "File extension not recognized, or unsuppported!") From 91a02433291db74a930913e1b2ded8624596bd67 Mon Sep 17 00:00:00 2001 From: Thibaud Toullier Date: Thu, 30 Apr 2026 11:44:01 +0200 Subject: [PATCH 4/4] :sparkles: add `zarr.storage.StoreLike` + ensure_slash --- h5grove/content.py | 9 +++++---- h5grove/utils.py | 10 +++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/h5grove/content.py b/h5grove/content.py index 3e083e1..5656e76 100644 --- a/h5grove/content.py +++ b/h5grove/content.py @@ -54,6 +54,7 @@ is_file_type, FileTypeEnum, close_file, + ensure_slash, ) @@ -299,7 +300,7 @@ def create_content( @contextlib.contextmanager def get_content_from_file( - filepath: str | Path, + filepath: str | Path | zarr.storage.StoreLike, path: str | None, create_error: Callable[[int, str], Exception], resolve_links_arg: str | None = LinkResolution.ONLY_VALID, @@ -330,7 +331,7 @@ def get_content_from_file( @contextlib.contextmanager def get_list_of_paths( - filepath: str | Path, + filepath: str | Path | zarr.storage.StoreLike, base_path: str | None, create_error: Callable[[int, str], Exception], resolve_links_arg: str | None = LinkResolution.ONLY_VALID, @@ -370,6 +371,6 @@ def get_path(name: bytes): else: entity = get_entity_from_file(f, base_path, resolve_links) if isinstance(entity, zarr.Group): - yield [m[0] for m in entity.members(max_depth=None)] + yield [ensure_slash(m[0]) for m in entity.members(max_depth=None)] else: - yield entity.path + yield ensure_slash(entity.path) diff --git a/h5grove/utils.py b/h5grove/utils.py index e119218..114daac 100644 --- a/h5grove/utils.py +++ b/h5grove/utils.py @@ -421,6 +421,8 @@ def is_file_type(filepath, file_type = FileTypeEnum.H5PY): case FileTypeEnum.H5PY: return is_h5py_file(filepath_str) case FileTypeEnum.ZARR: + if isinstance(filepath, zarr.storage.StoreLike): + return True return is_zarr_file(filepath_str) def close_file(filepath, f): @@ -430,10 +432,11 @@ def close_file(filepath, f): f.store.close() def open_file_with_error_fallback( - filepath: str | Path, + filepath: str | Path | zarr.storage.StoreLike, create_error: Callable[[int, str], Exception], options: dict[str, Any] = {}, ) -> TFile: + try: if is_file_type(filepath, file_type=FileTypeEnum.H5PY): f = h5py.File(filepath, "r", **options) @@ -449,3 +452,8 @@ def open_file_with_error_fallback( raise e return f + +def ensure_slash(path): + if not path.startswith('/'): + return "/" + path + return path