Skip to content

feat: add zarr store#89

Draft
cauriol wants to merge 26 commits into
mainfrom
zarr_store
Draft

feat: add zarr store#89
cauriol wants to merge 26 commits into
mainfrom
zarr_store

fix: remove donwloadlink for zarr

d74acfb
Select commit
Loading
Failed to load commit list.
Sign in for the full log view
GitHub Actions / Test Results failed Apr 13, 2026 in 0s

1 fail, 113 pass in 24s

  2 files  ±0    2 suites  ±0   24s ⏱️ -1s
114 tests ±0  113 ✅ ±0  0 💤 ±0  1 ❌ ±0 
228 runs  ±0  226 ✅ ±0  0 💤 ±0  2 ❌ ±0 

Results for commit d74acfb. ± Comparison against earlier commit b301dbb.

Annotations

Check warning on line 0 in tests.test_zarr

See this annotation in the file changed.

@github-actions github-actions / Test Results

All 2 runs failed: test_zarr_file_display (tests.test_zarr)

artifacts/unit-test-results-python3.13-ubuntu-latest/junit-report.xml [took 0s]
artifacts/unit-test-results-python3.9-ubuntu-latest/junit-report.xml [took 0s]
Raw output
AttributeError: 'EOProduct' object has no attribute 'request_asset'
defaults = TestDefaults(collection='S2_MSI_L1C', bbox_wkt='POLYGON ((0.0 43.0, 1.0 43.0, 1.0 44.0, 0.0 44.0, 0.0 43.0))', bbox_ge..., [0, 43]]]}, bbox_csv='0,43,1,44', bbox_list=[0, 43, 1, 44], start='2018-01-20T00:00:00Z', end='2018-01-25T00:00:00Z')
mock_data_download_requests_get = <MagicMock name='get' id='139816199999168'>

    async def test_zarr_file_display(
        defaults,
        mock_data_download_requests_get,
    ):
        """get_data_with_file should request streaming for a file inside a .zarr asset."""
        collection = defaults.collection
        item_id = "dummy_id"
        client = BaseDataDownloadClient()
        product = EOProduct(
            "cop_dataspace",
            dict(
                geometry="POINT (0 0)",
                title="dummy_product",
                id=item_id,
            ),
            collection=collection,
        )
        product.assets.update({"example.zarr": {"href": "https://data/cop_dataspace/example.zarr"}})
        config = PluginConfig()
        config.priority = 0
        downloader = HTTPDownload("cop_dataspace", config)
        product.register_downloader(downloader=downloader, authenticator=None)
    
        dag = mock.Mock()
        dag.search.return_value = SearchResult([product])
        request = mock.Mock()
        request.app.state.dag = dag
        request.base_url._url = "http://testserver/"
        mock_data_download_requests_get.return_value = mock.Mock(
            status_code=200,
            headers={"Content-Type": "text/plain"},
        )
        mock_data_download_requests_get.return_value.iter_content.return_value = iter([b"hello"])
    
>       response = client.get_data_with_file(
            "cop_dataspace",
            collection,
            item_id,
            "example.zarr",
            request,
            "group/foo.txt",
        )

tests/test_zarr.py:123: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
stac_fastapi/eodag/extensions/data_download.py:126: in get_data_with_file
    return self.get_data(federation_backend, collection_id, item_id, asset_name, request, file_path)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <stac_fastapi.eodag.extensions.data_download.BaseDataDownloadClient object at 0x7f297eaec7d0>
federation_backend = 'cop_dataspace', collection_id = 'S2_MSI_L1C'
item_id = 'dummy_id', asset_name = 'example.zarr'
request = <Mock id='139816200001856'>, file_path = 'group/foo.txt'

    def get_data(
        self,
        federation_backend: str,
        collection_id: str,
        item_id: str,
        asset_name: Optional[str],
        request: Request,
        file_path: Optional[str] = None,
    ) -> Union[StreamingResponse, RedirectResponse, JSONResponse]:
        """Download an asset"""
    
        dag = cast(EODataAccessGateway, request.app.state.dag)  # type: ignore
    
        # check if the collection is known
        try:
            dag.get_collection_from_alias(collection_id)
        except NoMatchingCollection as e:
            raise NotFoundError(e) from e
    
        search_results = dag.search(id=item_id, collection=collection_id, provider=federation_backend)
        if len(search_results) > 0:
            product = cast(EOProduct, search_results[0])
    
        else:
            raise NotFoundError(
                f"Could not find {item_id} item in {collection_id} collection",
                f" for backend {federation_backend}.",
            )
    
        settings = get_settings()
        auto_order_whitelist = settings.auto_order_whitelist
        if federation_backend in auto_order_whitelist:
            logger.info(f"Provider {federation_backend} is whitelisted, ordering product before download")
    
            auth = product.downloader_auth.authenticate() if product.downloader_auth else None
            logger.debug(f"Polling product {product}")
            try:
                product.downloader.order(product=product, auth=auth)  # type: ignore
            # when a NotAvailableError is catched, it means the product is not ready and still needs to be polled
            except NotAvailableError:
                product.properties["order:status"] = STAGING_STATUS
            except Exception as e:
                if (
                    isinstance(e, DownloadError) or isinstance(e, ValidationError)
                ) and "order status could not be checked" in e.args[0]:
                    raise NotFoundError(f"Item {item_id} does not exist. Please order it first") from e
                raise NotFoundError(e) from e
    
        try:
            auth = product.downloader_auth.authenticate() if product.downloader_auth else None
        except Exception as e:
            logger.error(f"Authentication failed: {e}")
            raise NotAvailableError("Token not ready, authentication failed. Please try again later.") from e
    
        if product.downloader is None:
            logger.error("No downloader available for %s", product)
            raise NotFoundError(
                f"Impossible to download {item_id} item in {collection_id} collection",
                f" for backend {federation_backend}.",
            )
        if product.properties.get("order:status", ONLINE_STATUS) != ONLINE_STATUS:
            # "title" property is a fake one create by EODAG, set it to the item ID
            # (the same one as order ID) to make error message clearer
            product.properties["title"] = product.properties["id"]
            # "orderLink" property is set to auth provider conf matching url to create its auth plugin
            status_link_metadata = product.downloader.config.order_on_response["metadata_mapping"]["eodag:status_link"]
            product.properties["eodag:order_link"] = product.properties["eodag:status_link"] = get_metadata_path_value(
                status_link_metadata
            ).format(orderId=item_id)
    
            search_link_metadata = product.downloader.config.order_on_response["metadata_mapping"].get(
                "eodag:search_link"
            )
            if search_link_metadata:
                product.properties["eodag:search_link"] = get_metadata_path_value(search_link_metadata).format(
                    orderId=item_id
                )
    
            order_status_method = getattr(product.downloader, "_order_status", None)
            if not order_status_method:
                raise MisconfiguredError("Product downloader must have the order status request method")
    
            auth = product.downloader_auth.authenticate() if product.downloader_auth else None
    
            logger.debug("Poll product")
            try:
                order_status_method(product=product, auth=auth)
            # when a NotAvailableError is catched, it means the product is not ready and still needs to be polled
            except NotAvailableError:
                product.properties["order:status"] = STAGING_STATUS
            except Exception as e:
                if (
                    isinstance(e, DownloadError) or isinstance(e, ValidationError)
                ) and "order status could not be checked" in e.args[0]:
                    raise NotFoundError(f"Item {item_id} does not exist. Please order it first") from e
                raise NotFoundError(e) from e
    
        zarr_asset_name = next(
            (name for name in product.assets if (name.endswith("zarr") and asset_name != "downloadLink")), None
        )
        if zarr_asset_name:
            asset_values = product.assets[zarr_asset_name]
            base_url = asset_values["href"]
            target_url = f"{base_url.rstrip('/')}/{file_path.lstrip('/')}"
    
>           r = product.request_asset(url=target_url)
                ^^^^^^^^^^^^^^^^^^^^^
E           AttributeError: 'EOProduct' object has no attribute 'request_asset'

stac_fastapi/eodag/extensions/data_download.py:233: AttributeError