Skip to content

Do not allow exceptions in ManifestStore #683

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jul 18, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 5 additions & 10 deletions virtualizarr/manifests/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,7 @@
__all__ = ["ManifestStore"]


_ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = (
FileNotFoundError,
IsADirectoryError,
NotADirectoryError,
)
_ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = ()


@dataclass
Expand Down Expand Up @@ -259,11 +255,10 @@ async def get(
f"Could not find a store to use for {path} in the store registry"
)

# Truncate path to match Obstore expectations
key = urlparse(path).path
path_in_store = urlparse(path).path
if hasattr(store, "prefix") and store.prefix:
# strip the prefix from key
key = key.removeprefix(str(store.prefix))
prefix = str(store.prefix).lstrip("/")
path_in_store = path_in_store.lstrip("/").removeprefix(prefix).lstrip("/")

# Transform the input byte range to account for the chunk location in the file
chunk_end_exclusive = offset + length
Expand All @@ -274,7 +269,7 @@ async def get(
# Actually get the bytes
try:
bytes = await store.get_range_async(
key,
path_in_store,
start=byte_range.start,
end=byte_range.end,
)
Expand Down
35 changes: 22 additions & 13 deletions virtualizarr/tests/test_manifests/test_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import TYPE_CHECKING

import numpy as np
import obstore as obs
import pytest
from obstore.store import MemoryStore
from zarr.abc.store import (
Expand Down Expand Up @@ -341,20 +342,28 @@ def test_single_group_to_dataset(
expected_loadable_variables,
):
marr1 = manifest_array(
shape=(3, 2, 5), chunks=(1, 2, 1), dimension_names=["x", "y", "t"]
shape=(3, 2, 5),
chunks=(1, 2, 1),
dimension_names=["x", "y", "t"],
codecs=None,
)
marr2 = manifest_array(shape=(3, 2), chunks=(1, 2), dimension_names=["x", "y"])
marr3 = manifest_array(shape=(5,), chunks=(5,), dimension_names=["t"])

paths1 = list({v["path"] for v in marr1.manifest.values()})
paths2 = list({v["path"] for v in marr2.manifest.values()})
paths3 = list({v["path"] for v in marr2.manifest.values()})
unique_paths = list(set(paths1 + paths2 + paths3))
stores = {}
for path in unique_paths:
store = MemoryStore()
stores[get_store_prefix(path)] = store
store_registry = ObjectStoreRegistry(stores=stores)
marr2 = manifest_array(
shape=(3, 2), chunks=(1, 2), dimension_names=["x", "y"], codecs=None
)
marr3 = manifest_array(
shape=(5,), chunks=(5,), dimension_names=["t"], codecs=None
)

store = MemoryStore()
for marr in [marr1, marr2, marr3]:
unique_paths = list({v["path"] for v in marr.manifest.values()})
for path in unique_paths:
obs.put(
store,
path.split("/")[-1],
np.ones(marr.chunks, dtype=marr.dtype).tobytes(),
)
store_registry = ObjectStoreRegistry({"file://": store})

manifest_group = ManifestGroup(
arrays={
Expand Down
4 changes: 2 additions & 2 deletions virtualizarr/tests/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@


def obstore_local(file_url: str) -> ObjectStore:
path = Path(file_url)
store = LocalStore(prefix=path.parent)
path = Path(file_url).resolve()
store = LocalStore(prefix=str(path.parent))
return store


Expand Down
Loading