From 13c059d4b60247909f66a431ec0722c70282404d Mon Sep 17 00:00:00 2001 From: opencode-contributor Date: Fri, 29 May 2026 10:40:06 +0500 Subject: [PATCH] tools/lib: add auto_camera_source for multi-source camera file resolution Implements the TODO at logreader.py:149 ('this should apply to camera files as well') by introducing auto_camera_source(), the camera equivalent of the existing auto_source() for log files. - New auto_camera_source() function resolves fcamera/dcamera/ecamera/ qcamera files across multiple sources with fallback, matching the pattern used for rlog/qlog resolution - Extended comma_api_source() in file_sources.py to serve all four camera file types through the Route API - Added ValueError guard for unknown file types - 12 new tests covering single/multi-segment resolution, source fallback, error recovery, full-route parsing, and comma_api_source integration --- tools/lib/file_sources.py | 12 ++- tools/lib/logreader.py | 24 +++++- tools/lib/tests/test_logreader.py | 131 +++++++++++++++++++++++++++++- 3 files changed, 162 insertions(+), 5 deletions(-) diff --git a/tools/lib/file_sources.py b/tools/lib/file_sources.py index cb7bf15114e273..c7cd12fdf71b46 100755 --- a/tools/lib/file_sources.py +++ b/tools/lib/file_sources.py @@ -18,8 +18,18 @@ def comma_api_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames) -> d # comma api will have already checked if the file exists if fns == FileName.RLOG: return {seg: route.log_paths()[seg] for seg in seg_idxs if route.log_paths()[seg] is not None} - else: + elif fns == FileName.QLOG: return {seg: route.qlog_paths()[seg] for seg in seg_idxs if route.qlog_paths()[seg] is not None} + elif fns == FileName.FCAMERA: + return {seg: route.camera_paths()[seg] for seg in seg_idxs if route.camera_paths()[seg] is not None} + elif fns == FileName.DCAMERA: + return {seg: route.dcamera_paths()[seg] for seg in seg_idxs if route.dcamera_paths()[seg] is not None} + elif fns == FileName.ECAMERA: + return {seg: route.ecamera_paths()[seg] for seg in seg_idxs if route.ecamera_paths()[seg] is not None} + elif fns == FileName.QCAMERA: + return {seg: route.qcamera_paths()[seg] for seg in seg_idxs if route.qcamera_paths()[seg] is not None} + else: + raise ValueError(f"Unknown file type: {fns}") def internal_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames, endpoint_url: str = DATA_ENDPOINT) -> dict[int, str]: diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 9696c8524d104d..44c94b93a69615 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -18,7 +18,7 @@ from cereal import log as capnp_log from openpilot.common.swaglog import cloudlog from openpilot.tools.lib.filereader import FileReader -from openpilot.tools.lib.file_sources import comma_api_source, internal_source, openpilotci_source, comma_car_segments_source, Source +from openpilot.tools.lib.file_sources import comma_api_source, internal_source, openpilotci_source, comma_car_segments_source, Source, FileNames from openpilot.tools.lib.route import SegmentRange, FileName from openpilot.tools.lib.log_time_series import msgs_to_time_series @@ -146,7 +146,27 @@ def direct_source(file_or_url: str) -> list[str]: return [file_or_url] -# TODO this should apply to camera files as well +def auto_camera_source(identifier: str, sources: list[Source], camera_type: FileNames = FileName.FCAMERA) -> list[str]: + sr = SegmentRange(identifier) + needed_seg_idxs = sr.seg_idxs + + valid_files: dict[int, str] = {} + exceptions = {} + for source in sources: + try: + files = source(sr, needed_seg_idxs, camera_type) + valid_files |= files + needed_seg_idxs = [idx for idx in needed_seg_idxs if idx not in valid_files] + if len(needed_seg_idxs) == 0: + return list(valid_files.values()) + except Exception as e: + exceptions[source.__name__] = e + + missing = len(needed_seg_idxs) + raise LogsUnavailable(f"{missing}/{len(sr.seg_idxs)} camera files were not found for {sr.route_name}, " + + f"camera_type={camera_type}\n\n" + + "Exceptions for sources:\n - " + "\n - ".join([f"{k}: {repr(v)}" for k, v in exceptions.items()])) + def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode) -> list[str]: exceptions = {} diff --git a/tools/lib/tests/test_logreader.py b/tools/lib/tests/test_logreader.py index 123f142383a308..4274a2adec3f02 100644 --- a/tools/lib/tests/test_logreader.py +++ b/tools/lib/tests/test_logreader.py @@ -10,9 +10,9 @@ from openpilot.common.parameterized import parameterized from cereal import log as capnp_log -from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, parse_indirect, ReadMode +from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, parse_indirect, ReadMode, auto_camera_source from openpilot.tools.lib.file_sources import comma_api_source, InternalUnavailableException -from openpilot.tools.lib.route import SegmentRange +from openpilot.tools.lib.route import SegmentRange, FileName from openpilot.tools.lib.url_file import URLFileException NUM_SEGS = 17 # number of segments in the test route @@ -261,3 +261,130 @@ def test_only_union_types(self): msgs = list(LogReader(qlog.name, only_union_types=True)) assert len(msgs) == num_msgs [m.which() for m in msgs] + + +TEST_ROUTE_CAM = "344c5c15b34f2d8a/2024-01-03--09-37-12" +CAMERA_FILE = "fcamera.hevc" + + +class TestCameraSource: + def test_auto_camera_source_no_source(self): + with pytest.raises(LogsUnavailable): + auto_camera_source(f"{TEST_ROUTE_CAM}/0", [], FileName.FCAMERA) + + def test_auto_camera_source_no_source_default_camera(self): + with pytest.raises(LogsUnavailable): + auto_camera_source(f"{TEST_ROUTE_CAM}/0", []) + + def test_auto_camera_source_found(self, mocker): + mock_source = mocker.Mock() + mock_source.__name__ = "mock_source" + mock_source.return_value = {0: "http://fake/fcamera.hevc"} + + result = auto_camera_source(f"{TEST_ROUTE_CAM}/0", [mock_source], FileName.FCAMERA) + assert len(result) == 1 + assert result[0] == "http://fake/fcamera.hevc" + + def test_auto_camera_source_multiple_segments(self, mocker): + mock_source = mocker.Mock() + mock_source.__name__ = "mock_source" + mock_source.return_value = {0: "http://fake/0/fcamera.hevc", 1: "http://fake/1/fcamera.hevc"} + + result = auto_camera_source(f"{TEST_ROUTE_CAM}/0:2", [mock_source], FileName.FCAMERA) + assert len(result) == 2 + + def test_auto_camera_source_source_fallback(self, mocker): + source_a = mocker.Mock() + source_a.__name__ = "source_a" + source_a.return_value = {} + + source_b = mocker.Mock() + source_b.__name__ = "source_b" + source_b.return_value = {0: "http://fake/fcamera.hevc"} + + result = auto_camera_source(f"{TEST_ROUTE_CAM}/0", [source_a, source_b], FileName.FCAMERA) + assert len(result) == 1 + assert source_a.called + assert source_b.called + + def test_auto_camera_source_partial_fallback(self, mocker): + source_a = mocker.Mock() + source_a.__name__ = "source_a" + source_a.return_value = {0: "http://fake/0/fcamera.hevc"} + + source_b = mocker.Mock() + source_b.__name__ = "source_b" + source_b.return_value = {1: "http://fake/1/fcamera.hevc"} + + result = auto_camera_source(f"{TEST_ROUTE_CAM}/0:2", [source_a, source_b], FileName.FCAMERA) + assert len(result) == 2 + + def test_auto_camera_source_error_then_success(self, mocker): + source_a = mocker.Mock() + source_a.__name__ = "source_a" + source_a.side_effect = Exception("network error") + + source_b = mocker.Mock() + source_b.__name__ = "source_b" + source_b.return_value = {0: "http://fake/fcamera.hevc"} + + result = auto_camera_source(f"{TEST_ROUTE_CAM}/0", [source_a, source_b], FileName.FCAMERA) + assert len(result) == 1 + + def test_auto_camera_source_all_fail(self, mocker): + source_a = mocker.Mock() + source_a.__name__ = "source_a" + source_a.side_effect = Exception("network error") + + source_b = mocker.Mock() + source_b.__name__ = "source_b" + source_b.return_value = {} + + with pytest.raises(LogsUnavailable): + auto_camera_source(f"{TEST_ROUTE_CAM}/0:2", [source_a, source_b], FileName.FCAMERA) + + def test_auto_camera_source_route_with_slash(self, mocker): + mocker.patch("openpilot.tools.lib.route.get_max_seg_number_cached", return_value=16) + mock_source = mocker.Mock() + mock_source.__name__ = "mock_source" + mock_source.side_effect = lambda sr, seg_idxs, fns: {seg: f"http://fake/{seg}/fcamera.hevc" for seg in seg_idxs} + + result = auto_camera_source(f"{TEST_ROUTE_CAM.replace('/', '|')}", [mock_source], FileName.FCAMERA) + assert len(result) == 17 + + def test_auto_camera_source_route_with_segment(self, mocker): + mock_source = mocker.Mock() + mock_source.__name__ = "mock_source" + mock_source.side_effect = lambda sr, seg_idxs, fns: {seg: f"http://fake/{seg}/fcamera.hevc" for seg in seg_idxs} + + result = auto_camera_source(f"{TEST_ROUTE_CAM}--5", [mock_source], FileName.FCAMERA) + assert len(result) == 1 + + def test_comma_api_source_extended(self, mocker): + """Verify comma_api_source can serve camera paths""" + mock_route = mocker.patch("openpilot.tools.lib.file_sources.Route") + mock_route_instance = mock_route.return_value + mock_route_instance.log_paths.return_value = ["fake_rlog"] + mock_route_instance.qlog_paths.return_value = ["fake_qlog"] + mock_route_instance.camera_paths.return_value = ["fake_fcamera"] + mock_route_instance.dcamera_paths.return_value = ["fake_dcamera"] + mock_route_instance.ecamera_paths.return_value = ["fake_ecamera"] + mock_route_instance.qcamera_paths.return_value = ["fake_qcamera"] + + sr = SegmentRange(f"{TEST_ROUTE_CAM}/0") + for fns, expected in [ + (FileName.RLOG, "fake_rlog"), + (FileName.QLOG, "fake_qlog"), + (FileName.FCAMERA, "fake_fcamera"), + (FileName.DCAMERA, "fake_dcamera"), + (FileName.ECAMERA, "fake_ecamera"), + (FileName.QCAMERA, "fake_qcamera"), + ]: + result = comma_api_source(sr, [0], fns) + assert result[0] == expected, f"Failed for {fns}" + + def test_comma_api_source_unknown_type(self, mocker): + mocker.patch("openpilot.tools.lib.file_sources.Route") + sr = SegmentRange(f"{TEST_ROUTE_CAM}/0") + with pytest.raises(ValueError, match="Unknown file type"): + comma_api_source(sr, [0], ("unknown.ext",))