When testing the eiger on i15-1, I continuously ran into an issue where after 3 or 4 successful scans, the device would time out on disarm. Trying to do another scan after this point wouldn't work. Restarting the IOC by deleting the resource in argocd fixed it until it occurred again.
Traceback (most recent call last):
File "/python/cpython-3.11.15-linux-x86_64-gnu/lib/python3.11/asyncio/tasks.py", line 500, in wait_for
return fut.result()
^^^^^^^^^^^^
File "/workspace/ophyd-async/src/ophyd_async/epics/core/_p4p.py", line 414, in put
await context().put(self.write_pv, {"value": write_value}, wait=wait)
File "/app/.venv/lib/python3.11/site-packages/p4p/client/asyncio.py", line 207, in put
return (await self._put_one(name, values, request=request, get=get))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/p4p/client/asyncio.py", line 236, in _put_one
value = await F
^^^^^^^
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/workspace/ophyd-async/src/ophyd_async/core/_signal.py", line 41, in _wait_for
return await asyncio.wait_for(coro, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/python/cpython-3.11.15-linux-x86_64-gnu/lib/python3.11/asyncio/tasks.py", line 502, in wait_for
raise exceptions.TimeoutError() from exc
TimeoutError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/workspace/ophyd-async/src/ophyd_async/core/_status.py", line 37, in wait_with_error_message
await awaitable
File "/workspace/ophyd-async/src/ophyd_async/core/_detector.py", line 785, in unstage
await asyncio.gather(*coros)
File "/workspace/ophyd-async/src/ophyd_async/fastcs/eiger/_acquire_logic.py", line 23, in ensure_stopped
await self.detector.disarm.trigger()
File "/workspace/ophyd-async/src/ophyd_async/core/_status.py", line 37, in wait_with_error_message
await awaitable
File "/workspace/ophyd-async/src/ophyd_async/core/_signal.py", line 330, in trigger
await _wait_for(self._connector.backend.put(None), timeout, source)
File "/workspace/ophyd-async/src/ophyd_async/core/_signal.py", line 43, in _wait_for
raise TimeoutError(source) from exc
TimeoutError: pva://BL15J-EA-EIGER-01:Detector:Disarm
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/app/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1585, in _run
msg = self._plan_stack[-1].throw(stashed_exception or resp)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/crystallography-bluesky/src/crystallography_bluesky/i15_1/plans/take_eiger_data.py", line 138, in take_eiger_and_i0_data
yield from inner_run()
File "/app/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1279, in dec_inner
return (yield from plan)
^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 1252, in baseline_wrapper
return (yield from plan_mutator(plan, insert_baseline))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 96, in plan_mutator
msg = plan_stack[-1].throw(exception)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1279, in dec_inner
return (yield from plan)
^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 1013, in stage_wrapper
return (yield from finalize_wrapper(inner(), unstage_devices()))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 567, in finalize_wrapper
yield from ensure_generator(final_plan_instance)
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
inner_ret = yield msg
^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 2378, in _status_object_completed
raise FailedStatus(ret) from exc
bluesky.utils.FailedStatus: <AsyncStatus, device: fastcs_eiger, task: <coroutine object AsyncStatusBase.__init__.<locals>.wait_with_error_message at 0x7fe57a92ec40>, errored: TimeoutError('pva://BL15J-EA-EIGER-01:Detector:Disarm')>
2026-05-20 13:37:41,778 INFO bluesky.RE.state Change state on <bluesky.run_engine.RunEngine object at 0x7fe581795f50> from 'running' -> 'idle'
2026-05-20 13:37:41,779 ERROR blueapi.worker.task_worker Task failed
2026-05-20 13:37:41,780 ERROR blueapi.worker.task_worker <AsyncStatus, device: fastcs_eiger, task: <coroutine object AsyncStatusBase.__init__.<locals>.wait_with_error_message at 0x7fe57a92ec40>, errored: TimeoutError('pva://BL15J-EA-EIGER-01:Detector:Disarm')>
Traceback (most recent call last):
File "/python/cpython-3.11.15-linux-x86_64-gnu/lib/python3.11/asyncio/tasks.py", line 500, in wait_for
return fut.result()
^^^^^^^^^^^^
File "/workspace/ophyd-async/src/ophyd_async/epics/core/_p4p.py", line 414, in put
await context().put(self.write_pv, {"value": write_value}, wait=wait)
File "/app/.venv/lib/python3.11/site-packages/p4p/client/asyncio.py", line 207, in put
return (await self._put_one(name, values, request=request, get=get))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/p4p/client/asyncio.py", line 236, in _put_one
value = await F
^^^^^^^
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/workspace/ophyd-async/src/ophyd_async/core/_signal.py", line 41, in _wait_for
return await asyncio.wait_for(coro, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/python/cpython-3.11.15-linux-x86_64-gnu/lib/python3.11/asyncio/tasks.py", line 502, in wait_for
raise exceptions.TimeoutError() from exc
TimeoutError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/workspace/ophyd-async/src/ophyd_async/core/_status.py", line 37, in wait_with_error_message
await awaitable
File "/workspace/ophyd-async/src/ophyd_async/core/_detector.py", line 785, in unstage
await asyncio.gather(*coros)
File "/workspace/ophyd-async/src/ophyd_async/fastcs/eiger/_acquire_logic.py", line 23, in ensure_stopped
await self.detector.disarm.trigger()
File "/workspace/ophyd-async/src/ophyd_async/core/_status.py", line 37, in wait_with_error_message
await awaitable
File "/workspace/ophyd-async/src/ophyd_async/core/_signal.py", line 330, in trigger
await _wait_for(self._connector.backend.put(None), timeout, source)
File "/workspace/ophyd-async/src/ophyd_async/core/_signal.py", line 43, in _wait_for
raise TimeoutError(source) from exc
TimeoutError: pva://BL15J-EA-EIGER-01:Detector:Disarm
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/app/.venv/lib/python3.11/site-packages/blueapi/worker/task_worker.py", line 441, in process_task
result = self._current.task.do_task(self._ctx)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/blueapi/worker/task.py", line 41, in do_task
result = ctx.run_engine(func(**prepared_params))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 976, in __call__
plan_return = self._resume_task(init_func=_build_task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1121, in _resume_task
raise exc
File "/app/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1756, in _run
raise err
File "/app/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1585, in _run
msg = self._plan_stack[-1].throw(stashed_exception or resp)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/crystallography-bluesky/src/crystallography_bluesky/i15_1/plans/take_eiger_data.py", line 138, in take_eiger_and_i0_data
yield from inner_run()
File "/app/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1279, in dec_inner
return (yield from plan)
^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 1252, in baseline_wrapper
return (yield from plan_mutator(plan, insert_baseline))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 96, in plan_mutator
msg = plan_stack[-1].throw(exception)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1279, in dec_inner
return (yield from plan)
^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 1013, in stage_wrapper
return (yield from finalize_wrapper(inner(), unstage_devices()))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 567, in finalize_wrapper
yield from ensure_generator(final_plan_instance)
File "/app/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
inner_ret = yield msg
^^^^^^^^^
File "/app/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 2378, in _status_object_completed
raise FailedStatus(ret) from exc
bluesky.utils.FailedStatus: <AsyncStatus, device: fastcs_eiger, task: <coroutine object AsyncStatusBase.__init__.<locals>.wait_with_error_message at 0x7fe57a92ec40>, errored: TimeoutError('pva://BL15J-EA-EIGER-01:Detector:Disarm')>
When testing the eiger on i15-1, I continuously ran into an issue where after 3 or 4 successful scans, the device would time out on
disarm. Trying to do another scan after this point wouldn't work. Restarting the IOC by deleting the resource in argocd fixed it until it occurred again.The reason for this should be investigated and fixed
Stack trace:
Acceptance Criteria