You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Describe the bug
A clear and concise description of what the bug is.
To Reproduce
To help us reproducing this bug, please provide information below:
Your Python version
The version of Mars you use
Versions of crucial packages, such as numpy, scipy and pandas
Full stack of the error.
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/brain_predict/lib/sql_laundry.py", line 140, in download_marsdata_from_odps
result_df = mars_o.to_mars_dataframe(tmp_table_name, chunk_size=chunk_size,).execute()
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/core/entity/tileables.py", line 462, in execute
result = self.data.execute(session=session, **kw)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/core/entity/executable.py", line 142, in execute
return execute(self, session=session, **kw)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/deploy/oscar/session.py", line 1864, in execute
**kwargs,
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/deploy/oscar/session.py", line 1653, in execute
timeout=self._isolated_session.timeout
File "/usr/local/python3/lib/python3.7/concurrent/futures/_base.py", line 435, in result
return self.__get_result()
File "/usr/local/python3/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/deploy/oscar/session.py", line 1838, in _execute
await execution_info
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/deploy/oscar/session.py", line 107, in wait
return await self._aio_task
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/deploy/oscar/session.py", line 957, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/services/task/supervisor/processor.py", line 359, in run
await self._process_stage_chunk_graph(*stage_args)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/services/task/supervisor/processor.py", line 241, in _process_stage_chunk_graph
stage_id, subtask_graph, chunk_graph, tile_context
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/services/task/execution/mars/executor.py", line 250, in execute_subtask_graph
return await stage_processor.run()
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/services/task/execution/mars/stage.py", line 252, in run
return await self._get_stage_result()
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/services/task/execution/mars/stage.py", line 136, in _get_stage_result
await self._update_result_meta(execution_chunk_results)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/services/task/execution/mars/stage.py", line 302, in _update_result_meta
worker_metas = await asyncio.gather(*coros)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/oscar/batch.py", line 153, in _async_batch
return await self.batch_func(args_list, kwargs_list)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/services/meta/api/oscar.py", line 182, in batch_get_chunk_meta
return await self._meta_store.get_meta.batch(*get_chunk_metas)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/oscar/backends/context.py", line 196, in send
return self._process_result_message(result)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/oscar/backends/context.py", line 76, in _process_result_message
raise message.as_instanceof_cause()
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/oscar/backends/pool.py", line 677, in send
result = await self._run_coro(message.message_id, coro)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/oscar/backends/pool.py", line 370, in _run_coro
return await coro
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/oscar/api.py", line 120, in __on_receive__
return await super().__on_receive__(message)
File "mars/oscar/core.pyx", line 510, in __on_receive__
File "mars/oscar/core.pyx", line 490, in mars.oscar.core._BaseActor.__on_receive__
File "mars/oscar/core.pyx", line 491, in mars.oscar.core._BaseActor.__on_receive__
File "mars/oscar/core.pyx", line 499, in mars.oscar.core._BaseActor.__on_receive__
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/oscar/batch.py", line 153, in _async_batch
return await self.batch_func(args_list, kwargs_list)
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/services/meta/store/dictionary.py", line 101, in batch_get_meta
metas.append(self._get_meta(*args, **kwargs))
File "/home/admin/ray-pack/tmp/job/4c060080/pyenv/lib/python3.7/site-packages/mars/services/meta/store/dictionary.py", line 80, in _get_meta
meta = self._store[object_id]
KeyError: [address=ray://ray-cluster-1656468237/4/0, pid=483612] 'f955907f4409be5cb344abc1b1cec01b_0'
Minimized code to reproduce the error.
Expected behavior
A clear and concise description of what you expected to happen.
Additional context
Add any other context about the problem here.
The text was updated successfully, but these errors were encountered:
Describe the bug
A clear and concise description of what the bug is.
To Reproduce
To help us reproducing this bug, please provide information below:
Expected behavior
A clear and concise description of what you expected to happen.
Additional context
Add any other context about the problem here.
The text was updated successfully, but these errors were encountered: