Views
No views yet
Traceback (most recent call last):
File "/usr/local/lib/python3.11/asyncio/tasks.py", line 500, in wait_for
return fut.result()
^^^^^^^^^^^^
File "/root/phosphobot/am/gr00t.py", line 1436, in read_output
async for line in process.stdout:
File "/usr/local/lib/python3.11/asyncio/streams.py", line 765, in __anext__
val = await self.readline()
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/asyncio/streams.py", line 566, in readline
line = await self.readuntil(sep)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/asyncio/streams.py", line 658, in readuntil
await self._wait_for_data('readuntil')
File "/usr/local/lib/python3.11/asyncio/streams.py", line 543, in _wait_for_data
await self._waiter
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/phosphobot/am/gr00t.py", line 1447, in _call_training_script
await asyncio.wait_for(read_output(), timeout=timeout_seconds)
File "/usr/local/lib/python3.11/asyncio/tasks.py", line 502, in wait_for
raise exceptions.TimeoutError() from exc
TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/root/src/helper.py", line 139, in train_gr00t_on_modal
trainer.train(
File "/root/phosphobot/am/gr00t.py", line 1250, in train
asyncio.run(
File "/usr/local/lib/python3.11/asyncio/runners.py", line 190, in run
return runner.run(main)
^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
File "/root/phosphobot/am/gr00t.py", line 1452, in _call_training_script
raise TimeoutError(
TimeoutError: Training process exceeded timeout of 3600 seconds. Please consider lowering the number of epochs and/or batch size.
1{
2 "validation_dataset_name": null,
3 "batch-size": 16,
4 "num-epochs": 40,
5 "save-steps": 3000,
6 "learning_rate": 0.0001,
7 "data_dir": "/tmp/outputs/data",
8 "validation_data_dir": "/tmp/outputs/validation_data",
9 "output_dir": "/tmp/outputs/train"
10}