llm internal error.log

---------------------------------------------------------------------------
SystemException                           Traceback (most recent call last)
SystemException: 

The above exception was the direct cause of the following exception:

RayTaskError(TypeError)                   Traceback (most recent call last)
/home/ray/default/e2e-audio/e2e_audio/curation.ipynb Cell 16 line 1
----> 1 print(ds.take(1))

File ~/anaconda3/lib/python3.12/site-packages/ray/data/dataset.py:2909, in Dataset.take(self, limit)
   2906 output = []
   2908 limited_ds = self.limit(limit)
-> 2909 for row in limited_ds.iter_rows():
   2910     output.append(row)
   2911     if len(output) >= limit:

File ~/anaconda3/lib/python3.12/site-packages/ray/data/iterator.py:215, in DataIterator.iter_rows.<locals>._wrapped_iterator()
    214 def _wrapped_iterator():
--> 215     for batch in batch_iterable:
    216         batch = BlockAccessor.for_block(BlockAccessor.batch_to_block(batch))
    217         for row in batch.iter_rows(public_row_format=True):

File ~/anaconda3/lib/python3.12/site-packages/ray/data/iterator.py:155, in DataIterator.iter_batches.<locals>._create_iterator()
    146 time_start = time.perf_counter()
    147 # Iterate through the dataset from the start each time
    148 # _iterator_gen is called.
    149 # This allows multiple iterations of the dataset without
    150 # needing to explicitly call `iter_batches()` multiple times.
    151 (
    152     ref_bundles_iterator,
    153     stats,
    154     blocks_owned_by_consumer,
--> 155 ) = self._to_ref_bundle_iterator()
    157 iterator = iter(
    158     iter_batches(
    159         ref_bundles_iterator,
   (...)
    170     )
    171 )
    173 dataset_tag = self._get_dataset_tag()

File ~/anaconda3/lib/python3.12/site-packages/ray/data/_internal/iterator/iterator_impl.py:27, in DataIteratorImpl._to_ref_bundle_iterator(self)
     24 def _to_ref_bundle_iterator(
     25     self,
     26 ) -> Tuple[Iterator[RefBundle], Optional[DatasetStats], bool]:
---> 27     ref_bundles_iterator, stats = self._base_dataset._execute_to_iterator()
     28     return ref_bundles_iterator, stats, False

File ~/anaconda3/lib/python3.12/site-packages/ray/data/dataset.py:5926, in Dataset._execute_to_iterator(self)
   5925 def _execute_to_iterator(self) -> Tuple[Iterator[RefBundle], DatasetStats]:
-> 5926     bundle_iter, stats, executor = self._plan.execute_to_iterator()
   5927     # Capture current executor to be able to clean it up properly, once
   5928     # dataset is garbage-collected
   5929     self._current_executor = executor

File ~/anaconda3/lib/python3.12/site-packages/ray/data/exceptions.py:89, in omit_traceback_stdout.<locals>.handle_trace(*args, **kwargs)
     87     raise e.with_traceback(None)
     88 else:
---> 89     raise e.with_traceback(None) from SystemException()

RayTaskError(TypeError): ray::MapBatches(vLLMEngineStageUDF).submit() (pid=16220, ip=10.0.149.32, actor_id=cc3b8fa84212b111fa122d7306000000, repr=MapWorker(MapBatches(vLLMEngineStageUDF)))
    yield from _map_task(
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/data/_internal/execution/operators/map_operator.py", line 541, in _map_task
    for b_out in map_transformer.apply_transform(iter(blocks), ctx):
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/data/_internal/execution/operators/map_transformer.py", line 532, in __call__
    for data in iter:
                ^^^^
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/data/_internal/execution/operators/map_transformer.py", line 211, in _udf_timed_iter
    output = next(input)
             ^^^^^^^^^^^
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/data/_internal/execution/operators/map_transformer.py", line 327, in __call__
    yield from self._batch_fn(input, ctx)
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/data/_internal/planner/plan_udf_map_op.py", line 491, in transform_fn
    raise out_item
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/data/_internal/planner/plan_udf_map_op.py", line 455, in process_item
    async for output_item in output_item_iterator:
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/llm/_internal/batch/stages/base.py", line 171, in __call__
    async for output in self.udf(inputs):
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 552, in udf
    request, output = await resp
                      ^^^^^^^^^^
  File "/home/ray/anaconda3/lib/python3.12/asyncio/tasks.py", line 631, in _wait_for_one
    return f.result()  # May raise f.exception().
           ^^^^^^^^^^
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 335, in generate_async
    request = await self._prepare_llm_request(row)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ray/anaconda3/lib/python3.12/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 303, in _prepare_llm_request
    params = vllm.SamplingParams(
             ^^^^^^^^^^^^^^^^^^^^
TypeError: Unexpected keyword argument 'guided_choice'