Skip to content

Commit 650a285

Browse files
committed
rebatch arrow iterables before formatted iterable
1 parent 67907b1 commit 650a285

File tree

1 file changed

+3
-1
lines changed

1 file changed

+3
-1
lines changed

src/datasets/iterable_dataset.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -2194,7 +2194,9 @@ def _prepare_ex_iterable_for_iteration(
21942194
self, batch_size: int = 1, drop_last_batch: bool = False
21952195
) -> _BaseExamplesIterable:
21962196
ex_iterable = self._ex_iterable
2197-
if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table):
2197+
if (self._formatting or (self.features and ex_iterable.features != self.features)) and (
2198+
ex_iterable.iter_arrow or self._formatting.is_table
2199+
):
21982200
ex_iterable = RebatchedArrowExamplesIterable(
21992201
ex_iterable, batch_size=batch_size, drop_last_batch=drop_last_batch
22002202
)

0 commit comments

Comments
 (0)