Skip to content

Commit bd4be5e

Browse files
gh-126317: Simplify pickle code by using itertools.batched() (GH-126323)
1 parent 10eeec2 commit bd4be5e

File tree

1 file changed

+22
-39
lines changed

1 file changed

+22
-39
lines changed

Lib/pickle.py

+22-39
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
from types import FunctionType
2727
from copyreg import dispatch_table
2828
from copyreg import _extension_registry, _inverted_registry, _extension_cache
29-
from itertools import islice
29+
from itertools import batched
3030
from functools import partial
3131
import sys
3232
from sys import maxsize
@@ -1033,31 +1033,26 @@ def _batch_appends(self, items, obj):
10331033
write(APPEND)
10341034
return
10351035

1036-
it = iter(items)
10371036
start = 0
1038-
while True:
1039-
tmp = list(islice(it, self._BATCHSIZE))
1040-
n = len(tmp)
1041-
if n > 1:
1037+
for batch in batched(items, self._BATCHSIZE):
1038+
batch_len = len(batch)
1039+
if batch_len != 1:
10421040
write(MARK)
1043-
for i, x in enumerate(tmp, start):
1041+
for i, x in enumerate(batch, start):
10441042
try:
10451043
save(x)
10461044
except BaseException as exc:
10471045
exc.add_note(f'when serializing {_T(obj)} item {i}')
10481046
raise
10491047
write(APPENDS)
1050-
elif n:
1048+
else:
10511049
try:
1052-
save(tmp[0])
1050+
save(batch[0])
10531051
except BaseException as exc:
10541052
exc.add_note(f'when serializing {_T(obj)} item {start}')
10551053
raise
10561054
write(APPEND)
1057-
# else tmp is empty, and we're done
1058-
if n < self._BATCHSIZE:
1059-
return
1060-
start += n
1055+
start += batch_len
10611056

10621057
def save_dict(self, obj):
10631058
if self.bin:
@@ -1086,32 +1081,26 @@ def _batch_setitems(self, items, obj):
10861081
write(SETITEM)
10871082
return
10881083

1089-
it = iter(items)
1090-
while True:
1091-
tmp = list(islice(it, self._BATCHSIZE))
1092-
n = len(tmp)
1093-
if n > 1:
1084+
for batch in batched(items, self._BATCHSIZE):
1085+
if len(batch) != 1:
10941086
write(MARK)
1095-
for k, v in tmp:
1087+
for k, v in batch:
10961088
save(k)
10971089
try:
10981090
save(v)
10991091
except BaseException as exc:
11001092
exc.add_note(f'when serializing {_T(obj)} item {k!r}')
11011093
raise
11021094
write(SETITEMS)
1103-
elif n:
1104-
k, v = tmp[0]
1095+
else:
1096+
k, v = batch[0]
11051097
save(k)
11061098
try:
11071099
save(v)
11081100
except BaseException as exc:
11091101
exc.add_note(f'when serializing {_T(obj)} item {k!r}')
11101102
raise
11111103
write(SETITEM)
1112-
# else tmp is empty, and we're done
1113-
if n < self._BATCHSIZE:
1114-
return
11151104

11161105
def save_set(self, obj):
11171106
save = self.save
@@ -1124,21 +1113,15 @@ def save_set(self, obj):
11241113
write(EMPTY_SET)
11251114
self.memoize(obj)
11261115

1127-
it = iter(obj)
1128-
while True:
1129-
batch = list(islice(it, self._BATCHSIZE))
1130-
n = len(batch)
1131-
if n > 0:
1132-
write(MARK)
1133-
try:
1134-
for item in batch:
1135-
save(item)
1136-
except BaseException as exc:
1137-
exc.add_note(f'when serializing {_T(obj)} element')
1138-
raise
1139-
write(ADDITEMS)
1140-
if n < self._BATCHSIZE:
1141-
return
1116+
for batch in batched(obj, self._BATCHSIZE):
1117+
write(MARK)
1118+
try:
1119+
for item in batch:
1120+
save(item)
1121+
except BaseException as exc:
1122+
exc.add_note(f'when serializing {_T(obj)} element')
1123+
raise
1124+
write(ADDITEMS)
11421125
dispatch[set] = save_set
11431126

11441127
def save_frozenset(self, obj):

0 commit comments

Comments
 (0)