Skip to content

Commit f927204

Browse files
authored
gh-129005: Align FileIO.readall() allocation (#129458)
Both now use a pre-allocated buffer of length `bufsize`, fill it using a readinto(), and have matching "expand buffer" logic. On my machine this takes: `./python -m test -M8g -uall test_largefile -m test_large_read -v` from ~3.7 seconds to ~3.4 seconds.
1 parent 6c63afc commit f927204

File tree

2 files changed

+20
-9
lines changed

2 files changed

+20
-9
lines changed

Lib/_pyio.py

+18-9
Original file line numberDiff line numberDiff line change
@@ -1674,22 +1674,31 @@ def readall(self):
16741674
except OSError:
16751675
pass
16761676

1677-
result = bytearray()
1677+
result = bytearray(bufsize)
1678+
bytes_read = 0
16781679
while True:
1679-
if len(result) >= bufsize:
1680-
bufsize = len(result)
1681-
bufsize += max(bufsize, DEFAULT_BUFFER_SIZE)
1682-
n = bufsize - len(result)
1680+
if bytes_read >= bufsize:
1681+
# Parallels _io/fileio.c new_buffersize
1682+
if bufsize > 65536:
1683+
addend = bufsize >> 3
1684+
else:
1685+
addend = bufsize + 256
1686+
if addend < DEFAULT_BUFFER_SIZE:
1687+
addend = DEFAULT_BUFFER_SIZE
1688+
bufsize += addend
1689+
result[bytes_read:bufsize] = b'\0'
1690+
assert bufsize - bytes_read > 0, "Should always try and read at least one byte"
16831691
try:
1684-
chunk = os.read(self._fd, n)
1692+
n = os.readinto(self._fd, memoryview(result)[bytes_read:])
16851693
except BlockingIOError:
1686-
if result:
1694+
if bytes_read > 0:
16871695
break
16881696
return None
1689-
if not chunk: # reached the end of the file
1697+
if n == 0: # reached the end of the file
16901698
break
1691-
result += chunk
1699+
bytes_read += n
16921700

1701+
del result[bytes_read:]
16931702
return bytes(result)
16941703

16951704
def readinto(self, buffer):
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
``_pyio.FileIO.readall()`` now allocates, resizes, and fills a data buffer using
2+
the same algorithm ``_io.FileIO.readall()`` uses.

0 commit comments

Comments
 (0)