Skip to content

Commit 0ee8476

Browse files
committed
gh-159503: Avoid copying the unsent remainder of a bytes write in asyncio selector transports
When a large bytes write is queued behind pending data, every partial send re-queued the leftover as b[nbytes:], copying the rest of the object each time. Keep the remainder as a memoryview in _adjust_leftover_buffer() and _write_send(), as write() already does on its first-send path.
1 parent fdec4df commit 0ee8476

3 files changed

Lines changed: 33 additions & 2 deletions

File tree

‎Lib/asyncio/selector_events.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,7 +1143,7 @@ def _adjust_leftover_buffer(self, nbytes: int) -> None:
11431143
if b_len <= nbytes:
11441144
nbytes -= b_len
11451145
else:
1146-
buffer.appendleft(b[nbytes:])
1146+
buffer.appendleft(memoryview(b)[nbytes:])
11471147
break
11481148

11491149
def _write_send(self):
@@ -1155,7 +1155,7 @@ def _write_send(self):
11551155
n = self._sock.send(buffer)
11561156
if n != len(buffer):
11571157
# Not all data was written
1158-
self._buffer.appendleft(buffer[n:])
1158+
self._buffer.appendleft(memoryview(buffer)[n:])
11591159
self._buffer_size -= n
11601160
except (BlockingIOError, InterruptedError):
11611161
self._buffer.appendleft(buffer)

‎Lib/test/test_asyncio/test_selector_events.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -972,6 +972,22 @@ def test_write_sendmsg_partial(self):
972972
self.assertTrue(self.loop.writers)
973973
self.assertEqual(list_to_buffer([b'ta']), transport._buffer)
974974

975+
@unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg')
976+
def test_write_sendmsg_partial_bytes_no_copy(self):
977+
# gh-159503: the unsent remainder of a bytes object must not be
978+
# copied, otherwise a large write costs O(n**2).
979+
data = b'data'
980+
self.sock.sendmsg = mock.Mock()
981+
self.sock.sendmsg.return_value = 2
982+
983+
transport = self.socket_transport(sendmsg=True)
984+
transport._buffer.append(data)
985+
self.loop._add_writer(7, transport._write_ready)
986+
transport._write_ready()
987+
self.assertEqual(list_to_buffer([b'ta']), transport._buffer)
988+
self.assertIsInstance(transport._buffer[0], memoryview)
989+
self.assertIs(transport._buffer[0].obj, data)
990+
975991
@unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg')
976992
def test_write_sendmsg_half_buffer(self):
977993
data = [memoryview(b'data1'), memoryview(b'data2')]
@@ -1081,6 +1097,20 @@ def test_write_ready_partial(self):
10811097
self.loop.assert_writer(7, transport._write_ready)
10821098
self.assertEqual(list_to_buffer([b'ta']), transport._buffer)
10831099

1100+
def test_write_ready_partial_bytes_no_copy(self):
1101+
# gh-159503: the unsent remainder of a bytes object must not be
1102+
# copied, otherwise a large write costs O(n**2).
1103+
data = b'data'
1104+
self.sock.send.return_value = 2
1105+
1106+
transport = self.socket_transport()
1107+
transport._buffer.append(data)
1108+
self.loop._add_writer(7, transport._write_ready)
1109+
transport._write_ready()
1110+
self.assertEqual(list_to_buffer([b'ta']), transport._buffer)
1111+
self.assertIsInstance(transport._buffer[0], memoryview)
1112+
self.assertIs(transport._buffer[0].obj, data)
1113+
10841114
def test_write_ready_partial_none(self):
10851115
data = b'data'
10861116
self.sock.send.return_value = 0
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix quadratic copying in :mod:`asyncio` selector socket transports when a large :class:`bytes` write is only partly sent while other data is buffered. The unsent remainder is now kept as a :class:`memoryview` instead of being copied on every partial send.

0 commit comments

Comments
 (0)