Set PoolManager blocksize if urllib3 is v2 (boto#3612)

Author: hssyoo

.changes/next-release/enhancement-HTTPsession-50449.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "HTTP session",
+  "description": "Set blocksize from default 16KB to 128KB if urllib3 v2 is installed"
+}

botocore/httpsession.py

@@ -19,6 +19,7 @@
 )
 from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError
 from urllib3.exceptions import SSLError as URLLib3SSLError
+from urllib3.poolmanager import PoolKey
 from urllib3.util.retry import Retry
 from urllib3.util.ssl_ import (
     OP_NO_COMPRESSION,
@@ -81,6 +82,15 @@
 DEFAULT_TIMEOUT = 60
 MAX_POOL_CONNECTIONS = 10
 DEFAULT_CA_BUNDLE = os.path.join(os.path.dirname(__file__), 'cacert.pem')
+BUFFER_SIZE = None
+if hasattr(PoolKey, 'key_blocksize'):
+    # urllib3 2.0 implemented its own chunking logic and set
+    # a default blocksize of 16KB. This creates a noticeable
+    # performance bottleneck when transferring objects
+    # larger than 100MB. Based on experiments, a blocksize
+    # of 128KB significantly improves throughput before
+    # getting diminishing returns.
+    BUFFER_SIZE = 1024 * 128
 
 try:
     from certifi import where
@@ -343,6 +353,8 @@ def _get_pool_manager_kwargs(self, **extra_kwargs):
             'cert_file': self._cert_file,
             'key_file': self._key_file,
         }
+        if BUFFER_SIZE:
+            pool_manager_kwargs['blocksize'] = BUFFER_SIZE
         pool_manager_kwargs.update(**extra_kwargs)
         return pool_manager_kwargs
 

tests/unit/test_http_session.py

@@ -15,6 +15,7 @@
     ProxyConnectionError,
 )
 from botocore.httpsession import (
+    BUFFER_SIZE,
     ProxyConfiguration,
     URLLib3Session,
     get_cert_path,
@@ -156,6 +157,8 @@ def _assert_manager_call(self, manager, *assert_args, **assert_kwargs):
             'cert_file': None,
             'key_file': None,
         }
+        if BUFFER_SIZE:
+            call_kwargs['blocksize'] = BUFFER_SIZE
         call_kwargs.update(assert_kwargs)
         manager.assert_called_with(*assert_args, **call_kwargs)