From 857c76b7de62be6fdf31cba1c8da88e0148919cf Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 17 Mar 2026 15:27:05 +1100 Subject: [PATCH 1/4] change default transfer parameters --- .../inc/azure/storage/blobs/blob_options.hpp | 10 +++++++--- .../src/block_blob_client.cpp | 4 ++-- sdk/storage/azure-storage-common/CMakeLists.txt | 1 + .../common/internal/concurrent_transfer.hpp | 2 ++ .../src/concurrent_transfer.cpp | 16 ++++++++++++++++ .../storage/files/datalake/datalake_options.hpp | 4 +++- .../azure/storage/files/shares/share_options.hpp | 10 +++++++--- 7 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 sdk/storage/azure-storage-common/src/concurrent_transfer.cpp diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 7ad7759071..1cbce7edfe 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -771,12 +772,14 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The maximum number of bytes in a single request. */ - int64_t ChunkSize = 4 * 1024 * 1024; + int64_t ChunkSize = 16 * 1024 * 1024; /** * @brief The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = 5; + int32_t Concurrency = _internal::GetHardwareConcurrency() / 2 != 0 + ? _internal::GetHardwareConcurrency() / 2 + : 1; } TransferOptions; /** @@ -1023,7 +1026,8 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = 5; + int32_t Concurrency + = _internal::GetHardwareConcurrency() != 0 ? _internal::GetHardwareConcurrency() : 1; } TransferOptions; /** diff --git a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp index 7401e5203f..2628c9f3b9 100644 --- a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp @@ -178,7 +178,7 @@ namespace Azure { namespace Storage { namespace Blobs { const UploadBlockBlobFromOptions& options, const Azure::Core::Context& context) const { - constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL; + constexpr int64_t DefaultStageBlockSize = 16 * 1024 * 1024ULL; constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL; constexpr int64_t MaxBlockNumber = 50000; constexpr int64_t BlockGrainSize = 1 * 1024 * 1024; @@ -270,7 +270,7 @@ namespace Azure { namespace Storage { namespace Blobs { const UploadBlockBlobFromOptions& options, const Azure::Core::Context& context) const { - constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL; + constexpr int64_t DefaultStageBlockSize = 16 * 1024 * 1024ULL; constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL; constexpr int64_t MaxBlockNumber = 50000; constexpr int64_t BlockGrainSize = 1 * 1024 * 1024; diff --git a/sdk/storage/azure-storage-common/CMakeLists.txt b/sdk/storage/azure-storage-common/CMakeLists.txt index a91e1baaf7..f31c1ce44d 100644 --- a/sdk/storage/azure-storage-common/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/CMakeLists.txt @@ -70,6 +70,7 @@ set( set( AZURE_STORAGE_COMMON_SOURCE src/account_sas_builder.cpp + src/concurrent_transfer.cpp src/crypt.cpp src/file_io.cpp src/private/package_version.hpp diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/concurrent_transfer.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/concurrent_transfer.hpp index 9897c518ae..55f0ed6297 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/concurrent_transfer.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/concurrent_transfer.hpp @@ -13,6 +13,8 @@ namespace Azure { namespace Storage { namespace _internal { + int GetHardwareConcurrency(); + inline void ConcurrentTransfer( int64_t offset, int64_t length, diff --git a/sdk/storage/azure-storage-common/src/concurrent_transfer.cpp b/sdk/storage/azure-storage-common/src/concurrent_transfer.cpp new file mode 100644 index 0000000000..d130eb69d8 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/concurrent_transfer.cpp @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "azure/storage/common/internal/concurrent_transfer.hpp" + +#include + +namespace Azure { namespace Storage { namespace _internal { + + int GetHardwareConcurrency() + { + static int c = std::thread::hardware_concurrency(); + return c; + } + +}}} // namespace Azure::Storage::_internal \ No newline at end of file diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 26220a39ae..38eb156f03 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -1017,7 +1018,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { /** * The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = 5; + int32_t Concurrency + = _internal::GetHardwareConcurrency() != 0 ? _internal::GetHardwareConcurrency() : 1; } TransferOptions; /** diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp index a4f808c31c..5d675303d1 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -1325,12 +1326,14 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { /** * The maximum number of bytes in a single request. */ - int64_t ChunkSize = 4 * 1024 * 1024; + int64_t ChunkSize = 16 * 1024 * 1024; /** * The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = 5; + int32_t Concurrency = _internal::GetHardwareConcurrency() / 2 != 0 + ? _internal::GetHardwareConcurrency() / 2 + : 1; } TransferOptions; }; @@ -1399,7 +1402,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { /** * The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = 5; + int32_t Concurrency + = _internal::GetHardwareConcurrency() != 0 ? _internal::GetHardwareConcurrency() : 1; } TransferOptions; }; From 47d82a6e3638e6013d863e46938e75cb34cc04bd Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Wed, 20 May 2026 10:38:18 +1000 Subject: [PATCH 2/4] Revert some changes --- .../inc/azure/storage/blobs/blob_options.hpp | 9 +++------ .../azure-storage-blobs/src/block_blob_client.cpp | 4 ++-- .../azure/storage/files/datalake/datalake_options.hpp | 3 +-- .../inc/azure/storage/files/shares/share_options.hpp | 6 ++---- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 1cbce7edfe..12439e7ea6 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -772,14 +772,12 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The maximum number of bytes in a single request. */ - int64_t ChunkSize = 16 * 1024 * 1024; + int64_t ChunkSize = 4 * 1024 * 1024; /** * @brief The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = _internal::GetHardwareConcurrency() / 2 != 0 - ? _internal::GetHardwareConcurrency() / 2 - : 1; + int32_t Concurrency = std::min(96, std::max(8, _internal::GetHardwareConcurrency())); } TransferOptions; /** @@ -1026,8 +1024,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency - = _internal::GetHardwareConcurrency() != 0 ? _internal::GetHardwareConcurrency() : 1; + int32_t Concurrency = std::min(96, std::max(8, _internal::GetHardwareConcurrency())); } TransferOptions; /** diff --git a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp index 2628c9f3b9..7401e5203f 100644 --- a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp @@ -178,7 +178,7 @@ namespace Azure { namespace Storage { namespace Blobs { const UploadBlockBlobFromOptions& options, const Azure::Core::Context& context) const { - constexpr int64_t DefaultStageBlockSize = 16 * 1024 * 1024ULL; + constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL; constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL; constexpr int64_t MaxBlockNumber = 50000; constexpr int64_t BlockGrainSize = 1 * 1024 * 1024; @@ -270,7 +270,7 @@ namespace Azure { namespace Storage { namespace Blobs { const UploadBlockBlobFromOptions& options, const Azure::Core::Context& context) const { - constexpr int64_t DefaultStageBlockSize = 16 * 1024 * 1024ULL; + constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL; constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL; constexpr int64_t MaxBlockNumber = 50000; constexpr int64_t BlockGrainSize = 1 * 1024 * 1024; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 38eb156f03..8d9b8bcb7e 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -1018,8 +1018,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { /** * The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency - = _internal::GetHardwareConcurrency() != 0 ? _internal::GetHardwareConcurrency() : 1; + int32_t Concurrency = std::min(96, std::max(8, _internal::GetHardwareConcurrency())); } TransferOptions; /** diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp index 5d675303d1..cef07d1c2b 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp @@ -1326,14 +1326,12 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { /** * The maximum number of bytes in a single request. */ - int64_t ChunkSize = 16 * 1024 * 1024; + int64_t ChunkSize = 4 * 1024 * 1024; /** * The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = _internal::GetHardwareConcurrency() / 2 != 0 - ? _internal::GetHardwareConcurrency() / 2 - : 1; + int32_t Concurrency = std::min(96, std::max(8, _internal::GetHardwareConcurrency())); } TransferOptions; }; From c6c5d369ea8593e81371f2e84aedfc486785b511 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Wed, 20 May 2026 10:41:30 +1000 Subject: [PATCH 3/4] fix --- .../inc/azure/storage/blobs/blob_options.hpp | 5 +++-- .../inc/azure/storage/files/datalake/datalake_options.hpp | 3 ++- .../inc/azure/storage/files/shares/share_options.hpp | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 12439e7ea6..e883f84c86 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -777,7 +778,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = std::min(96, std::max(8, _internal::GetHardwareConcurrency())); + int32_t Concurrency = (std::min)(96, (std::max)(8, _internal::GetHardwareConcurrency())); } TransferOptions; /** @@ -1024,7 +1025,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = std::min(96, std::max(8, _internal::GetHardwareConcurrency())); + int32_t Concurrency = (std::min)(96, (std::max)(8, _internal::GetHardwareConcurrency())); } TransferOptions; /** diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 8d9b8bcb7e..5c26ce81ec 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -1018,7 +1019,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { /** * The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = std::min(96, std::max(8, _internal::GetHardwareConcurrency())); + int32_t Concurrency = (std::min)(96, (std::max)(8, _internal::GetHardwareConcurrency())); } TransferOptions; /** diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp index cef07d1c2b..623d93c7e5 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -1331,7 +1332,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { /** * The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency = std::min(96, std::max(8, _internal::GetHardwareConcurrency())); + int32_t Concurrency = (std::min)(96, (std::max)(8, _internal::GetHardwareConcurrency())); } TransferOptions; }; From 8e7ab3eb79277a63c875e8ac7374e71e809ee297 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Wed, 20 May 2026 10:44:35 +1000 Subject: [PATCH 4/4] fix --- sdk/storage/azure-storage-common/src/concurrent_transfer.cpp | 4 ++-- .../inc/azure/storage/files/shares/share_options.hpp | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-common/src/concurrent_transfer.cpp b/sdk/storage/azure-storage-common/src/concurrent_transfer.cpp index d130eb69d8..14b3d77a71 100644 --- a/sdk/storage/azure-storage-common/src/concurrent_transfer.cpp +++ b/sdk/storage/azure-storage-common/src/concurrent_transfer.cpp @@ -9,8 +9,8 @@ namespace Azure { namespace Storage { namespace _internal { int GetHardwareConcurrency() { - static int c = std::thread::hardware_concurrency(); + static int c = static_cast(std::thread::hardware_concurrency()); return c; } -}}} // namespace Azure::Storage::_internal \ No newline at end of file +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp index 623d93c7e5..527641dc28 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp @@ -1401,8 +1401,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { /** * The maximum number of threads that may be used in a parallel transfer. */ - int32_t Concurrency - = _internal::GetHardwareConcurrency() != 0 ? _internal::GetHardwareConcurrency() : 1; + int32_t Concurrency = (std::min)(96, (std::max)(8, _internal::GetHardwareConcurrency())); } TransferOptions; };