From dd0786d87205dfd0ffe40e2e4fe59ba9a79d3071 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 21 Aug 2026 14:03:50 +0000 Subject: [PATCH 1/7] Add tests for address overlap --- test/provider_tracking.cpp | 107 ++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/test/provider_tracking.cpp b/test/provider_tracking.cpp index 52d5eeb8fd..3ad11c9fdd 100644 --- a/test/provider_tracking.cpp +++ b/test/provider_tracking.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Intel Corporation +// Copyright (C) 2025-2026 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -145,6 +145,75 @@ TEST_P(TrackingProviderTest, whole_size_success) { ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } +TEST_P(TrackingProviderTest, identical_address_ranges) { + // Two pools allocate identical address ranges. Freeing through pool0 must + // remove only its entry and leave the address associated with pool1. + umf_memory_pool_handle_t pool0 = pool.get(); + size_t size = FIXED_BUFFER_SIZE - (2 * page_size); + void *ptr0 = umfPoolAlignedMalloc(pool0, size, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size, &provider1, &pool1); + + void *ptr1 = umfPoolMalloc(pool1, size); + ASSERT_EQ(ptr1, ptr0); + + umf_memory_pool_handle_t found_pool = nullptr; + umf_result_t umf_result = umfPoolByPtr(ptr0, &found_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(found_pool, pool1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + found_pool = nullptr; + umf_result = umfPoolByPtr(ptr1, &found_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_EQ(found_pool, pool1); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfPoolDestroy(pool1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfMemoryProviderDestroy(provider1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(TrackingProviderTest, identical_address_ranges_umf_free) { + // Two pools return the same address. umfFree called for ptr0 is expected + // to leave the second allocation associated with pool1. + umf_memory_pool_handle_t pool0 = pool.get(); + size_t size = FIXED_BUFFER_SIZE - (2 * page_size); + void *ptr0 = umfPoolAlignedMalloc(pool0, size, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size, &provider1, &pool1); + + void *ptr1 = umfPoolMalloc(pool1, size); + ASSERT_EQ(ptr1, ptr0); + + umf_result_t umf_result = umfFree(ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_memory_pool_handle_t found_pool = nullptr; + umf_result = umfPoolByPtr(ptr1, &found_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_EQ(found_pool, pool1); + + umf_result = umfPoolFree(found_pool, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfPoolDestroy(pool1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfMemoryProviderDestroy(provider1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + TEST_P(TrackingProviderTest, half_size_success) { umf_result_t umf_result; size_t size0; @@ -210,6 +279,42 @@ TEST_P(TrackingProviderTest, failure_exceeding_size) { ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } +TEST_P(TrackingProviderTest, partial_overlap) { + // The second range starts inside the first and extends beyond it. Since it + // belongs to another pool, tracking it should succeed and resolve pool1. + umf_memory_pool_handle_t pool0 = pool.get(); + size_t size0 = 4 * page_size; + void *ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + void *overlap_begin = static_cast(ptr0) + page_size; + size_t size1 = size0; + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(overlap_begin, size1, &provider1, &pool1); + + void *ptr1 = umfPoolMalloc(pool1, size1); + EXPECT_NE(ptr1, nullptr); + + if (ptr1 != nullptr) { + umf_memory_pool_handle_t found_pool = nullptr; + umf_result_t umf_result = umfPoolByPtr(ptr1, &found_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_EQ(found_pool, pool1); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + umf_result_t umf_result = umfPoolDestroy(pool1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfMemoryProviderDestroy(provider1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + #define MAX_ARRAY 9 #define TEST_LEVEL_SUCCESS 7 #define TEST_LEVEL_FAILURE 8 From a0a55004d8af94f08004e60d21d5d0b142aa3bb9 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 21 Aug 2026 14:29:20 +0000 Subject: [PATCH 2/7] Return ambiguous error from umfFree in case of overlaping addr --- include/umf/base.h | 4 ++- include/umf/memory_pool.h | 4 ++- src/memory_pool.c | 18 +++++++++-- src/provider/provider_tracking.c | 53 ++++++++++++++++++++++++++++++-- src/provider/provider_tracking.h | 7 ++++- test/provider_tracking.cpp | 12 +++++--- 6 files changed, 86 insertions(+), 12 deletions(-) diff --git a/include/umf/base.h b/include/umf/base.h index 11d7b723b4..48f243f2ab 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2025 Intel Corporation + * Copyright (C) 2023-2026 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -50,6 +50,8 @@ typedef enum umf_result_t { UMF_RESULT_ERROR_OUT_OF_RESOURCES = 8, ///< Out of internal resources UMF_RESULT_ERROR_INVALID_CTL_PATH = 9, ///< CTL path is not supported or not found + UMF_RESULT_ERROR_AMBIGUOUS = + 10, ///< Multiple allocations match the supplied argument UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown error } umf_result_t; diff --git a/include/umf/memory_pool.h b/include/umf/memory_pool.h index 5662684fa7..81a456290e 100644 --- a/include/umf/memory_pool.h +++ b/include/umf/memory_pool.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2025 Intel Corporation + * Copyright (C) 2023-2026 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -126,6 +126,8 @@ umf_result_t umfPoolFree(umf_memory_pool_handle_t hPool, void *ptr); /// @brief Frees the memory space pointed by ptr if it belongs to UMF pool, does nothing otherwise. /// @param ptr pointer to the allocated memory /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +/// UMF_RESULT_ERROR_AMBIGUOUS if multiple active allocations have the +/// same address. Use umfPoolFree() to select the allocation's pool. /// Whether any status other than UMF_RESULT_SUCCESS can be returned /// depends on the memory provider used by the pool. /// diff --git a/src/memory_pool.c b/src/memory_pool.c index 2161d5d45c..3f293079fe 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2025 Intel Corporation + * Copyright (C) 2023-2026 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -542,8 +542,22 @@ umf_result_t umfPoolDestroy(umf_memory_pool_handle_t hPool) { } umf_result_t umfFree(void *ptr) { + umf_result_t ret = UMF_RESULT_SUCCESS; + if (umfMemoryTrackerHasMultiplePools()) { + size_t exact_match_count = 0; + ret = umfMemoryTrackerGetAllocInfoExactCount(ptr, &exact_match_count); + if (ret != UMF_RESULT_SUCCESS) { + return ret; + } + if (exact_match_count > 1) { + LOG_ERR("cannot free ambiguous pointer %p matching %zu allocations", + ptr, exact_match_count); + return UMF_RESULT_ERROR_AMBIGUOUS; + } + } + umf_memory_pool_handle_t hPool = NULL; - umf_result_t ret = umfPoolByPtr(ptr, &hPool); + ret = umfPoolByPtr(ptr, &hPool); if (ret == UMF_RESULT_SUCCESS) { LOG_DEBUG("calling umfPoolFree(pool=%p, ptr=%p) ...", (void *)hPool, ptr); diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index e736a4e490..1b628bc053 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -43,6 +43,9 @@ struct umf_memory_tracker_t { // when one memory pool acts as a memory provider // for another memory pool (nested memory pooling). critnib *alloc_segments_map[MAX_LEVELS_OF_ALLOC_SEGMENT_MAP]; + // Monotonic count used to skip ambiguity checks until a second tracked + // pool has been created. + uint64_t pools_created; utils_mutex_t splitMergeMutex; umf_ba_pool_t *ipc_info_allocator; critnib *ipc_segments_map; @@ -583,6 +586,47 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, return UMF_RESULT_SUCCESS; } +umf_result_t umfMemoryTrackerGetAllocInfoExactCount(const void *ptr, + size_t *count) { + if (UNLIKELY(ptr == NULL || count == NULL)) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (UNLIKELY(TRACKER == NULL || TRACKER->alloc_segments_map[0] == NULL)) { + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + *count = 0; + for (int level = 0; level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; ++level) { + uintptr_t key = 0; + tracker_alloc_info_t *value = NULL; + void *ref_value = NULL; + int found = + critnib_find(TRACKER->alloc_segments_map[level], (uintptr_t)ptr, + FIND_LE, (void *)&key, (void **)&value, &ref_value); + + if (found && value != NULL && key == (uintptr_t)ptr) { + ++*count; + } + + if (ref_value) { + critnib_release(TRACKER->alloc_segments_map[level], ref_value); + } + } + + return UMF_RESULT_SUCCESS; +} + +int umfMemoryTrackerHasMultiplePools(void) { + if (UNLIKELY(TRACKER == NULL)) { + return 0; + } + + uint64_t pools_created = 0; + utils_atomic_load_acquire_u64(&TRACKER->pools_created, &pools_created); + return pools_created > 1; +} + umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, umf_ipc_info_t *pIpcInfo) { assert(pIpcInfo); @@ -1410,8 +1454,13 @@ umf_result_t umfTrackingMemoryProviderCreate( (void *)params.pool, (void *)params.ipcCache, (void *)params.hIpcMappedCache); - return umfMemoryProviderCreate(&UMF_TRACKING_MEMORY_PROVIDER_OPS, ¶ms, - hTrackingProvider); + umf_result_t ret = umfMemoryProviderCreate( + &UMF_TRACKING_MEMORY_PROVIDER_OPS, ¶ms, hTrackingProvider); + if (ret == UMF_RESULT_SUCCESS) { + utils_atomic_increment_u64(¶ms.hTracker->pools_created); + } + + return ret; } void umfTrackingMemoryProviderGetUpstreamProvider( diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index 254bbf6786..7ce0460c10 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2025 Intel Corporation + * Copyright (C) 2023-2026 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -49,6 +49,11 @@ typedef struct tracker_alloc_info_t { umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, tracker_alloc_info_t **info); +umf_result_t umfMemoryTrackerGetAllocInfoExactCount(const void *ptr, + size_t *count); + +int umfMemoryTrackerHasMultiplePools(void); + typedef struct umf_ipc_info_t { umf_memory_properties_handle_t props; diff --git a/test/provider_tracking.cpp b/test/provider_tracking.cpp index 3ad11c9fdd..27c1c02ba1 100644 --- a/test/provider_tracking.cpp +++ b/test/provider_tracking.cpp @@ -183,8 +183,8 @@ TEST_P(TrackingProviderTest, identical_address_ranges) { } TEST_P(TrackingProviderTest, identical_address_ranges_umf_free) { - // Two pools return the same address. umfFree called for ptr0 is expected - // to leave the second allocation associated with pool1. + // Two pools return the same address. umfFree cannot select one allocation, + // so it must report ambiguity and leave both allocations tracked. umf_memory_pool_handle_t pool0 = pool.get(); size_t size = FIXED_BUFFER_SIZE - (2 * page_size); void *ptr0 = umfPoolAlignedMalloc(pool0, size, utils_get_page_size()); @@ -198,14 +198,16 @@ TEST_P(TrackingProviderTest, identical_address_ranges_umf_free) { ASSERT_EQ(ptr1, ptr0); umf_result_t umf_result = umfFree(ptr0); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_AMBIGUOUS); umf_memory_pool_handle_t found_pool = nullptr; umf_result = umfPoolByPtr(ptr1, &found_pool); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - EXPECT_EQ(found_pool, pool1); + ASSERT_EQ(found_pool, pool1); - umf_result = umfPoolFree(found_pool, ptr1); + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = umfPoolFree(pool0, ptr0); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); umf_result = umfPoolDestroy(pool1); From 8a12d2e5ada719dfd590b767f57a834b12ae64b0 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 21 Aug 2026 15:37:42 +0000 Subject: [PATCH 3/7] Add address space support to memory providers --- include/umf/base.h | 14 +- include/umf/memory_provider.h | 16 + include/umf/memory_provider_ops.h | 19 +- include/umf/providers/provider_fixed_memory.h | 10 +- src/libumf.def | 3 + src/libumf.map | 7 +- src/memory_provider.c | 32 ++ src/provider/provider_cuda.c | 18 + src/provider/provider_devdax_memory.c | 11 + src/provider/provider_file_memory.c | 11 + src/provider/provider_fixed_memory.c | 32 +- src/provider/provider_level_zero.c | 18 + src/provider/provider_os_memory.c | 11 + src/provider/provider_tracking.c | 421 +++++++++++------- test/common/provider.hpp | 5 + test/memoryProviderAPI.cpp | 69 +++ test/poolFixtures.hpp | 4 +- test/provider_fixed_memory.cpp | 32 +- test/provider_tracking.cpp | 22 +- test/provider_tracking_fixture_tests.cpp | 12 +- test/utils/cpp_helpers.hpp | 1 + 21 files changed, 600 insertions(+), 168 deletions(-) diff --git a/include/umf/base.h b/include/umf/base.h index 48f243f2ab..1e4e620d7f 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -28,7 +28,7 @@ extern "C" { #define UMF_MINOR_VERSION(_ver) (_ver & 0x0000ffff) /// @brief Current version of the UMF headers -#define UMF_VERSION_CURRENT UMF_MAKE_VERSION(1, 0) +#define UMF_VERSION_CURRENT UMF_MAKE_VERSION(1, 1) /// @brief Operation results typedef enum umf_result_t { @@ -55,6 +55,18 @@ typedef enum umf_result_t { UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown error } umf_result_t; +/// @brief Process-local identity of a virtual address space. +typedef struct umf_memory_provider_address_space_t { + /// Opaque namespace token. NULL identifies the host address space. UMF + /// compares non-NULL tokens by address and never dereferences them. + /// Providers whose pointers belong to the same non-host address-space + /// namespace must return the same token. The token address must remain + /// stable while providers using it exist. + const void *namespace_token; + uintptr_t context; + uintptr_t device; +} umf_memory_provider_address_space_t; + /// @brief Handle to the memory properties structure typedef struct umf_memory_properties_t *umf_memory_properties_handle_t; diff --git a/include/umf/memory_provider.h b/include/umf/memory_provider.h index d38202e8a6..0320009a0f 100644 --- a/include/umf/memory_provider.h +++ b/include/umf/memory_provider.h @@ -139,6 +139,22 @@ umf_result_t umfMemoryProviderGetCacheLineSize(umf_memory_provider_handle_t hProvider, size_t *size); +/// +/// @brief Retrieve the process-local address-space identity used by allocations +/// from the provider. +/// @details The returned namespace token is NULL for the host address space. +/// Providers that share another address-space namespace must return +/// the same stable non-NULL token. +/// @param hProvider handle to the memory provider +/// @param address_space [out] pointer to the address-space identity +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +/// UMF_RESULT_ERROR_NOT_SUPPORTED if the provider cannot determine its +/// address space. +/// +umf_result_t umfMemoryProviderGetAddressSpace( + umf_memory_provider_handle_t hProvider, + umf_memory_provider_address_space_t *address_space); + /// /// @brief Discard physical pages within the virtual memory mapping associated at the given addr /// and \p size. This call is asynchronous and may delay purging the pages indefinitely. diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index 77c277d3d5..6e72f3de96 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -21,7 +21,7 @@ extern "C" { /// @brief Version of the Memory Provider ops structure. /// NOTE: This is equal to the latest UMF version, in which the ops structure /// has been modified. -#define UMF_PROVIDER_OPS_VERSION_CURRENT UMF_MAKE_VERSION(1, 2) +#define UMF_PROVIDER_OPS_VERSION_CURRENT UMF_MAKE_VERSION(1, 3) /// /// @brief This structure comprises function pointers used by corresponding @@ -336,6 +336,23 @@ typedef struct umf_memory_provider_ops_t { /// umf_result_t (*get_cache_line_size)(void *provider, size_t *size); + // The following operations were added in ops version 1.3 + + /// + /// @brief Retrieve the process-local context and device identities used by + /// allocations from the provider. + /// @details The callback must return a NULL namespace token for the host + /// address space. Providers that share another address-space + /// namespace must return the same stable non-NULL token. + /// @param provider pointer to the memory provider + /// @param address_space [out] pointer to the address-space identity + /// @return UMF_RESULT_SUCCESS on success or appropriate error code on + /// failure. UMF_RESULT_ERROR_NOT_SUPPORTED if the provider cannot + /// determine its address space. + /// + umf_result_t (*get_address_space)( + void *provider, umf_memory_provider_address_space_t *address_space); + } umf_memory_provider_ops_t; #ifdef __cplusplus diff --git a/include/umf/providers/provider_fixed_memory.h b/include/umf/providers/provider_fixed_memory.h index fcedd5c005..b28df3d5f1 100644 --- a/include/umf/providers/provider_fixed_memory.h +++ b/include/umf/providers/provider_fixed_memory.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024-2025 Intel Corporation + * Copyright (C) 2024-2026 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -41,6 +41,14 @@ umf_result_t umfFixedMemoryProviderParamsCreate( umf_result_t umfFixedMemoryProviderParamsSetMemory( umf_fixed_memory_provider_params_handle_t hParams, void *ptr, size_t size); +/// @brief Set the address space containing the fixed memory region. +/// @param hParams [in] handle to the parameters of the Fixed Memory Provider. +/// @param addressSpace [in] process-local address-space identity. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfFixedMemoryProviderParamsSetAddressSpace( + umf_fixed_memory_provider_params_handle_t hParams, + const umf_memory_provider_address_space_t *addressSpace); + /// @brief Destroy parameters struct. /// @param hParams [in] handle to the parameters of the Fixed Memory Provider. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. diff --git a/src/libumf.def b/src/libumf.def index d7b39f9fa6..98cb63bd8c 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -160,3 +160,6 @@ EXPORTS umfScalablePoolParamsSetName ; Added in UMF_1.2 umfMemoryProviderGetCacheLineSize +; Added in UMF_1.3 + umfFixedMemoryProviderParamsSetAddressSpace + umfMemoryProviderGetAddressSpace diff --git a/src/libumf.map b/src/libumf.map index bf8e7e2b25..5eeb321332 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -158,6 +158,11 @@ UMF_1.1 { umfScalablePoolParamsSetName; } UMF_1.0; -UMF_1.2 { +UMF_1.2 { umfMemoryProviderGetCacheLineSize; } UMF_1.1; + +UMF_1.3 { + umfFixedMemoryProviderParamsSetAddressSpace; + umfMemoryProviderGetAddressSpace; +} UMF_1.2; diff --git a/src/memory_provider.c b/src/memory_provider.c index b6574ddd41..a2270110cd 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -182,6 +182,14 @@ static umf_result_t umfDefaultGetCacheLineSize(void *provider, size_t *size) { return UMF_RESULT_SUCCESS; } +static umf_result_t +umfDefaultGetAddressSpace(void *provider, + umf_memory_provider_address_space_t *address_space) { + (void)provider; + (void)address_space; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { if (!ops->ext_purge_lazy) { ops->ext_purge_lazy = umfDefaultPurgeLazy; @@ -211,6 +219,10 @@ void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { ops->ext_get_allocation_properties_size = umfDefaultGetAllocationPropertiesSize; } + + if (!ops->get_address_space) { + ops->get_address_space = umfDefaultGetAddressSpace; + } } void assignOpsIpcDefaults(umf_memory_provider_ops_t *ops) { @@ -336,6 +348,13 @@ umf_result_t umfMemoryProviderCreate(const umf_memory_provider_ops_t *ops, memcpy(&compatible_ops, ops, offsetof(umf_memory_provider_ops_t, get_cache_line_size)); compatible_ops.get_cache_line_size = umfDefaultGetCacheLineSize; + compatible_ops.get_address_space = umfDefaultGetAddressSpace; + } else if (UMF_MINOR_VERSION(ops->version) == 2) { + LOG_INFO("Detected 1.2 version of Memory Provider ops, " + "upgrading to current version"); + memcpy(&compatible_ops, ops, + offsetof(umf_memory_provider_ops_t, get_address_space)); + compatible_ops.get_address_space = umfDefaultGetAddressSpace; } else { LOG_ERR("Unsupported Memory Provider ops version: %d", ops->version); @@ -507,6 +526,19 @@ umfMemoryProviderGetCacheLineSize(umf_memory_provider_handle_t hProvider, return res; } +umf_result_t umfMemoryProviderGetAddressSpace( + umf_memory_provider_handle_t hProvider, + umf_memory_provider_address_space_t *address_space) { + UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((address_space != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_result_t res = hProvider->ops.get_address_space( + hProvider->provider_priv, address_space); + + checkErrorAndSetLastProvider(res, hProvider); + return res; +} + umf_result_t umfMemoryProviderGetName(umf_memory_provider_handle_t hProvider, const char **name) { UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index ebe51d9db8..c751b1308b 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -59,6 +59,8 @@ typedef struct cu_memory_provider_t { char name[64]; } cu_memory_provider_t; +static const char CUDA_ADDRESS_SPACE_NAMESPACE = 0; + #define CTL_PROVIDER_TYPE cu_memory_provider_t #include "provider_ctl_stats_impl.h" @@ -833,6 +835,21 @@ static umf_result_t cu_memory_provider_get_cache_line_size(void *provider, return UMF_RESULT_SUCCESS; } +static umf_result_t cu_memory_provider_get_address_space( + void *provider, umf_memory_provider_address_space_t *address_space) { + cu_memory_provider_t *cu_provider = provider; + if (cu_provider->memory_type == UMF_MEMORY_TYPE_HOST) { + address_space->namespace_token = NULL; + address_space->context = 0; + address_space->device = 0; + } else { + address_space->namespace_token = &CUDA_ADDRESS_SPACE_NAMESPACE; + address_space->context = (uintptr_t)cu_provider->context; + address_space->device = (uintptr_t)cu_provider->device; + } + return UMF_RESULT_SUCCESS; +} + static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = cu_memory_provider_initialize, @@ -843,6 +860,7 @@ static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .get_recommended_page_size = cu_memory_provider_get_recommended_page_size, .get_min_page_size = cu_memory_provider_get_min_page_size, .get_cache_line_size = cu_memory_provider_get_cache_line_size, + .get_address_space = cu_memory_provider_get_address_space, .get_name = cu_memory_provider_get_name, // TODO /* diff --git a/src/provider/provider_devdax_memory.c b/src/provider/provider_devdax_memory.c index 7c79207081..e437d412bb 100644 --- a/src/provider/provider_devdax_memory.c +++ b/src/provider/provider_devdax_memory.c @@ -408,6 +408,16 @@ static umf_result_t devdax_get_name(void *provider, const char **name) { return UMF_RESULT_SUCCESS; } +static umf_result_t +devdax_get_address_space(void *provider, + umf_memory_provider_address_space_t *address_space) { + (void)provider; + address_space->namespace_token = NULL; + address_space->context = 0; + address_space->device = 0; + return UMF_RESULT_SUCCESS; +} + static umf_result_t devdax_allocation_split(void *provider, void *ptr, size_t totalSize, size_t firstSize) { @@ -606,6 +616,7 @@ static umf_memory_provider_ops_t UMF_DEVDAX_MEMORY_PROVIDER_OPS = { .get_recommended_page_size = devdax_get_recommended_page_size, .get_min_page_size = devdax_get_min_page_size, .get_cache_line_size = devdax_get_cache_line_size, + .get_address_space = devdax_get_address_space, .get_name = devdax_get_name, .ext_purge_lazy = devdax_purge_lazy, .ext_purge_force = devdax_purge_force, diff --git a/src/provider/provider_file_memory.c b/src/provider/provider_file_memory.c index 5b8baae55a..d796ee0d1a 100644 --- a/src/provider/provider_file_memory.c +++ b/src/provider/provider_file_memory.c @@ -708,6 +708,16 @@ static umf_result_t file_get_name(void *provider, const char **name) { return UMF_RESULT_SUCCESS; } +static umf_result_t +file_get_address_space(void *provider, + umf_memory_provider_address_space_t *address_space) { + (void)provider; + address_space->namespace_token = NULL; + address_space->context = 0; + address_space->device = 0; + return UMF_RESULT_SUCCESS; +} + static umf_result_t file_allocation_split(void *provider, void *ptr, size_t totalSize, size_t firstSize) { file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; @@ -965,6 +975,7 @@ static umf_memory_provider_ops_t UMF_FILE_MEMORY_PROVIDER_OPS = { .get_recommended_page_size = file_get_recommended_page_size, .get_min_page_size = file_get_min_page_size, .get_cache_line_size = file_get_cache_line_size, + .get_address_space = file_get_address_space, .get_name = file_get_name, .ext_purge_lazy = file_purge_lazy, .ext_purge_force = file_purge_force, diff --git a/src/provider/provider_fixed_memory.c b/src/provider/provider_fixed_memory.c index 9b12556a99..23cc7a937a 100644 --- a/src/provider/provider_fixed_memory.c +++ b/src/provider/provider_fixed_memory.c @@ -31,8 +31,9 @@ static const char *DEFAULT_NAME = "FIXED"; typedef struct fixed_memory_provider_t { - void *base; // base address of memory - size_t size; // size of the memory region + void *base; // base address of memory + size_t size; // size of the memory region + umf_memory_provider_address_space_t address_space; coarse_t *coarse; // coarse library handle ctl_stats_t stats; char name[64]; @@ -42,6 +43,7 @@ typedef struct fixed_memory_provider_t { typedef struct umf_fixed_memory_provider_params_t { void *ptr; size_t size; + umf_memory_provider_address_space_t address_space; char name[64]; } umf_fixed_memory_provider_params_t; @@ -158,6 +160,7 @@ static umf_result_t fixed_initialize(const void *params, void **provider) { fixed_provider->base = in_params->ptr; fixed_provider->size = in_params->size; + fixed_provider->address_space = in_params->address_space; *provider = fixed_provider; @@ -337,6 +340,14 @@ static umf_result_t fixed_get_cache_line_size(void *provider, size_t *size) { return UMF_RESULT_SUCCESS; } +static umf_result_t +fixed_get_address_space(void *provider, + umf_memory_provider_address_space_t *address_space) { + fixed_memory_provider_t *fixed_provider = provider; + *address_space = fixed_provider->address_space; + return UMF_RESULT_SUCCESS; +} + static umf_memory_provider_ops_t UMF_FIXED_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = fixed_initialize, @@ -358,6 +369,7 @@ static umf_memory_provider_ops_t UMF_FIXED_MEMORY_PROVIDER_OPS = { .ext_open_ipc_handle = NULL, .ext_close_ipc_handle = NULL, .ext_ctl = fixed_ctl, + .get_address_space = fixed_get_address_space, }; const umf_memory_provider_ops_t *umfFixedMemoryProviderOps(void) { @@ -382,6 +394,11 @@ umf_result_t umfFixedMemoryProviderParamsCreate( strncpy(params->name, DEFAULT_NAME, sizeof(params->name) - 1); params->name[sizeof(params->name) - 1] = '\0'; + params->address_space = (umf_memory_provider_address_space_t){ + .namespace_token = NULL, + .context = 0, + .device = 0, + }; umf_result_t ret = umfFixedMemoryProviderParamsSetMemory(params, ptr, size); if (ret != UMF_RESULT_SUCCESS) { @@ -426,6 +443,17 @@ umf_result_t umfFixedMemoryProviderParamsSetMemory( return UMF_RESULT_SUCCESS; } +umf_result_t umfFixedMemoryProviderParamsSetAddressSpace( + umf_fixed_memory_provider_params_handle_t hParams, + const umf_memory_provider_address_space_t *addressSpace) { + if (hParams == NULL || addressSpace == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + hParams->address_space = *addressSpace; + return UMF_RESULT_SUCCESS; +} + umf_result_t umfFixedMemoryProviderParamsSetName( umf_fixed_memory_provider_params_handle_t hParams, const char *name) { if (hParams == NULL) { diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index db1466e299..e02d4d93c5 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -98,6 +98,8 @@ typedef struct ze_memory_provider_t { ctl_stats_t stats; } ze_memory_provider_t; +static const char LEVEL_ZERO_ADDRESS_SPACE_NAMESPACE = 0; + typedef struct ze_ops_t { #if defined(ZE_CACHELINE_SIZE_EXT_NAME) ze_result_t (*zeDriverGet)(uint32_t *, ze_driver_handle_t *); @@ -1033,6 +1035,21 @@ static umf_result_t ze_memory_provider_get_name(void *provider, return UMF_RESULT_SUCCESS; } +static umf_result_t ze_memory_provider_get_address_space( + void *provider, umf_memory_provider_address_space_t *address_space) { + ze_memory_provider_t *ze_provider = provider; + if (ze_provider->memory_type == ZE_MEMORY_TYPE_HOST) { + address_space->namespace_token = NULL; + address_space->context = 0; + address_space->device = 0; + } else { + address_space->namespace_token = &LEVEL_ZERO_ADDRESS_SPACE_NAMESPACE; + address_space->context = (uintptr_t)ze_provider->context; + address_space->device = (uintptr_t)ze_provider->device; + } + return UMF_RESULT_SUCCESS; +} + static umf_result_t ze_memory_provider_allocation_merge(void *hProvider, void *lowPtr, void *highPtr, @@ -1472,6 +1489,7 @@ static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .get_recommended_page_size = ze_memory_provider_get_recommended_page_size, .get_min_page_size = ze_memory_provider_get_min_page_size, .get_cache_line_size = ze_memory_provider_get_cache_line_size, + .get_address_space = ze_memory_provider_get_address_space, .get_name = ze_memory_provider_get_name, .ext_purge_lazy = ze_memory_provider_purge_lazy, .ext_purge_force = ze_memory_provider_purge_force, diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index c0b87cf919..9f317d70ae 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -1401,6 +1401,16 @@ static umf_result_t os_get_cache_line_size(void *provider, size_t *size) { return UMF_RESULT_SUCCESS; } +static umf_result_t +os_get_address_space(void *provider, + umf_memory_provider_address_space_t *address_space) { + (void)provider; + address_space->namespace_token = NULL; + address_space->context = 0; + address_space->device = 0; + return UMF_RESULT_SUCCESS; +} + static umf_memory_provider_ops_t UMF_OS_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = os_initialize, @@ -1411,6 +1421,7 @@ static umf_memory_provider_ops_t UMF_OS_MEMORY_PROVIDER_OPS = { .get_recommended_page_size = os_get_recommended_page_size, .get_min_page_size = os_get_min_page_size, .get_cache_line_size = os_get_cache_line_size, + .get_address_space = os_get_address_space, .get_name = os_get_name, .ext_purge_lazy = os_purge_lazy, .ext_purge_force = os_purge_force, diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 1b628bc053..bad47b25f2 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -37,12 +37,15 @@ uint64_t IPC_HANDLE_ID = 0; uint64_t unique_alloc_id = 0; // requires atomic access +typedef struct tracker_address_space_t { + umf_memory_provider_address_space_t id; + critnib *alloc_segments_map[MAX_LEVELS_OF_ALLOC_SEGMENT_MAP]; + struct tracker_address_space_t *next; +} tracker_address_space_t; + struct umf_memory_tracker_t { umf_ba_pool_t *alloc_info_allocator; - // Multilevel maps are needed to support the case - // when one memory pool acts as a memory provider - // for another memory pool (nested memory pooling). - critnib *alloc_segments_map[MAX_LEVELS_OF_ALLOC_SEGMENT_MAP]; + tracker_address_space_t *address_spaces; // Monotonic count used to skip ambiguity checks until a second tracked // pool has been created. uint64_t pools_created; @@ -58,12 +61,75 @@ typedef struct tracker_ipc_info_t { ipc_opened_cache_value_t *ipc_cache_value; } tracker_ipc_info_t; +static void free_leaf(void *leaf_allocator, void *ptr); + +static int +address_spaces_equal(const umf_memory_provider_address_space_t *lhs, + const umf_memory_provider_address_space_t *rhs) { + return lhs->namespace_token == rhs->namespace_token && + lhs->context == rhs->context && lhs->device == rhs->device; +} + +static void destroy_address_space(tracker_address_space_t *address_space) { + for (int level = 0; level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; ++level) { + if (address_space->alloc_segments_map[level]) { + critnib_delete(address_space->alloc_segments_map[level]); + } + } + umf_ba_global_free(address_space); +} + +static umf_result_t +get_or_create_address_space(umf_memory_tracker_handle_t tracker, + const umf_memory_provider_address_space_t *id, + tracker_address_space_t **address_space) { + int lock_result = utils_mutex_lock(&tracker->splitMergeMutex); + if (lock_result) { + return UMF_RESULT_ERROR_UNKNOWN; + } + + for (tracker_address_space_t *current = tracker->address_spaces; current; + current = current->next) { + if (address_spaces_equal(¤t->id, id)) { + *address_space = current; + utils_mutex_unlock(&tracker->splitMergeMutex); + return UMF_RESULT_SUCCESS; + } + } + + tracker_address_space_t *new_space = + umf_ba_global_alloc(sizeof(*new_space)); + if (!new_space) { + utils_mutex_unlock(&tracker->splitMergeMutex); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + memset(new_space, 0, sizeof(*new_space)); + new_space->id = *id; + + for (int level = 0; level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; ++level) { + new_space->alloc_segments_map[level] = + critnib_new(free_leaf, tracker->alloc_info_allocator); + if (!new_space->alloc_segments_map[level]) { + destroy_address_space(new_space); + utils_mutex_unlock(&tracker->splitMergeMutex); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + } + + new_space->next = tracker->address_spaces; + utils_atomic_store_release_ptr((void **)&tracker->address_spaces, + new_space); + *address_space = new_space; + utils_mutex_unlock(&tracker->splitMergeMutex); + return UMF_RESULT_SUCCESS; +} + // Get the most nested (on the highest level) allocation segment in the map with the `ptr` key. // If `no_children` is set to 1, the function will return the entry // only if it has no children on the higher level. // The function returns the entry if found, otherwise NULL. static tracker_alloc_info_t *get_most_nested_alloc_segment( - umf_memory_tracker_handle_t hTracker, const void *ptr, int *_level, + tracker_address_space_t *address_space, const void *ptr, int *_level, uintptr_t *_parent_key, tracker_alloc_info_t **_parent_value, void **_ref_value, void **_ref_parent_value, int no_children) { @@ -87,9 +153,9 @@ static tracker_alloc_info_t *get_most_nested_alloc_segment( do { assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); - found = - critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, - FIND_LE, (void *)&rkey, (void **)&rvalue, &ref_value); + found = critnib_find(address_space->alloc_segments_map[level], + (uintptr_t)ptr, FIND_LE, (void *)&rkey, + (void **)&rvalue, &ref_value); if (!found) { assert(ref_value == NULL); break; @@ -97,7 +163,8 @@ static tracker_alloc_info_t *get_most_nested_alloc_segment( if (rvalue == NULL) { if (ref_value) { - critnib_release(hTracker->alloc_segments_map[level], ref_value); + critnib_release(address_space->alloc_segments_map[level], + ref_value); } parent_value = NULL; parent_key = 0; @@ -131,8 +198,9 @@ static tracker_alloc_info_t *get_most_nested_alloc_segment( if (ref_parent_value) { assert(level >= 2); // release the previous reference to the parent value - critnib_release(hTracker->alloc_segments_map[level - 2], - ref_parent_value); + critnib_release( + address_space->alloc_segments_map[level - 2], + ref_parent_value); } ref_parent_value = ref_value; } else if (ref_value) { @@ -140,7 +208,7 @@ static tracker_alloc_info_t *get_most_nested_alloc_segment( // we have to release the current 'ref_value' reference // before it will be overwritten in the next critnib_find() call. assert(level >= 1); - critnib_release(hTracker->alloc_segments_map[level - 1], + critnib_release(address_space->alloc_segments_map[level - 1], ref_value); ref_value = NULL; } @@ -174,7 +242,8 @@ static tracker_alloc_info_t *get_most_nested_alloc_segment( } static umf_result_t -umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, +umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, + tracker_address_space_t *address_space, int level, umf_memory_pool_handle_t pool, const void *ptr, size_t size, uintptr_t parent_key, tracker_alloc_info_t *parent_value, @@ -216,7 +285,7 @@ umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, #endif assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); - int ret = critnib_insert(hTracker->alloc_segments_map[level], + int ret = critnib_insert(address_space->alloc_segments_map[level], (uintptr_t)ptr, value, 0); if (ret == 0) { LOG_DEBUG("memory region is added, tracker=%p, level=%i, pool=%p, " @@ -233,7 +302,7 @@ umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, (void *)parent_value->props.pool, (void *)parent_key, parent_value->props.base_size); assert(ref_parent_value); - critnib_release(hTracker->alloc_segments_map[level - 1], + critnib_release(address_space->alloc_segments_map[level - 1], ref_parent_value); } return UMF_RESULT_SUCCESS; @@ -252,6 +321,7 @@ umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, } static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, + tracker_address_space_t *address_space, umf_memory_pool_handle_t pool, const void *ptr, size_t size) { assert(ptr); @@ -272,9 +342,9 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, // in the critnib maps that contains the given 'ptr' pointer. do { assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); - found = - critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, - FIND_LE, (void *)&rkey, (void **)&rvalue, &ref_value); + found = critnib_find(address_space->alloc_segments_map[level], + (uintptr_t)ptr, FIND_LE, (void *)&rkey, + (void **)&rvalue, &ref_value); if (!found) { assert(ref_value == NULL); break; @@ -282,7 +352,8 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, if (!rvalue) { if (ref_value) { - critnib_release(hTracker->alloc_segments_map[level], ref_value); + critnib_release(address_space->alloc_segments_map[level], + ref_value); } parent_value = NULL; parent_key = 0; @@ -325,7 +396,7 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, parent_value = rvalue; if (ref_parent_value) { assert(level >= 1); - critnib_release(hTracker->alloc_segments_map[level - 1], + critnib_release(address_space->alloc_segments_map[level - 1], ref_parent_value); } ref_parent_value = ref_value; @@ -335,16 +406,18 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, } while (found && ((uintptr_t)ptr < rkey + rsize) && n_children); if (ref_value && ref_value != ref_parent_value) { - critnib_release(hTracker->alloc_segments_map[level], ref_value); + critnib_release(address_space->alloc_segments_map[level], ref_value); } - return umfMemoryTrackerAddAtLevel(hTracker, level, pool, ptr, size, - parent_key, parent_value, + return umfMemoryTrackerAddAtLevel(hTracker, address_space, level, pool, ptr, + size, parent_key, parent_value, ref_parent_value); } -static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, - const void *ptr) { +static umf_result_t +umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, + tracker_address_space_t *address_space, + const void *ptr) { assert(ptr); // TODO: there is no support for removing partial ranges (or multiple entries @@ -361,7 +434,7 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, void *ref_value = NULL; void *ref_parent_value = NULL; tracker_alloc_info_t *value = get_most_nested_alloc_segment( - hTracker, ptr, &level, &parent_key, &parent_value, &ref_value, + address_space, ptr, &level, &parent_key, &parent_value, &ref_value, &ref_parent_value, 1 /* no_children */); if (!value) { LOG_ERR("pointer %p not found in the alloc_segments_map", ptr); @@ -372,10 +445,10 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, // release the reference to the value got from get_most_nested_alloc_segment() assert(ref_value); - critnib_release(hTracker->alloc_segments_map[level], ref_value); + critnib_release(address_space->alloc_segments_map[level], ref_value); - value = critnib_remove(hTracker->alloc_segments_map[level], (uintptr_t)ptr, - &ref_value); + value = critnib_remove(address_space->alloc_segments_map[level], + (uintptr_t)ptr, &ref_value); assert(value); LOG_DEBUG("memory region removed: tracker=%p, level=%i, pool=%p, ptr=%p, " @@ -385,7 +458,7 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, // release the reference to the value got from critnib_remove() assert(ref_value); - critnib_release(hTracker->alloc_segments_map[level], ref_value); + critnib_release(address_space->alloc_segments_map[level], ref_value); if (parent_value) { size_t n_children = @@ -400,7 +473,7 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, assert(ref_parent_value); assert(level >= 1); // release the ref_parent_value got from get_most_nested_alloc_segment() - critnib_release(hTracker->alloc_segments_map[level - 1], + critnib_release(address_space->alloc_segments_map[level - 1], ref_parent_value); } @@ -493,24 +566,9 @@ umfMemoryTrackerRemoveIpcSegment(umf_memory_tracker_handle_t hTracker, return UMF_RESULT_SUCCESS; } -umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, - tracker_alloc_info_t **info) { - assert(info); - - if (UNLIKELY(ptr == NULL)) { - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (UNLIKELY(TRACKER == NULL)) { - LOG_ERR("tracker does not exist"); - return UMF_RESULT_ERROR_NOT_SUPPORTED; - } - - if (UNLIKELY(TRACKER->alloc_segments_map[0] == NULL)) { - LOG_ERR("tracker's alloc_segments_map does not exist"); - return UMF_RESULT_ERROR_NOT_SUPPORTED; - } - +static tracker_alloc_info_t * +get_alloc_info_from_address_space(tracker_address_space_t *address_space, + const void *ptr) { tracker_alloc_info_t *top_most_value = NULL; tracker_alloc_info_t *rvalue = NULL; uintptr_t rkey = 0; @@ -525,9 +583,9 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, do { assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); - found = - critnib_find(TRACKER->alloc_segments_map[level], (uintptr_t)ptr, - FIND_LE, (void *)&rkey, (void **)&rvalue, &ref_value); + found = critnib_find(address_space->alloc_segments_map[level], + (uintptr_t)ptr, FIND_LE, (void *)&rkey, + (void **)&rvalue, &ref_value); if (!found) { assert(ref_value == NULL); break; @@ -535,7 +593,8 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, if (!rvalue) { if (ref_value) { - critnib_release(TRACKER->alloc_segments_map[level], ref_value); + critnib_release(address_space->alloc_segments_map[level], + ref_value); } top_most_value = NULL; rkey = 0; @@ -555,7 +614,7 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, top_most_value = rvalue; if (ref_top_most_value) { assert(level >= 1); - critnib_release(TRACKER->alloc_segments_map[level - 1], + critnib_release(address_space->alloc_segments_map[level - 1], ref_top_most_value); } ref_top_most_value = ref_value; @@ -570,18 +629,51 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, if (!top_most_value) { if (ref_value) { - critnib_release(TRACKER->alloc_segments_map[level], ref_value); + critnib_release(address_space->alloc_segments_map[level], + ref_value); + } + return NULL; + } + + assert(ref_top_most_value); + critnib_release(address_space->alloc_segments_map[ref_level], + ref_top_most_value); + + return top_most_value; +} + +umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, + tracker_alloc_info_t **info) { + assert(info); + + if (UNLIKELY(ptr == NULL)) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (UNLIKELY(TRACKER == NULL)) { + LOG_ERR("tracker does not exist"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + tracker_alloc_info_t *latest = NULL; + tracker_address_space_t *address_space = NULL; + utils_atomic_load_acquire_ptr((void **)&TRACKER->address_spaces, + (void **)&address_space); + for (; address_space; address_space = address_space->next) { + tracker_alloc_info_t *candidate = + get_alloc_info_from_address_space(address_space, ptr); + if (candidate && (!latest || candidate->props.id > latest->props.id)) { + latest = candidate; } + } + if (!latest) { LOG_DEBUG("pointer %p not found in the tracker, TRACKER=%p", ptr, (void *)TRACKER); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - *info = top_most_value; - - assert(ref_top_most_value); - critnib_release(TRACKER->alloc_segments_map[ref_level], ref_top_most_value); + *info = latest; return UMF_RESULT_SUCCESS; } @@ -592,25 +684,35 @@ umf_result_t umfMemoryTrackerGetAllocInfoExactCount(const void *ptr, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (UNLIKELY(TRACKER == NULL || TRACKER->alloc_segments_map[0] == NULL)) { + if (UNLIKELY(TRACKER == NULL)) { return UMF_RESULT_ERROR_NOT_SUPPORTED; } *count = 0; - for (int level = 0; level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; ++level) { - uintptr_t key = 0; - tracker_alloc_info_t *value = NULL; - void *ref_value = NULL; - int found = - critnib_find(TRACKER->alloc_segments_map[level], (uintptr_t)ptr, - FIND_LE, (void *)&key, (void **)&value, &ref_value); - - if (found && value != NULL && key == (uintptr_t)ptr) { - ++*count; - } + tracker_address_space_t *address_space = NULL; + utils_atomic_load_acquire_ptr((void **)&TRACKER->address_spaces, + (void **)&address_space); + for (; address_space; address_space = address_space->next) { + for (int level = 0; level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; ++level) { + uintptr_t key = 0; + tracker_alloc_info_t *value = NULL; + void *ref_value = NULL; + int found = critnib_find(address_space->alloc_segments_map[level], + (uintptr_t)ptr, FIND_LE, (void *)&key, + (void **)&value, &ref_value); + + if (found && value != NULL && key == (uintptr_t)ptr) { + ++*count; + } - if (ref_value) { - critnib_release(TRACKER->alloc_segments_map[level], ref_value); + if (ref_value) { + critnib_release(address_space->alloc_segments_map[level], + ref_value); + } + + if (found && value != NULL && key == (uintptr_t)ptr) { + break; + } } } @@ -686,6 +788,7 @@ typedef struct ipc_cache_value_t { typedef struct umf_tracking_memory_provider_t { umf_memory_provider_handle_t hUpstream; umf_memory_tracker_handle_t hTracker; + tracker_address_space_t *address_space; umf_memory_pool_handle_t pool; critnib *ipcCache; ipc_opened_cache_handle_t hIpcMappedCache; @@ -709,7 +812,8 @@ static umf_result_t trackingAlloc(void *hProvider, size_t size, return ret; } - ret = umfMemoryTrackerAdd(p->hTracker, p->pool, ptr, size); + ret = + umfMemoryTrackerAdd(p->hTracker, p->address_space, p->pool, ptr, size); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to add allocated region to the tracker, ptr = %p, size " "= %zu, ret = %d", @@ -749,8 +853,8 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, // Find the most nested (on the highest level) entry in the map // with the `ptr` key and with no children - only such entry can be split. tracker_alloc_info_t *value = get_most_nested_alloc_segment( - provider->hTracker, ptr, &level, &parent_key, &parent_value, &ref_value, - &ref_parent_value, 1 /* no_children */); + provider->address_space, ptr, &level, &parent_key, &parent_value, + &ref_value, &ref_parent_value, 1 /* no_children */); if (!value) { LOG_ERR("region for split is not found in the tracker"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; @@ -777,9 +881,9 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, // We'll have a duplicate entry for the range [highPtr, highValue->size] but this is fine, // the value is the same anyway and we forbid removing that range concurrently - ret = umfMemoryTrackerAddAtLevel(provider->hTracker, level, provider->pool, - highPtr, secondSize, parent_key, - parent_value, ref_parent_value); + ret = umfMemoryTrackerAddAtLevel( + provider->hTracker, provider->address_space, level, provider->pool, + highPtr, secondSize, parent_key, parent_value, ref_parent_value); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to add the split region to the tracker, ptr=%p, " "size=%zu, ret=%d", @@ -798,7 +902,8 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, // update the size of the first part utils_atomic_store_release_u64((uint64_t *)&value->props.base_size, firstSize); - critnib_release(provider->hTracker->alloc_segments_map[level], ref_value); + critnib_release(provider->address_space->alloc_segments_map[level], + ref_value); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); @@ -838,8 +943,8 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, void *ref_highValue = NULL; tracker_alloc_info_t *lowValue = get_most_nested_alloc_segment( - provider->hTracker, lowPtr, &lowLevel, NULL, NULL, &ref_lowValue, NULL, - 0 /* no_children */); + provider->address_space, lowPtr, &lowLevel, NULL, NULL, &ref_lowValue, + NULL, 0 /* no_children */); if (!lowValue) { LOG_FATAL("no left value"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; @@ -852,8 +957,8 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, } tracker_alloc_info_t *highValue = get_most_nested_alloc_segment( - provider->hTracker, highPtr, &highLevel, NULL, NULL, &ref_highValue, - NULL, 0 /* no_children */); + provider->address_space, highPtr, &highLevel, NULL, NULL, + &ref_highValue, NULL, 0 /* no_children */); if (!highValue) { LOG_FATAL("no right value"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; @@ -895,13 +1000,14 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, size_t low_children = lowValue->n_children; size_t high_children = highValue->n_children; - critnib_release(provider->hTracker->alloc_segments_map[lowLevel], + critnib_release(provider->address_space->alloc_segments_map[lowLevel], ref_lowValue); - critnib_release(provider->hTracker->alloc_segments_map[highLevel], + critnib_release(provider->address_space->alloc_segments_map[highLevel], ref_highValue); - critnib_remove_release(provider->hTracker->alloc_segments_map[highLevel], - (uintptr_t)highPtr); + critnib_remove_release( + provider->address_space->alloc_segments_map[highLevel], + (uintptr_t)highPtr); LOG_DEBUG("merged memory regions (level=%i): lowPtr=%p (child=%zu), " "highPtr=%p (child=%zu), totalSize=%zu", @@ -939,7 +1045,7 @@ static umf_result_t trackingFree(void *hProvider, void *ptr, size_t size) { // could allocate the memory at address `ptr` before a call to umfMemoryTrackerRemove // resulting in inconsistent state. if (ptr) { - ret_remove = umfMemoryTrackerRemove(p->hTracker, ptr); + ret_remove = umfMemoryTrackerRemove(p->hTracker, p->address_space, ptr); if (ret_remove != UMF_RESULT_SUCCESS) { // DO NOT return an error here, because the tracking provider // cannot change behaviour of the upstream provider. @@ -975,8 +1081,8 @@ static umf_result_t trackingFree(void *hProvider, void *ptr, size_t size) { return ret; } - if (umfMemoryTrackerAdd(p->hTracker, p->pool, ptr, size) != - UMF_RESULT_SUCCESS) { + if (umfMemoryTrackerAdd(p->hTracker, p->address_space, p->pool, ptr, + size) != UMF_RESULT_SUCCESS) { LOG_ERR("cannot add memory back to the tracker, ptr=%p, size=%zu", ptr, size); } @@ -995,7 +1101,8 @@ static umf_result_t trackingInitialize(const void *params, void **ret) { *provider = *((const umf_tracking_memory_provider_t *)params); if (provider->hUpstream == NULL || provider->hTracker == NULL || - provider->pool == NULL || provider->ipcCache == NULL) { + provider->address_space == NULL || provider->pool == NULL || + provider->ipcCache == NULL) { return UMF_RESULT_ERROR_INVALID_ARGUMENT; } @@ -1008,28 +1115,33 @@ static void check_if_tracker_is_empty(umf_memory_tracker_handle_t hTracker, umf_memory_pool_handle_t pool) { size_t n_items = 0; - for (int i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { - uintptr_t last_key = 0; - uintptr_t rkey; - tracker_alloc_info_t *rvalue; - void *ref_value = NULL; - - while (1 == critnib_find(hTracker->alloc_segments_map[i], last_key, - FIND_G, &rkey, (void **)&rvalue, &ref_value)) { - if (rvalue && ((rvalue->props.pool == pool) || pool == NULL)) { - n_items++; - LOG_DEBUG( - "found abandoned allocation in the tracking provider: " - "pool=%p, ptr=%p, size=%zu", - (void *)rvalue->props.pool, (void *)rkey, - (size_t)rvalue->props.base_size); - } + for (tracker_address_space_t *address_space = hTracker->address_spaces; + address_space; address_space = address_space->next) { + for (int i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + uintptr_t last_key = 0; + uintptr_t rkey; + tracker_alloc_info_t *rvalue; + void *ref_value = NULL; + + while (1 == critnib_find(address_space->alloc_segments_map[i], + last_key, FIND_G, &rkey, (void **)&rvalue, + &ref_value)) { + if (rvalue && ((rvalue->props.pool == pool) || pool == NULL)) { + n_items++; + LOG_DEBUG( + "found abandoned allocation in the tracking provider: " + "pool=%p, ptr=%p, size=%zu", + (void *)rvalue->props.pool, (void *)rkey, + (size_t)rvalue->props.base_size); + } - if (ref_value) { - critnib_release(hTracker->alloc_segments_map[i], ref_value); - } + if (ref_value) { + critnib_release(address_space->alloc_segments_map[i], + ref_value); + } - last_key = rkey; + last_key = rkey; + } } } @@ -1095,6 +1207,14 @@ static umf_result_t trackingGetCacheLineSize(void *provider, size_t *size) { return umfMemoryProviderGetCacheLineSize(p->hUpstream, size); } +static umf_result_t +trackingGetAddressSpace(void *provider, + umf_memory_provider_address_space_t *address_space) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderGetAddressSpace(p->hUpstream, address_space); +} + static umf_result_t trackingPurgeLazy(void *provider, void *ptr, size_t size) { umf_tracking_memory_provider_t *p = (umf_tracking_memory_provider_t *)provider; @@ -1406,6 +1526,7 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .get_min_page_size = trackingGetMinPageSize, .get_recommended_page_size = trackingGetRecommendedPageSize, .get_cache_line_size = trackingGetCacheLineSize, + .get_address_space = trackingGetAddressSpace, .get_name = trackingName, .ext_purge_force = trackingPurgeForce, .ext_purge_lazy = trackingPurgeLazy, @@ -1438,6 +1559,24 @@ umf_result_t umfTrackingMemoryProviderCreate( LOG_ERR("failed, TRACKER is NULL"); return UMF_RESULT_ERROR_UNKNOWN; } + + umf_memory_provider_address_space_t address_space_id = {0}; + umf_result_t ret = + umfMemoryProviderGetAddressSpace(hUpstream, &address_space_id); + if (ret == UMF_RESULT_ERROR_NOT_SUPPORTED) { + address_space_id.namespace_token = hUpstream; + address_space_id.context = 0; + address_space_id.device = 0; + } else if (ret != UMF_RESULT_SUCCESS) { + return ret; + } + + ret = get_or_create_address_space(params.hTracker, &address_space_id, + ¶ms.address_space); + if (ret != UMF_RESULT_SUCCESS) { + return ret; + } + params.pool = hPool; params.ipcCache = critnib_new(free_ipc_cache_value, NULL); if (!params.ipcCache) { @@ -1454,8 +1593,8 @@ umf_result_t umfTrackingMemoryProviderCreate( (void *)params.pool, (void *)params.ipcCache, (void *)params.hIpcMappedCache); - umf_result_t ret = umfMemoryProviderCreate( - &UMF_TRACKING_MEMORY_PROVIDER_OPS, ¶ms, hTrackingProvider); + ret = umfMemoryProviderCreate(&UMF_TRACKING_MEMORY_PROVIDER_OPS, ¶ms, + hTrackingProvider); if (ret == UMF_RESULT_SUCCESS) { utils_atomic_increment_u64(¶ms.hTracker->pools_created); } @@ -1513,21 +1652,11 @@ umf_result_t umfMemoryTrackerCreate(umf_memory_tracker_handle_t *handle_out) { goto err_destroy_alloc_info_allocator; } - int i; - for (i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { - handle->alloc_segments_map[i] = - critnib_new(free_leaf, alloc_info_allocator); - if (!handle->alloc_segments_map[i]) { - ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - goto err_destroy_alloc_segments_map; - } - } - handle->ipc_info_allocator = umf_ba_create(sizeof(struct tracker_ipc_info_t)); if (!handle->ipc_info_allocator) { ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - goto err_destroy_alloc_segments_map; + goto err_destroy_mutex; } handle->ipc_segments_map = @@ -1537,19 +1666,13 @@ umf_result_t umfMemoryTrackerCreate(umf_memory_tracker_handle_t *handle_out) { goto err_destroy_ipc_info_allocator; } - LOG_DEBUG("tracker created, handle=%p, alloc_segments_map=%p", - (void *)handle, (void *)handle->alloc_segments_map); + LOG_DEBUG("tracker created, handle=%p", (void *)handle); *handle_out = handle; return ret; err_destroy_ipc_info_allocator: umf_ba_destroy(handle->ipc_info_allocator); -err_destroy_alloc_segments_map: - for (i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { - if (handle->alloc_segments_map[i]) { - critnib_delete(handle->alloc_segments_map[i]); - } - } +err_destroy_mutex: utils_mutex_destroy_not_free(&handle->splitMergeMutex); err_destroy_alloc_info_allocator: umf_ba_destroy(alloc_info_allocator); @@ -1574,15 +1697,13 @@ void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle) { check_if_tracker_is_empty(handle, NULL); #endif /* NDEBUG */ - // We have to zero all inner pointers, - // because the tracker handle can be copied - // and used in many places. - for (int i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { - if (handle->alloc_segments_map[i]) { - critnib_delete(handle->alloc_segments_map[i]); - handle->alloc_segments_map[i] = NULL; - } + tracker_address_space_t *address_space = handle->address_spaces; + while (address_space) { + tracker_address_space_t *next = address_space->next; + destroy_address_space(address_space); + address_space = next; } + handle->address_spaces = NULL; utils_mutex_destroy_not_free(&handle->splitMergeMutex); umf_ba_destroy(handle->alloc_info_allocator); handle->alloc_info_allocator = NULL; @@ -1601,16 +1722,14 @@ umf_result_t umfMemoryTrackerIterateAll(int (*func)(uintptr_t key, void *value, return UMF_RESULT_ERROR_NOT_SUPPORTED; } - if (UNLIKELY(TRACKER->alloc_segments_map[0] == NULL)) { - LOG_ERR("tracker's alloc_segments_map does not exist"); - return UMF_RESULT_ERROR_NOT_SUPPORTED; - } - - for (int level = 0; level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; level++) { - critnib *alloc_segment = TRACKER->alloc_segments_map[level]; - LOG_DEBUG("iterating tracker's %d segment: %p", level, - (void *)alloc_segment); - critnib_iter_all(alloc_segment, func, privdata); + for (tracker_address_space_t *address_space = TRACKER->address_spaces; + address_space; address_space = address_space->next) { + for (int level = 0; level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; level++) { + critnib *alloc_segment = address_space->alloc_segments_map[level]; + LOG_DEBUG("iterating tracker's %d segment: %p", level, + (void *)alloc_segment); + critnib_iter_all(alloc_segment, func, privdata); + } } return UMF_RESULT_SUCCESS; diff --git a/test/common/provider.hpp b/test/common/provider.hpp index 4c510cb22f..534565fc18 100644 --- a/test/common/provider.hpp +++ b/test/common/provider.hpp @@ -88,6 +88,11 @@ typedef struct provider_base_t { *size = 64; return UMF_RESULT_SUCCESS; } + umf_result_t + get_address_space([[maybe_unused]] umf_memory_provider_address_space_t + *address_space) noexcept { + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } umf_result_t get_name(const char **name) noexcept { *name = "base"; return UMF_RESULT_SUCCESS; diff --git a/test/memoryProviderAPI.cpp b/test/memoryProviderAPI.cpp index 0f1b10e841..78163b8815 100644 --- a/test/memoryProviderAPI.cpp +++ b/test/memoryProviderAPI.cpp @@ -12,9 +12,78 @@ #include "provider.hpp" #include "provider_null.h" #include "test_helpers.h" +#include using umf_test::test; +static const char TEST_ADDRESS_SPACE_NAMESPACE = 0; + +static umf_result_t +getTestAddressSpace(void *provider, + umf_memory_provider_address_space_t *address_space) { + (void)provider; + address_space->namespace_token = &TEST_ADDRESS_SPACE_NAMESPACE; + address_space->context = 42; + address_space->device = 7; + return UMF_RESULT_SUCCESS; +} + +TEST_F(test, memoryProviderGetAddressSpace) { + umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; + provider_ops.get_address_space = getTestAddressSpace; + auto provider = umf_test::wrapProviderUnique( + umf_test::createProviderChecked(&provider_ops, nullptr)); + + umf_memory_provider_address_space_t address_space = {}; + EXPECT_EQ(umfMemoryProviderGetAddressSpace(provider.get(), &address_space), + UMF_RESULT_SUCCESS); + EXPECT_EQ(address_space.namespace_token, &TEST_ADDRESS_SPACE_NAMESPACE); + EXPECT_EQ(address_space.context, 42U); + EXPECT_EQ(address_space.device, 7U); +} + +TEST_F(test, memoryProviderGetAddressSpaceNotSupported) { + auto provider = umf_test::wrapProviderUnique(nullProviderCreate()); + umf_memory_provider_address_space_t address_space = {}; + + EXPECT_EQ(umfMemoryProviderGetAddressSpace(provider.get(), &address_space), + UMF_RESULT_ERROR_NOT_SUPPORTED); + EXPECT_EQ(umfMemoryProviderGetAddressSpace(nullptr, &address_space), + UMF_RESULT_ERROR_INVALID_ARGUMENT); + EXPECT_EQ(umfMemoryProviderGetAddressSpace(provider.get(), nullptr), + UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(test, memoryProviderGetAddressSpaceHost) { + umf_os_memory_provider_params_handle_t params = nullptr; + ASSERT_EQ(umfOsMemoryProviderParamsCreate(¶ms), UMF_RESULT_SUCCESS); + + umf_memory_provider_handle_t raw_provider = nullptr; + ASSERT_EQ(umfMemoryProviderCreate(umfOsMemoryProviderOps(), params, + &raw_provider), + UMF_RESULT_SUCCESS); + auto provider = umf_test::wrapProviderUnique(raw_provider); + ASSERT_EQ(umfOsMemoryProviderParamsDestroy(params), UMF_RESULT_SUCCESS); + + umf_memory_provider_address_space_t address_space = {}; + EXPECT_EQ(umfMemoryProviderGetAddressSpace(provider.get(), &address_space), + UMF_RESULT_SUCCESS); + EXPECT_EQ(address_space.namespace_token, nullptr); + EXPECT_EQ(address_space.context, 0U); + EXPECT_EQ(address_space.device, 0U); +} + +TEST_F(test, memoryProviderGetAddressSpaceOps12Compatibility) { + umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; + provider_ops.version = UMF_MAKE_VERSION(1, 2); + auto provider = umf_test::wrapProviderUnique( + umf_test::createProviderChecked(&provider_ops, nullptr)); + + umf_memory_provider_address_space_t address_space = {}; + EXPECT_EQ(umfMemoryProviderGetAddressSpace(provider.get(), &address_space), + UMF_RESULT_ERROR_NOT_SUPPORTED); +} + TEST_F(test, memoryProviderTrace) { using calls_type = std::unordered_map; calls_type calls; diff --git a/test/poolFixtures.hpp b/test/poolFixtures.hpp index 2b42750239..3845740f4c 100644 --- a/test/poolFixtures.hpp +++ b/test/poolFixtures.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023-2025 Intel Corporation +// Copyright (C) 2023-2026 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -510,7 +510,7 @@ TEST_P(umfMultiPoolTest, memoryTracking) { } for (auto &p : ptrs) { - umfFree(std::get<0>(p)); + ASSERT_EQ(umfFree(std::get<0>(p)), UMF_RESULT_SUCCESS); } } diff --git a/test/provider_fixed_memory.cpp b/test/provider_fixed_memory.cpp index a72deda63a..9e0d1238ec 100644 --- a/test/provider_fixed_memory.cpp +++ b/test/provider_fixed_memory.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2024-2025 Intel Corporation +// Copyright (C) 2024-2026 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -511,6 +511,36 @@ TEST_F(test, params_invalid_set_memory) { ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } +TEST_F(test, params_set_host_address_space) { + constexpr size_t memory_size = 100; + char memory_buffer[memory_size]; + umf_fixed_memory_provider_params_handle_t params = nullptr; + ASSERT_EQ( + umfFixedMemoryProviderParamsCreate(memory_buffer, memory_size, ¶ms), + UMF_RESULT_SUCCESS); + + umf_memory_provider_address_space_t host_address_space = {}; + ASSERT_EQ(umfFixedMemoryProviderParamsSetAddressSpace(params, + &host_address_space), + UMF_RESULT_SUCCESS); + + umf_memory_provider_handle_t raw_provider = nullptr; + ASSERT_EQ(umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &raw_provider), + UMF_RESULT_SUCCESS); + auto provider = umf_test::wrapProviderUnique(raw_provider); + ASSERT_EQ(umfFixedMemoryProviderParamsDestroy(params), UMF_RESULT_SUCCESS); + + umf_memory_provider_address_space_t address_space = {}; + ASSERT_EQ(umfMemoryProviderGetAddressSpace(provider.get(), &address_space), + UMF_RESULT_SUCCESS); + + // By definition, the host address space has a NULL namespace token + EXPECT_EQ(address_space.namespace_token, nullptr); + EXPECT_EQ(address_space.context, 0U); + EXPECT_EQ(address_space.device, 0U); +} + // Split / merge tests TEST_P(FixedProviderTest, merge) { diff --git a/test/provider_tracking.cpp b/test/provider_tracking.cpp index 27c1c02ba1..3816a5d8ea 100644 --- a/test/provider_tracking.cpp +++ b/test/provider_tracking.cpp @@ -71,10 +71,9 @@ struct TrackingProviderTest size_t memory_size = 0; }; -static void -createPoolFromAllocation(void *ptr0, size_t size1, - umf_memory_provider_handle_t *_providerFromPtr, - umf_memory_pool_handle_t *_poolFromPtr) { +static void createPoolFromAllocation( + void *ptr0, size_t size1, umf_memory_provider_handle_t *_providerFromPtr, + umf_memory_pool_handle_t *_poolFromPtr, bool distinctAddressSpace = false) { umf_result_t umf_result; // Create provider parameters @@ -83,6 +82,15 @@ createPoolFromAllocation(void *ptr0, size_t size1, ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(params, nullptr); + static const char distinctNamespace = 0; + if (distinctAddressSpace) { + umf_memory_provider_address_space_t addressSpace = {&distinctNamespace, + 0, 0}; + umf_result = + umfFixedMemoryProviderParamsSetAddressSpace(params, &addressSpace); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + umf_memory_provider_handle_t provider1 = nullptr; umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, &provider1); @@ -155,7 +163,7 @@ TEST_P(TrackingProviderTest, identical_address_ranges) { umf_memory_provider_handle_t provider1 = nullptr; umf_memory_pool_handle_t pool1 = nullptr; - createPoolFromAllocation(ptr0, size, &provider1, &pool1); + createPoolFromAllocation(ptr0, size, &provider1, &pool1, true); void *ptr1 = umfPoolMalloc(pool1, size); ASSERT_EQ(ptr1, ptr0); @@ -192,7 +200,7 @@ TEST_P(TrackingProviderTest, identical_address_ranges_umf_free) { umf_memory_provider_handle_t provider1 = nullptr; umf_memory_pool_handle_t pool1 = nullptr; - createPoolFromAllocation(ptr0, size, &provider1, &pool1); + createPoolFromAllocation(ptr0, size, &provider1, &pool1, true); void *ptr1 = umfPoolMalloc(pool1, size); ASSERT_EQ(ptr1, ptr0); @@ -293,7 +301,7 @@ TEST_P(TrackingProviderTest, partial_overlap) { size_t size1 = size0; umf_memory_provider_handle_t provider1 = nullptr; umf_memory_pool_handle_t pool1 = nullptr; - createPoolFromAllocation(overlap_begin, size1, &provider1, &pool1); + createPoolFromAllocation(overlap_begin, size1, &provider1, &pool1, true); void *ptr1 = umfPoolMalloc(pool1, size1); EXPECT_NE(ptr1, nullptr); diff --git a/test/provider_tracking_fixture_tests.cpp b/test/provider_tracking_fixture_tests.cpp index 60ff9a1761..86c6d5ba99 100644 --- a/test/provider_tracking_fixture_tests.cpp +++ b/test/provider_tracking_fixture_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Intel Corporation +// Copyright (C) 2025-2026 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -48,6 +48,16 @@ struct provider_from_pool : public umf_test::provider_base_t { return UMF_RESULT_SUCCESS; } + umf_result_t get_address_space( + umf_memory_provider_address_space_t *address_space) noexcept { + umf_memory_provider_handle_t provider = nullptr; + umf_result_t ret = umfPoolGetMemoryProvider(pool, &provider); + if (ret != UMF_RESULT_SUCCESS) { + return ret; + } + return umfMemoryProviderGetAddressSpace(provider, address_space); + } + virtual ~provider_from_pool() { if (pool) { umfPoolDestroy(pool); diff --git a/test/utils/cpp_helpers.hpp b/test/utils/cpp_helpers.hpp index 6789b95bad..035636a3c0 100644 --- a/test/utils/cpp_helpers.hpp +++ b/test/utils/cpp_helpers.hpp @@ -105,6 +105,7 @@ template constexpr umf_memory_provider_ops_t providerOpsBase() { UMF_ASSIGN_OP(ops, T, get_recommended_page_size, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, get_min_page_size, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, get_cache_line_size, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, get_address_space, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, get_name, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, ext_purge_lazy, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, ext_purge_force, UMF_RESULT_ERROR_UNKNOWN); From 48f8058a0bfd80b137d5ef57084ebf7caccce191 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 21 Aug 2026 15:44:56 +0000 Subject: [PATCH 4/7] Update docs --- docs/config/examples.rst | 39 ++++++++++++++++++++++++++++- docs/config/introduction.rst | 17 +++++++++++++ docs/config/spelling_exceptions.txt | 4 +++ include/umf/base.h | 4 +++ include/umf/memory_provider_ops.h | 2 +- 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/docs/config/examples.rst b/docs/config/examples.rst index 28b6ab02e7..354afdfe27 100644 --- a/docs/config/examples.rst +++ b/docs/config/examples.rst @@ -145,7 +145,44 @@ Custom memory provider You can find the full examples code in the `examples/custom_file_provider/custom_file_provider.c`_ file in the UMF repository. -TODO +Custom providers using operations version 1.3 or newer should implement the +``get_address_space`` callback. The callback returns the process-local address +space used by every allocation from that provider. UMF uses this information +to distinguish allocations that have the same numerical pointer value but +belong to different memory domains. + +The ``namespace_token`` field is NULL for the host address space. For other +address spaces, UMF compares tokens by pointer value and never dereferences +them. Use the same stable token for providers whose pointers share an +address-space namespace, and different tokens for independent namespaces. The +token must remain at a stable address for as long as any provider using it +exists. Other implementations can use the address of a private static object, +for example:: + + static const char CUSTOM_ADDRESS_SPACE_NAMESPACE; + + static umf_result_t custom_get_address_space( + void *provider, + umf_memory_provider_address_space_t *address_space) { + custom_provider_t *custom = provider; + + address_space->namespace_token = &CUSTOM_ADDRESS_SPACE_NAMESPACE; + address_space->context = (uintptr_t)custom->context; + address_space->device = (uintptr_t)custom->device; + return UMF_RESULT_SUCCESS; + } + + static umf_memory_provider_ops_t custom_ops = { + .version = UMF_PROVIDER_OPS_VERSION_CURRENT, + /* Other required callbacks. */ + .get_address_space = custom_get_address_space, + }; + +Set ``context`` or ``device`` to zero when that part of the identity does not +apply. Providers that cannot determine their address space can return +``UMF_RESULT_ERROR_NOT_SUPPORTED``; tracking then treats each provider instance +as a separate address space, so providers that intentionally share an address +space should return an explicit common identity instead. CTL example ============================================================================== diff --git a/docs/config/introduction.rst b/docs/config/introduction.rst index f90b26b41a..d88f01755d 100644 --- a/docs/config/introduction.rst +++ b/docs/config/introduction.rst @@ -78,6 +78,23 @@ would be a NUMA node mask for the OS memory provider, file path for the file-backed memory provider, etc. After creation, the memory provider context can't be changed. +Address spaces +-------------- + +Numerically identical pointer values can refer to different memory when they +come from different address spaces, for example from different GPU contexts. +Memory providers identify the address space used by their allocations with a +process-local tuple consisting of a namespace token, context, and device. UMF +uses this identity when tracking allocations so that overlapping virtual +address ranges from independent memory domains do not conflict. + +The namespace token distinguishes address-space implementations. Providers +that return pointers in the process host address space use a NULL namespace +token with zero context and device identifiers. GPU and custom providers can +use their own namespace token and native context and device identifiers. The +identity is process-local and is not intended for serialization or +inter-process exchange. + Pool Allocators =============== diff --git a/docs/config/spelling_exceptions.txt b/docs/config/spelling_exceptions.txt index f329b7c6f3..0f807373d6 100644 --- a/docs/config/spelling_exceptions.txt +++ b/docs/config/spelling_exceptions.txt @@ -12,6 +12,7 @@ daxX deallocating deallocation deallocations +dereferences dev Devdax Globals @@ -39,6 +40,8 @@ memtargets middleware minBytesToKeep multithreading +namespace +namespaces Nodemask nodemask numa @@ -77,6 +80,7 @@ umfOsMemoryProviderParamsDestroy umfPool umfPoolCalloc umfPoolDestroy +umfPoolFree umfPoolGetTag umfPoolMallocUsableSize umfPoolRealloc diff --git a/include/umf/base.h b/include/umf/base.h index 1e4e620d7f..c17bfba3d9 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -63,7 +63,11 @@ typedef struct umf_memory_provider_address_space_t { /// namespace must return the same token. The token address must remain /// stable while providers using it exist. const void *namespace_token; + /// Process-local native context identifier within the namespace, or zero + /// when the namespace does not use contexts. uintptr_t context; + /// Process-local native device identifier within the context, or zero when + /// the namespace does not use devices. uintptr_t device; } umf_memory_provider_address_space_t; diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index 6e72f3de96..20ceb7a155 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -339,7 +339,7 @@ typedef struct umf_memory_provider_ops_t { // The following operations were added in ops version 1.3 /// - /// @brief Retrieve the process-local context and device identities used by + /// @brief Retrieve the process-local address-space identity used by /// allocations from the provider. /// @details The callback must return a NULL namespace token for the host /// address space. Providers that share another address-space From 66c11a81ff6b0b60302f7eb7b354335413205c95 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Thu, 27 Aug 2026 17:38:58 +0000 Subject: [PATCH 5/7] Update compatibility wflow --- .github/workflows/reusable_compatibility.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_compatibility.yml b/.github/workflows/reusable_compatibility.yml index cba1483e47..a7a14e4762 100644 --- a/.github/workflows/reusable_compatibility.yml +++ b/.github/workflows/reusable_compatibility.yml @@ -102,7 +102,7 @@ jobs: UMF_LOG: level:warning;flush:debug;output:stderr;pid:no LD_LIBRARY_PATH: ${{github.workspace}}/latest_version/build/lib/ run: | - ctest --verbose -E "test_memoryProvider|test_disjoint_pool" + ctest --verbose -E "test_memoryProvider|test_disjoint_pool|test_provider_tracking_fixture_tests" - name: Run disabled tests individually with latest UMF libs (warnings enabled) working-directory: ${{github.workspace}}/tag_version/build @@ -112,6 +112,9 @@ jobs: run: | test/test_memoryProvider --gtest_filter="-*Trace" test/test_disjoint_pool --gtest_filter="-test.internals" + if [[ -x test/test_provider_tracking_fixture_tests ]]; then + test/test_provider_tracking_fixture_tests --gtest_filter="-*memoryTracking*" + fi # Browse all folders in the examples directory, build them using the # latest UMF version, and run them, excluding those in the exclude list. @@ -245,10 +248,13 @@ jobs: run: | $env:UMF_LOG="level:warning;flush:debug;output:stderr;pid:no" cp ${{github.workspace}}/latest_version/build/bin/Debug/umf.dll ${{github.workspace}}/tag_version/build/bin/Debug/umf.dll - ctest -C Debug --verbose -E "test_memoryProvider|test_disjoint_pool" + ctest -C Debug --verbose -E "test_memoryProvider|test_disjoint_pool|test_provider_tracking_fixture_tests" $env:Path = "${{github.workspace}}/tag_version/build/bin/Debug;${{env.VCPKG_BIN_PATH}};$env:Path" test/Debug/test_memoryProvider.exe --gtest_filter="-*Trace" test/Debug/test_disjoint_pool.exe --gtest_filter="-test.internals" + if (Test-Path test/Debug/test_provider_tracking_fixture_tests.exe) { + test/Debug/test_provider_tracking_fixture_tests.exe --gtest_filter="-*memoryTracking*" + } # Browse all folders in the examples directory, build them using the # latest UMF version, and run them, excluding those in the exclude list. From 5c1e5abf5723b8d85670002f63916e779fd05340 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 28 Aug 2026 08:31:24 +0000 Subject: [PATCH 6/7] xxx --- .github/workflows/pr_push.yml | 76 +---------------------------------- 1 file changed, 1 insertion(+), 75 deletions(-) diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index fffdc7373e..09d0921793 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -19,82 +19,8 @@ permissions: jobs: CodeChecks: uses: ./.github/workflows/reusable_checks.yml - FastBuild: - name: Fast builds - needs: [CodeChecks] - uses: ./.github/workflows/reusable_fast.yml - Build: - name: Basic builds - needs: [FastBuild] - uses: ./.github/workflows/reusable_basic.yml - DevDax: - needs: [FastBuild] - uses: ./.github/workflows/reusable_dax.yml - MultiNuma: - needs: [FastBuild] - uses: ./.github/workflows/reusable_multi_numa.yml - L0: - needs: [Build] - uses: ./.github/workflows/reusable_gpu.yml - with: - provider: "LEVEL_ZERO" - runner: "L0" - shared_lib: "['ON']" - L0-BMG: - needs: [Build] - uses: ./.github/workflows/reusable_gpu.yml - with: - provider: "LEVEL_ZERO" - runner: "L0-BMG" - shared_lib: "['ON']" - CUDA: - needs: [Build] - uses: ./.github/workflows/reusable_gpu.yml - with: - provider: "CUDA" - runner: "CUDA" - shared_lib: "['ON']" - Sanitizers: - needs: [FastBuild] - uses: ./.github/workflows/reusable_sanitizers.yml - QEMU: - needs: [FastBuild] - uses: ./.github/workflows/reusable_qemu.yml - with: - short_run: true - ProxyLib: - needs: [Build] - uses: ./.github/workflows/reusable_proxy_lib.yml - Valgrind: - needs: [Build] - uses: ./.github/workflows/reusable_valgrind.yml - Coverage: - # total coverage (on upstream only) - if: github.repository == 'oneapi-src/unified-memory-framework' - needs: [Build, DevDax, L0, CUDA, MultiNuma, QEMU, ProxyLib] - uses: ./.github/workflows/reusable_coverage.yml - secrets: inherit - with: - trigger: "${{github.event_name}}" - Coverage_partial: - # partial coverage (on forks) - if: github.repository != 'oneapi-src/unified-memory-framework' - needs: [Build, QEMU, ProxyLib] - uses: ./.github/workflows/reusable_coverage.yml - CodeQL: - needs: [Build] - permissions: - contents: read - security-events: write - uses: ./.github/workflows/reusable_codeql.yml - Trivy: - needs: [Build] - permissions: - contents: read - security-events: write - uses: ./.github/workflows/reusable_trivy.yml Compatibility: - needs: [Build] + #needs: [Build] uses: ./.github/workflows/reusable_compatibility.yml strategy: matrix: From 800c5a966b8acac23fa99897d39d7dafdb80fbc1 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 28 Aug 2026 09:27:46 +0000 Subject: [PATCH 7/7] a --- .github/workflows/reusable_compatibility.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_compatibility.yml b/.github/workflows/reusable_compatibility.yml index a7a14e4762..4640d8d502 100644 --- a/.github/workflows/reusable_compatibility.yml +++ b/.github/workflows/reusable_compatibility.yml @@ -396,7 +396,7 @@ jobs: UMF_LOG: level:warning;flush:debug;output:stderr;pid:no LD_LIBRARY_PATH: ${{github.workspace}}/latest_version/build/lib/ run: | - ctest --verbose -E "test_memoryProvider|test_disjoint_pool|test_provider_level_zero|test_provider_level_zero_dlopen_global|test_provider_level_zero_dlopen_local" + ctest --verbose -E "test_memoryProvider|test_disjoint_pool|test_provider_tracking_fixture_tests|test_provider_level_zero|test_provider_level_zero_dlopen_global|test_provider_level_zero_dlopen_local" - name: Run disabled tests individually with latest UMF libs (warnings enabled) working-directory: ${{github.workspace}}/tag_version/build @@ -407,6 +407,9 @@ jobs: run: | test/test_memoryProvider --gtest_filter="-*Trace" test/test_disjoint_pool --gtest_filter="-test.internals" + if [[ -x test/test_provider_tracking_fixture_tests ]]; then + test/test_provider_tracking_fixture_tests --gtest_filter="-*memoryTracking*" + fi if [[ "$MATRIX_PROVIDER" == "LEVEL_ZERO" ]]; then test/test_provider_level_zero --gtest_filter="-*allocInvalidSize/2" test/test_provider_level_zero_dlopen_global --gtest_filter="-*allocInvalidSize/2"