From 1bc96e11a59f7cb674a3532d898c280152b0ec24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Tue, 27 Oct 2026 12:42:35 +0100 Subject: [PATCH 1/2] [Utilities.CleverDicts] add identity storage mode When K == V and the dict maps the keys of indices 1:n to themselves, only n is recorded and no backing storage is used. This is typically the case for the variable map of an IndexMap when the destination creates its variables in the same order as the source. --- src/Utilities/CleverDicts.jl | 89 +++++++++++++++++++++- test/Utilities/test_CleverDicts.jl | 118 +++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 1 deletion(-) diff --git a/src/Utilities/CleverDicts.jl b/src/Utilities/CleverDicts.jl index d12a793867..1a1916af8d 100644 --- a/src/Utilities/CleverDicts.jl +++ b/src/Utilities/CleverDicts.jl @@ -52,6 +52,14 @@ indices. Note that the integers are `Int64`s. Provided no keys are deleted, the backing storage is a `Vector{V}`. Once a key has been deleted, the backing storage switches to an `OrderedDict{K, V}`. +If `K == V` and every item set so far maps the key of index `i` to itself for +`i` in `1:n`, no backing storage is used: only `n` is recorded. This is the +case, for example, for the variable map of an `Utilities.IndexMap` +when the destination model creates its variables in the same order as the +source model. Use `is_identity` to query whether this is the case. +As soon as a key is deleted or an item that is not the identity is set, the +backing storage switches to a `Vector{V}`. + Use the `add_item` to enforce adding item in sequence. Once an object is added out of order `add_item` does not work anymore and the storage is switched to `OrderedDict{K, V}` if it is not one already. @@ -82,6 +90,9 @@ mutable struct CleverDict{K,V,F<:Function,I<:Function} <: AbstractDict{K,V} hash::F inverse_hash::I is_dense::Bool + # If `true`, the dict maps `inverse_hash(i)` to itself for `i` in + # `1:last_index` and both `vector` and `dict` are empty. + is_identity::Bool vector::Vector{V} dict::OrderedDict{K,V} function CleverDict{K,V}( @@ -93,6 +104,7 @@ mutable struct CleverDict{K,V,F<:Function,I<:Function} <: AbstractDict{K,V} hash, inverse_hash, true, + K === V, V[], OrderedDict{K,V}(), ) @@ -101,6 +113,17 @@ end _is_dense(c::CleverDict) = c.is_dense +""" + is_identity(c::CleverDict)::Bool + +Return `true` if `c` is known to map each of its keys to itself and its keys +are the keys of indices `1:length(c)`. + +A return value of `false` does not imply that `c` is not the identity, for +example, if a key was deleted and then set back again. +""" +is_identity(c::CleverDict) = c.is_identity + function _inverse_hash(c::CleverDict{K}, index::Integer) where {K} return c.inverse_hash(Int64(index))::K end @@ -112,6 +135,19 @@ function _inverse_hash( return index_to_key(K, Int64(index))::K end +# Switch from the identity storage to the dense vector storage. +function _materialize_identity!(c::CleverDict{K,V}) where {K,V} + @assert is_identity(c) + n = c.last_index + vector = Vector{V}(undef, n) + for i in 1:n + vector[i] = _inverse_hash(c, i) + end + c.vector = vector + c.is_identity = false + return +end + """ add_item(c::CleverDict{K, V}, val::Val)::K where {K, V} @@ -129,19 +165,26 @@ function add_item(c::CleverDict{K,V}, val::V)::K where {K,V} return key end -function Base.empty!(c::CleverDict) +function Base.empty!(c::CleverDict{K,V}) where {K,V} empty!(c.vector) empty!(c.dict) c.last_index = 0 c.is_dense = true + c.is_identity = K === V return end function Base.length(c::CleverDict) + if is_identity(c) + return Int(c.last_index) + end return _is_dense(c) ? length(c.vector) : length(c.dict) end function Base.haskey(c::CleverDict{K}, key::K) where {K} + if is_identity(c) + return 1 <= c.hash(key)::Int64 <= c.last_index + end if _is_dense(c) return 1 <= c.hash(key)::Int64 <= length(c.vector) end @@ -149,6 +192,9 @@ function Base.haskey(c::CleverDict{K}, key::K) where {K} end function Base.get(c::CleverDict, key, default) + if is_identity(c) + return haskey(c, key) ? key : default + end if _is_dense(c) if !haskey(c, key) return default @@ -159,6 +205,12 @@ function Base.get(c::CleverDict, key, default) end function Base.getindex(c::CleverDict{K,V}, key::K) where {K,V} + if is_identity(c) + if !haskey(c, key) + throw(KeyError(key)) + end + return key + end if _is_dense(c) if !haskey(c, key) throw(KeyError(key)) @@ -170,6 +222,13 @@ end function Base.setindex!(c::CleverDict{K,V}, value::V, key::K)::V where {K,V} h = c.hash(key)::Int64 + if is_identity(c) + if value == key && 1 <= h <= c.last_index + 1 + c.last_index = max(c.last_index, h) + return value + end + _materialize_identity!(c) + end if c.last_index != -1 if h == c.last_index + 1 c.last_index = h @@ -208,6 +267,9 @@ function _rehash(c::CleverDict{K}) where {K} end function Base.delete!(c::CleverDict{K}, k::K) where {K} + if is_identity(c) + _materialize_identity!(c) + end if _is_dense(c) _rehash(c) end @@ -226,6 +288,9 @@ function Base.getindex(c::CleverDict{K,V}, index::LinearIndex)::V where {K,V} if !(1 <= index.i <= length(c)) throw(KeyError(index)) end + if is_identity(c) + return _inverse_hash(c, index.i) + end # Get the `index` linear element. If `c.vector` is currently `nothing` # (that is, there has been a deletion), rebuild `c.vector`. This is a # trade-off: We could ensure `c.vector` is always updated, but this requires @@ -248,12 +313,18 @@ function Base.getindex(c::CleverDict{K,V}, index::LinearIndex)::V where {K,V} end function Base.isempty(c::CleverDict) + if is_identity(c) + return iszero(c.last_index) + end return _is_dense(c) ? isempty(c.vector) : isempty(c.dict) end Base.haskey(::CleverDict, key) = false function Base.iterate(c::CleverDict{K,V}) where {K,V} + if is_identity(c) + return iterate(c, 1) + end if _is_dense(c) itr = iterate(c.vector) if itr === nothing @@ -269,6 +340,13 @@ function Base.iterate(c::CleverDict{K,V}) where {K,V} end function Base.iterate(c::CleverDict{K,V}, s::Int) where {K,V} + if is_identity(c) + if s > c.last_index + return + end + key = _inverse_hash(c, s) + return key => convert(V, key), s + 1 + end if _is_dense(c) itr = iterate(c.vector, s) if itr === nothing @@ -287,6 +365,9 @@ end # on `Base`s fallback that uses the iterator protocol function Base.sizehint!(c::CleverDict{K,V}, n) where {K,V} + if is_identity(c) + return + end if _is_dense(c) sizehint!(c.vector, n) else @@ -296,6 +377,9 @@ function Base.sizehint!(c::CleverDict{K,V}, n) where {K,V} end function Base.resize!(c::CleverDict{K,V}, n) where {K,V} + if is_identity(c) + _materialize_identity!(c) + end if _is_dense(c) if n < length(c.vector) error( @@ -312,6 +396,9 @@ end # TODO `map!(f, values(dict::AbstractDict))` requires Julia 1.2 or later, # use `map_values` once we drop Julia 1.1 and earlier. function map_values!(f::Function, d::CleverDict) + if is_identity(d) + _materialize_identity!(d) + end if _is_dense(d) map!(f, d.vector, d.vector) else diff --git a/test/Utilities/test_CleverDicts.jl b/test/Utilities/test_CleverDicts.jl index db8a092ad2..7c9cc4c96a 100644 --- a/test/Utilities/test_CleverDicts.jl +++ b/test/Utilities/test_CleverDicts.jl @@ -287,6 +287,124 @@ function test_resize() return end +function test_identity() + d = CleverDicts.CleverDict{MOI.VariableIndex,MOI.VariableIndex}() + @test CleverDicts.is_identity(d) + @test isempty(d) + @test iterate(d) === nothing + x = MOI.VariableIndex.(1:3) + for xi in x + d[xi] = xi + end + d[x[2]] = x[2] + @test CleverDicts.is_identity(d) + @test isempty(d.vector) + @test isempty(d.dict) + @test !isempty(d) + @test length(d) == 3 + @test haskey(d, x[3]) + @test !haskey(d, MOI.VariableIndex(4)) + @test !haskey(d, MOI.VariableIndex(0)) + @test d[x[2]] == x[2] + @test_throws KeyError(MOI.VariableIndex(4)) d[MOI.VariableIndex(4)] + @test get(d, x[1], nothing) == x[1] + @test get(d, MOI.VariableIndex(4), nothing) === nothing + @test d[CleverDicts.LinearIndex(3)] == x[3] + @test_throws KeyError(CleverDicts.LinearIndex(4)) d[CleverDicts.LinearIndex( + 4, + )] + @test collect(d) == [xi => xi for xi in x] + @test collect(keys(d)) == x + @test collect(values(d)) == x + sizehint!(d, 10) + @test CleverDicts.is_identity(d) + @test CleverDicts.add_item(d, MOI.VariableIndex(4)) == MOI.VariableIndex(4) + @test CleverDicts.is_identity(d) + @test length(d) == 4 + empty!(d) + @test CleverDicts.is_identity(d) + @test isempty(d) + return +end + +function test_identity_not_same_types() + d = CleverDicts.CleverDict{MOI.VariableIndex,Int}() + @test !CleverDicts.is_identity(d) + empty!(d) + @test !CleverDicts.is_identity(d) + return +end + +function _identity_dict(n) + d = CleverDicts.CleverDict{MOI.VariableIndex,MOI.VariableIndex}() + for i in 1:n + d[MOI.VariableIndex(i)] = MOI.VariableIndex(i) + end + @test CleverDicts.is_identity(d) + return d +end + +function _test_identity_materialized(d, n) + @test !CleverDicts.is_identity(d) + for i in 1:n + x = MOI.VariableIndex(i) + @test d[x] == x + end + return +end + +function test_identity_set_not_identity() + d = _identity_dict(3) + x = MOI.VariableIndex(2) + d[x] = MOI.VariableIndex(3) + @test !CleverDicts.is_identity(d) + @test d[x] == MOI.VariableIndex(3) + @test d.vector == MOI.VariableIndex.([1, 3, 3]) + d = _identity_dict(3) + x = MOI.VariableIndex(4) + d[x] = MOI.VariableIndex(1) + _test_identity_materialized(d, 3) + @test d[x] == MOI.VariableIndex(1) + @test length(d.vector) == 4 + return +end + +function test_identity_set_out_of_order() + d = _identity_dict(3) + x = MOI.VariableIndex(5) + d[x] = x + _test_identity_materialized(d, 3) + @test d[x] == x + @test !CleverDicts._is_dense(d) + @test length(d) == 4 + return +end + +function test_identity_delete() + d = _identity_dict(3) + delete!(d, MOI.VariableIndex(2)) + @test !CleverDicts.is_identity(d) + @test collect(keys(d)) == MOI.VariableIndex.([1, 3]) + @test d[CleverDicts.LinearIndex(2)] == MOI.VariableIndex(3) + return +end + +function test_identity_resize() + d = _identity_dict(2) + resize!(d, 3) + _test_identity_materialized(d, 2) + @test length(d.vector) == 3 + return +end + +function test_identity_map_values() + d = _identity_dict(2) + CleverDicts.map_values!(x -> MOI.VariableIndex(x.value + 1), d) + @test !CleverDicts.is_identity(d) + @test d.vector == MOI.VariableIndex.([2, 3]) + return +end + end # module TestCleverDicts.runtests() From ed138cdf46a91919e3b279353951b177b1987602 Mon Sep 17 00:00:00 2001 From: Oscar Dowson Date: Wed, 28 Oct 2026 09:34:19 +1300 Subject: [PATCH 2/2] Refactor CleverDicts for improved vector handling --- src/Utilities/CleverDicts.jl | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/Utilities/CleverDicts.jl b/src/Utilities/CleverDicts.jl index 1a1916af8d..7d46378ff9 100644 --- a/src/Utilities/CleverDicts.jl +++ b/src/Utilities/CleverDicts.jl @@ -138,12 +138,7 @@ end # Switch from the identity storage to the dense vector storage. function _materialize_identity!(c::CleverDict{K,V}) where {K,V} @assert is_identity(c) - n = c.last_index - vector = Vector{V}(undef, n) - for i in 1:n - vector[i] = _inverse_hash(c, i) - end - c.vector = vector + c.vector = V[_inverse_hash(c, i) for i in 1:c.last_index] c.is_identity = false return end @@ -177,15 +172,16 @@ end function Base.length(c::CleverDict) if is_identity(c) return Int(c.last_index) + elseif _is_dense(c) + return length(c.vector) end - return _is_dense(c) ? length(c.vector) : length(c.dict) + return length(c.dict) end function Base.haskey(c::CleverDict{K}, key::K) where {K} if is_identity(c) return 1 <= c.hash(key)::Int64 <= c.last_index - end - if _is_dense(c) + elseif _is_dense(c) return 1 <= c.hash(key)::Int64 <= length(c.vector) end return haskey(c.dict, key) @@ -193,9 +189,11 @@ end function Base.get(c::CleverDict, key, default) if is_identity(c) - return haskey(c, key) ? key : default - end - if _is_dense(c) + if !haskey(c, key) + return default + end + return key + elseif _is_dense(c) if !haskey(c, key) return default end @@ -210,8 +208,7 @@ function Base.getindex(c::CleverDict{K,V}, key::K) where {K,V} throw(KeyError(key)) end return key - end - if _is_dense(c) + elseif _is_dense(c) if !haskey(c, key) throw(KeyError(key)) end @@ -315,8 +312,10 @@ end function Base.isempty(c::CleverDict) if is_identity(c) return iszero(c.last_index) + elseif _is_dense(c) + return isempty(c.vector) end - return _is_dense(c) ? isempty(c.vector) : isempty(c.dict) + return isempty(c.dict) end Base.haskey(::CleverDict, key) = false @@ -324,8 +323,7 @@ Base.haskey(::CleverDict, key) = false function Base.iterate(c::CleverDict{K,V}) where {K,V} if is_identity(c) return iterate(c, 1) - end - if _is_dense(c) + elseif _is_dense(c) itr = iterate(c.vector) if itr === nothing return @@ -346,8 +344,7 @@ function Base.iterate(c::CleverDict{K,V}, s::Int) where {K,V} end key = _inverse_hash(c, s) return key => convert(V, key), s + 1 - end - if _is_dense(c) + elseif _is_dense(c) itr = iterate(c.vector, s) if itr === nothing return