Skip to content

[Utilities.CleverDicts] add identity storage mode #3117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
blegat wants to merge 2 commits into
base: master
Choose a base branch
from
Open

[Utilities.CleverDicts] add identity storage mode #3117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1bc96e1
[Utilities.CleverDicts] add identity storage mode
blegat Oct 27, 2026
ed138cd
Refactor CleverDicts for improved vector handling
odow Oct 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 92 additions & 8 deletions src/Utilities/CleverDicts.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,14 @@ indices. Note that the integers are `Int64`s.
Provided no keys are deleted, the backing storage is a `Vector{V}`. Once a key
has been deleted, the backing storage switches to an `OrderedDict{K, V}`.

If `K == V` and every item set so far maps the key of index `i` to itself for
`i` in `1:n`, no backing storage is used: only `n` is recorded. This is the
case, for example, for the variable map of an `Utilities.IndexMap`
when the destination model creates its variables in the same order as the
source model. Use `is_identity` to query whether this is the case.
As soon as a key is deleted or an item that is not the identity is set, the
backing storage switches to a `Vector{V}`.

Use the `add_item` to enforce adding item in sequence. Once an object is added
out of order `add_item` does not work anymore and the storage is switched to
`OrderedDict{K, V}` if it is not one already.
Expand Down Expand Up @@ -82,6 +90,9 @@ mutable struct CleverDict{K,V,F<:Function,I<:Function} <: AbstractDict{K,V}
hash::F
inverse_hash::I
is_dense::Bool
# If `true`, the dict maps `inverse_hash(i)` to itself for `i` in
# `1:last_index` and both `vector` and `dict` are empty.
is_identity::Bool
vector::Vector{V}
dict::OrderedDict{K,V}
function CleverDict{K,V}(
Expand All @@ -93,6 +104,7 @@ mutable struct CleverDict{K,V,F<:Function,I<:Function} <: AbstractDict{K,V}
hash,
inverse_hash,
true,
K === V,
V[],
OrderedDict{K,V}(),
)
Expand All @@ -101,6 +113,17 @@ end

_is_dense(c::CleverDict) = c.is_dense

"""
is_identity(c::CleverDict)::Bool

Return `true` if `c` is known to map each of its keys to itself and its keys
are the keys of indices `1:length(c)`.

A return value of `false` does not imply that `c` is not the identity, for
example, if a key was deleted and then set back again.
"""
is_identity(c::CleverDict) = c.is_identity

@joaquimg joaquimg Oct 27, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe just keep this internal?

@odow odow Oct 27, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

He wants it public for #3118


function _inverse_hash(c::CleverDict{K}, index::Integer) where {K}
return c.inverse_hash(Int64(index))::K
end
Expand All @@ -112,6 +135,14 @@ function _inverse_hash(
return index_to_key(K, Int64(index))::K
end

# Switch from the identity storage to the dense vector storage.
function _materialize_identity!(c::CleverDict{K,V}) where {K,V}
@assert is_identity(c)
c.vector = V[_inverse_hash(c, i) for i in 1:c.last_index]
c.is_identity = false
return
end

"""
add_item(c::CleverDict{K, V}, val::Val)::K where {K, V}

Expand All @@ -129,27 +160,40 @@ function add_item(c::CleverDict{K,V}, val::V)::K where {K,V}
return key
end

function Base.empty!(c::CleverDict)
function Base.empty!(c::CleverDict{K,V}) where {K,V}
empty!(c.vector)
empty!(c.dict)
c.last_index = 0
c.is_dense = true
c.is_identity = K === V
return
end

function Base.length(c::CleverDict)
return _is_dense(c) ? length(c.vector) : length(c.dict)
if is_identity(c)
return Int(c.last_index)
elseif _is_dense(c)
return length(c.vector)
end
return length(c.dict)
end

function Base.haskey(c::CleverDict{K}, key::K) where {K}
if _is_dense(c)
if is_identity(c)
return 1 <= c.hash(key)::Int64 <= c.last_index
elseif _is_dense(c)
return 1 <= c.hash(key)::Int64 <= length(c.vector)
end
return haskey(c.dict, key)
end

function Base.get(c::CleverDict, key, default)
if _is_dense(c)
if is_identity(c)
if !haskey(c, key)
return default
end
return key
elseif _is_dense(c)
if !haskey(c, key)
return default
end
Expand All @@ -159,7 +203,12 @@ function Base.get(c::CleverDict, key, default)
end

function Base.getindex(c::CleverDict{K,V}, key::K) where {K,V}
if _is_dense(c)
if is_identity(c)
if !haskey(c, key)
throw(KeyError(key))
end
return key
elseif _is_dense(c)
if !haskey(c, key)
throw(KeyError(key))
end
Expand All @@ -170,6 +219,13 @@ end

function Base.setindex!(c::CleverDict{K,V}, value::V, key::K)::V where {K,V}
h = c.hash(key)::Int64
if is_identity(c)
if value == key && 1 <= h <= c.last_index + 1
c.last_index = max(c.last_index, h)
return value
end
_materialize_identity!(c)
end
if c.last_index != -1
if h == c.last_index + 1
c.last_index = h
Expand Down Expand Up @@ -208,6 +264,9 @@ function _rehash(c::CleverDict{K}) where {K}
end

function Base.delete!(c::CleverDict{K}, k::K) where {K}
if is_identity(c)
_materialize_identity!(c)
end
if _is_dense(c)
_rehash(c)
end
Expand All @@ -226,6 +285,9 @@ function Base.getindex(c::CleverDict{K,V}, index::LinearIndex)::V where {K,V}
if !(1 <= index.i <= length(c))
throw(KeyError(index))
end
if is_identity(c)
return _inverse_hash(c, index.i)
end
# Get the `index` linear element. If `c.vector` is currently `nothing`
# (that is, there has been a deletion), rebuild `c.vector`. This is a
# trade-off: We could ensure `c.vector` is always updated, but this requires
Expand All @@ -248,13 +310,20 @@ function Base.getindex(c::CleverDict{K,V}, index::LinearIndex)::V where {K,V}
end

function Base.isempty(c::CleverDict)
return _is_dense(c) ? isempty(c.vector) : isempty(c.dict)
if is_identity(c)
return iszero(c.last_index)
elseif _is_dense(c)
return isempty(c.vector)
end
return isempty(c.dict)
end

Base.haskey(::CleverDict, key) = false

function Base.iterate(c::CleverDict{K,V}) where {K,V}
if _is_dense(c)
if is_identity(c)
return iterate(c, 1)
elseif _is_dense(c)
itr = iterate(c.vector)
if itr === nothing
return
Expand All @@ -269,7 +338,13 @@ function Base.iterate(c::CleverDict{K,V}) where {K,V}
end

function Base.iterate(c::CleverDict{K,V}, s::Int) where {K,V}
if _is_dense(c)
if is_identity(c)
if s > c.last_index
return
end
key = _inverse_hash(c, s)
return key => convert(V, key), s + 1
elseif _is_dense(c)
itr = iterate(c.vector, s)
if itr === nothing
return
Expand All @@ -287,6 +362,9 @@ end
# on `Base`s fallback that uses the iterator protocol

function Base.sizehint!(c::CleverDict{K,V}, n) where {K,V}
if is_identity(c)
return
end
if _is_dense(c)
sizehint!(c.vector, n)
else
Expand All @@ -296,6 +374,9 @@ function Base.sizehint!(c::CleverDict{K,V}, n) where {K,V}
end

function Base.resize!(c::CleverDict{K,V}, n) where {K,V}
if is_identity(c)
_materialize_identity!(c)
end
if _is_dense(c)
if n < length(c.vector)
error(
Expand All @@ -312,6 +393,9 @@ end
# TODO `map!(f, values(dict::AbstractDict))` requires Julia 1.2 or later,
# use `map_values` once we drop Julia 1.1 and earlier.
function map_values!(f::Function, d::CleverDict)
if is_identity(d)
_materialize_identity!(d)
end
if _is_dense(d)
map!(f, d.vector, d.vector)
else
Expand Down
118 changes: 118 additions & 0 deletions test/Utilities/test_CleverDicts.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,124 @@ function test_resize()
return
end

function test_identity()
d = CleverDicts.CleverDict{MOI.VariableIndex,MOI.VariableIndex}()
@test CleverDicts.is_identity(d)
@test isempty(d)
@test iterate(d) === nothing
x = MOI.VariableIndex.(1:3)
for xi in x
d[xi] = xi
end
d[x[2]] = x[2]
@test CleverDicts.is_identity(d)
@test isempty(d.vector)
@test isempty(d.dict)
@test !isempty(d)
@test length(d) == 3
@test haskey(d, x[3])
@test !haskey(d, MOI.VariableIndex(4))
@test !haskey(d, MOI.VariableIndex(0))
@test d[x[2]] == x[2]
@test_throws KeyError(MOI.VariableIndex(4)) d[MOI.VariableIndex(4)]
@test get(d, x[1], nothing) == x[1]
@test get(d, MOI.VariableIndex(4), nothing) === nothing
@test d[CleverDicts.LinearIndex(3)] == x[3]
@test_throws KeyError(CleverDicts.LinearIndex(4)) d[CleverDicts.LinearIndex(
4,
)]
@test collect(d) == [xi => xi for xi in x]
@test collect(keys(d)) == x
@test collect(values(d)) == x
sizehint!(d, 10)
@test CleverDicts.is_identity(d)
@test CleverDicts.add_item(d, MOI.VariableIndex(4)) == MOI.VariableIndex(4)
@test CleverDicts.is_identity(d)
@test length(d) == 4
empty!(d)
@test CleverDicts.is_identity(d)
@test isempty(d)
return
end

function test_identity_not_same_types()
d = CleverDicts.CleverDict{MOI.VariableIndex,Int}()
@test !CleverDicts.is_identity(d)
empty!(d)
@test !CleverDicts.is_identity(d)
return
end

function _identity_dict(n)
d = CleverDicts.CleverDict{MOI.VariableIndex,MOI.VariableIndex}()
for i in 1:n
d[MOI.VariableIndex(i)] = MOI.VariableIndex(i)
end
@test CleverDicts.is_identity(d)
return d
end

function _test_identity_materialized(d, n)
@test !CleverDicts.is_identity(d)
for i in 1:n
x = MOI.VariableIndex(i)
@test d[x] == x
end
return
end

function test_identity_set_not_identity()
d = _identity_dict(3)
x = MOI.VariableIndex(2)
d[x] = MOI.VariableIndex(3)
@test !CleverDicts.is_identity(d)
@test d[x] == MOI.VariableIndex(3)
@test d.vector == MOI.VariableIndex.([1, 3, 3])
d = _identity_dict(3)
x = MOI.VariableIndex(4)
d[x] = MOI.VariableIndex(1)
_test_identity_materialized(d, 3)
@test d[x] == MOI.VariableIndex(1)
@test length(d.vector) == 4
return
end

function test_identity_set_out_of_order()
d = _identity_dict(3)
x = MOI.VariableIndex(5)
d[x] = x
_test_identity_materialized(d, 3)
@test d[x] == x
@test !CleverDicts._is_dense(d)
@test length(d) == 4
return
end

function test_identity_delete()
d = _identity_dict(3)
delete!(d, MOI.VariableIndex(2))
@test !CleverDicts.is_identity(d)
@test collect(keys(d)) == MOI.VariableIndex.([1, 3])
@test d[CleverDicts.LinearIndex(2)] == MOI.VariableIndex(3)
return
end

function test_identity_resize()
d = _identity_dict(2)
resize!(d, 3)
_test_identity_materialized(d, 2)
@test length(d.vector) == 3
return
end

function test_identity_map_values()
d = _identity_dict(2)
CleverDicts.map_values!(x -> MOI.VariableIndex(x.value + 1), d)
@test !CleverDicts.is_identity(d)
@test d.vector == MOI.VariableIndex.([2, 3])
return
end

end # module

TestCleverDicts.runtests()
Loading