From 4f5f686fb384772a1221970739ecff908dc552bc Mon Sep 17 00:00:00 2001 From: tyderian1978 Date: Tue, 3 Nov 2026 11:18:17 +1100 Subject: [PATCH] sources: look up existing IP addresses by address instead of a full scan add_update_interface() scanned every IP address in the inventory for each address a source reported, once to find the existing object and, with skip_fhrp_group_ips, once more for an FHRP group binding. Where NetBox is fed from an IPAM the inventory holds tens of thousands of IPs; with ~36,000 that scan was about a tenth of a vCenter run. SourceBase.ip_address_objects_for() keeps an index by address (without prefix length) and returns the candidates in inventory order. The inventory only appends, so the index catches up with objects added since the last lookup. The matching logic and its order are unchanged. --- module/sources/common/source_base.py | 28 +++++++++++--- tests/test_ip_address_index.py | 55 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 tests/test_ip_address_index.py diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 7538e5de..7570f64c 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -31,6 +31,24 @@ class SourceBase: init_successful = False name = None + def ip_address_objects_for(self, address: str) -> list: + """ + Return the inventory's IP address objects for an address (without prefix length), in + inventory order. The inventory only ever appends, so the index catches up with the + objects added since the last call instead of scanning every IP address per lookup. + """ + all_ips = self.inventory.get_all_items(NBIPAddress) + index = self.__dict__.setdefault("_ip_index", dict()) + indexed = self.__dict__.get("_ip_index_size", 0) + if indexed > len(all_ips): # a different inventory list: start over + index.clear() + indexed = 0 + for ip in all_ips[indexed:]: + key = grab(ip, "data.address", fallback="").split("/")[0] + index.setdefault(key, list()).append(ip) + self._ip_index_size = len(all_ips) + return index.get(address, list()) + def set_source_tag(self): self.source_tag = f"Source: {self.name}" @@ -588,13 +606,13 @@ def add_update_interface(self, interface_object, device_object, interface_data, # If skip_fhrp_group_ips is set and this address is already assigned to an FHRP # group in NetBox, leave it on the FHRP group instead of rebinding it to this - # interface (issue #445). The whole inventory is scanned for a match on this - # address, so the outcome does not depend on inventory order, and only a + # interface (issue #445). Every object for this address in the inventory is + # checked, so the outcome does not depend on inventory order, and only a # candidate in the same VRF counts, so an unrelated FHRP-group IP does not cause # this address to be skipped and every regular IP to be unbound (issue #476). if self.settings.skip_fhrp_group_ips: skip_fhrp_ip = False - for ip in self.inventory.get_all_items(NBIPAddress): + for ip in self.ip_address_objects_for(ip_object.ip.compressed): if grab(ip, "data.assigned_object_type", fallback="") != "ipam.fhrpgroup": continue if not grab(ip, "data.address", fallback="").startswith(f"{ip_object.ip.compressed}/"): @@ -612,11 +630,11 @@ def add_update_interface(self, interface_object, device_object, interface_data, this_ip_object = None skip_this_ip = False non_unique_ip_role = None - for ip in self.inventory.get_all_items(NBIPAddress): + for ip in self.ip_address_objects_for(ip_object.ip.compressed): # check if address matches (without prefix length) ip_address_string = grab(ip, "data.address", fallback="") - # not a matching address + # not a matching address (anymore) if not ip_address_string.startswith(f"{ip_object.ip.compressed}/"): continue diff --git a/tests/test_ip_address_index.py b/tests/test_ip_address_index.py new file mode 100644 index 00000000..e3e1a4c3 --- /dev/null +++ b/tests/test_ip_address_index.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +add_update_interface() looks up the existing IP address objects for every address a +source reports. SourceBase.ip_address_objects_for() answers that from an index by +address instead of scanning every IP in the inventory (tens of thousands where NetBox +is fed from an IPAM). It must return what the scan returned, in the same order, and +see objects added to the inventory after the index was built. +""" +from module.netbox.object_classes import NBIPAddress +from module.sources.common.source_base import SourceBase + + +def _source(inventory): + src = SourceBase() + src.inventory = inventory + return src + + +def _add(inventory, address): + return inventory.add_object(NBIPAddress, data={"address": address}, read_from_netbox=True) + + +def _scan(inventory, address): + # what add_update_interface() did before the index + return [ip for ip in inventory.get_all_items(NBIPAddress) + if ip.data.get("address", "").startswith(f"{address}/")] + + +def test_lookup_matches_a_full_scan_in_inventory_order(inventory): + for address in ["10.0.0.5/24", "10.0.0.50/24", "10.0.0.5/32", "2001:db8::5/64", "10.0.0.6/24"]: + _add(inventory, address) + src = _source(inventory) + + for address in ["10.0.0.5", "10.0.0.50", "10.0.0.6", "2001:db8::5", "10.0.0.7"]: + assert src.ip_address_objects_for(address) == _scan(inventory, address), address + + +def test_lookup_sees_objects_added_after_the_first_call(inventory): + first = _add(inventory, "10.0.0.5/24") + src = _source(inventory) + assert src.ip_address_objects_for("10.0.0.5") == [first] + + second = _add(inventory, "10.0.0.5/32") + _add(inventory, "10.0.0.9/24") + + assert src.ip_address_objects_for("10.0.0.5") == [first, second] + assert len(src.ip_address_objects_for("10.0.0.9")) == 1