diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 7538e5de..7570f64c 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -31,6 +31,24 @@ class SourceBase: init_successful = False name = None + def ip_address_objects_for(self, address: str) -> list: + """ + Return the inventory's IP address objects for an address (without prefix length), in + inventory order. The inventory only ever appends, so the index catches up with the + objects added since the last call instead of scanning every IP address per lookup. + """ + all_ips = self.inventory.get_all_items(NBIPAddress) + index = self.__dict__.setdefault("_ip_index", dict()) + indexed = self.__dict__.get("_ip_index_size", 0) + if indexed > len(all_ips): # a different inventory list: start over + index.clear() + indexed = 0 + for ip in all_ips[indexed:]: + key = grab(ip, "data.address", fallback="").split("/")[0] + index.setdefault(key, list()).append(ip) + self._ip_index_size = len(all_ips) + return index.get(address, list()) + def set_source_tag(self): self.source_tag = f"Source: {self.name}" @@ -588,13 +606,13 @@ def add_update_interface(self, interface_object, device_object, interface_data, # If skip_fhrp_group_ips is set and this address is already assigned to an FHRP # group in NetBox, leave it on the FHRP group instead of rebinding it to this - # interface (issue #445). The whole inventory is scanned for a match on this - # address, so the outcome does not depend on inventory order, and only a + # interface (issue #445). Every object for this address in the inventory is + # checked, so the outcome does not depend on inventory order, and only a # candidate in the same VRF counts, so an unrelated FHRP-group IP does not cause # this address to be skipped and every regular IP to be unbound (issue #476). if self.settings.skip_fhrp_group_ips: skip_fhrp_ip = False - for ip in self.inventory.get_all_items(NBIPAddress): + for ip in self.ip_address_objects_for(ip_object.ip.compressed): if grab(ip, "data.assigned_object_type", fallback="") != "ipam.fhrpgroup": continue if not grab(ip, "data.address", fallback="").startswith(f"{ip_object.ip.compressed}/"): @@ -612,11 +630,11 @@ def add_update_interface(self, interface_object, device_object, interface_data, this_ip_object = None skip_this_ip = False non_unique_ip_role = None - for ip in self.inventory.get_all_items(NBIPAddress): + for ip in self.ip_address_objects_for(ip_object.ip.compressed): # check if address matches (without prefix length) ip_address_string = grab(ip, "data.address", fallback="") - # not a matching address + # not a matching address (anymore) if not ip_address_string.startswith(f"{ip_object.ip.compressed}/"): continue diff --git a/tests/test_ip_address_index.py b/tests/test_ip_address_index.py new file mode 100644 index 00000000..e3e1a4c3 --- /dev/null +++ b/tests/test_ip_address_index.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +add_update_interface() looks up the existing IP address objects for every address a +source reports. SourceBase.ip_address_objects_for() answers that from an index by +address instead of scanning every IP in the inventory (tens of thousands where NetBox +is fed from an IPAM). It must return what the scan returned, in the same order, and +see objects added to the inventory after the index was built. +""" +from module.netbox.object_classes import NBIPAddress +from module.sources.common.source_base import SourceBase + + +def _source(inventory): + src = SourceBase() + src.inventory = inventory + return src + + +def _add(inventory, address): + return inventory.add_object(NBIPAddress, data={"address": address}, read_from_netbox=True) + + +def _scan(inventory, address): + # what add_update_interface() did before the index + return [ip for ip in inventory.get_all_items(NBIPAddress) + if ip.data.get("address", "").startswith(f"{address}/")] + + +def test_lookup_matches_a_full_scan_in_inventory_order(inventory): + for address in ["10.0.0.5/24", "10.0.0.50/24", "10.0.0.5/32", "2001:db8::5/64", "10.0.0.6/24"]: + _add(inventory, address) + src = _source(inventory) + + for address in ["10.0.0.5", "10.0.0.50", "10.0.0.6", "2001:db8::5", "10.0.0.7"]: + assert src.ip_address_objects_for(address) == _scan(inventory, address), address + + +def test_lookup_sees_objects_added_after_the_first_call(inventory): + first = _add(inventory, "10.0.0.5/24") + src = _source(inventory) + assert src.ip_address_objects_for("10.0.0.5") == [first] + + second = _add(inventory, "10.0.0.5/32") + _add(inventory, "10.0.0.9/24") + + assert src.ip_address_objects_for("10.0.0.5") == [first, second] + assert len(src.ip_address_objects_for("10.0.0.9")) == 1