sources: look up existing IP addresses by address instead of a full scan

module/sources/common/source_base.py

@@ -31,6 +31,24 @@ class SourceBase:
     init_successful = False
     name = None
 
+    def ip_address_objects_for(self, address: str) -> list:
+        """
+        Return the inventory's IP address objects for an address (without prefix length), in
+        inventory order. The inventory only ever appends, so the index catches up with the
+        objects added since the last call instead of scanning every IP address per lookup.
+        """
+        all_ips = self.inventory.get_all_items(NBIPAddress)
+        index = self.__dict__.setdefault("_ip_index", dict())
+        indexed = self.__dict__.get("_ip_index_size", 0)
+        if indexed > len(all_ips):  # a different inventory list: start over
+            index.clear()
+            indexed = 0
+        for ip in all_ips[indexed:]:
+            key = grab(ip, "data.address", fallback="").split("/")[0]
+            index.setdefault(key, list()).append(ip)
+        self._ip_index_size = len(all_ips)
+        return index.get(address, list())
+
     def set_source_tag(self):
         self.source_tag = f"Source: {self.name}"
 
@@ -588,13 +606,13 @@ def add_update_interface(self, interface_object, device_object, interface_data,
 
             # If skip_fhrp_group_ips is set and this address is already assigned to an FHRP
             # group in NetBox, leave it on the FHRP group instead of rebinding it to this
-            # interface (issue #445). The whole inventory is scanned for a match on this
-            # address, so the outcome does not depend on inventory order, and only a
+            # interface (issue #445). Every object for this address in the inventory is
+            # checked, so the outcome does not depend on inventory order, and only a
             # candidate in the same VRF counts, so an unrelated FHRP-group IP does not cause
             # this address to be skipped and every regular IP to be unbound (issue #476).
             if self.settings.skip_fhrp_group_ips:
                 skip_fhrp_ip = False
-                for ip in self.inventory.get_all_items(NBIPAddress):
+                for ip in self.ip_address_objects_for(ip_object.ip.compressed):
                     if grab(ip, "data.assigned_object_type", fallback="") != "ipam.fhrpgroup":
                         continue
                     if not grab(ip, "data.address", fallback="").startswith(f"{ip_object.ip.compressed}/"):
@@ -612,11 +630,11 @@ def add_update_interface(self, interface_object, device_object, interface_data,
             this_ip_object = None
             skip_this_ip = False
             non_unique_ip_role = None
-            for ip in self.inventory.get_all_items(NBIPAddress):
+            for ip in self.ip_address_objects_for(ip_object.ip.compressed):
                 # check if address matches (without prefix length)
                 ip_address_string = grab(ip, "data.address", fallback="")
 
-                # not a matching address
+                # not a matching address (anymore)
                 if not ip_address_string.startswith(f"{ip_object.ip.compressed}/"):
                     continue
 

tests/test_ip_address_index.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#  Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved.
+#
+#  netbox-sync.py
+#
+#  This work is licensed under the terms of the MIT license.
+#  For a copy, see file LICENSE.txt included in this
+#  repository or visit: <https://opensource.org/licenses/MIT>.
+
+"""
+add_update_interface() looks up the existing IP address objects for every address a
+source reports. SourceBase.ip_address_objects_for() answers that from an index by
+address instead of scanning every IP in the inventory (tens of thousands where NetBox
+is fed from an IPAM). It must return what the scan returned, in the same order, and
+see objects added to the inventory after the index was built.
+"""
+from module.netbox.object_classes import NBIPAddress
+from module.sources.common.source_base import SourceBase
+
+
+def _source(inventory):
+    src = SourceBase()
+    src.inventory = inventory
+    return src
+
+
+def _add(inventory, address):
+    return inventory.add_object(NBIPAddress, data={"address": address}, read_from_netbox=True)
+
+
+def _scan(inventory, address):
+    # what add_update_interface() did before the index
+    return [ip for ip in inventory.get_all_items(NBIPAddress)
+            if ip.data.get("address", "").startswith(f"{address}/")]
+
+
+def test_lookup_matches_a_full_scan_in_inventory_order(inventory):
+    for address in ["10.0.0.5/24", "10.0.0.50/24", "10.0.0.5/32", "2001:db8::5/64", "10.0.0.6/24"]:
+        _add(inventory, address)
+    src = _source(inventory)
+
+    for address in ["10.0.0.5", "10.0.0.50", "10.0.0.6", "2001:db8::5", "10.0.0.7"]:
+        assert src.ip_address_objects_for(address) == _scan(inventory, address), address
+
+
+def test_lookup_sees_objects_added_after_the_first_call(inventory):
+    first = _add(inventory, "10.0.0.5/24")
+    src = _source(inventory)
+    assert src.ip_address_objects_for("10.0.0.5") == [first]
+
+    second = _add(inventory, "10.0.0.5/32")
+    _add(inventory, "10.0.0.9/24")
+
+    assert src.ip_address_objects_for("10.0.0.5") == [first, second]
+    assert len(src.ip_address_objects_for("10.0.0.9")) == 1