From ec1ba92f9e1c92a1d4417c462ed18901ef58593e Mon Sep 17 00:00:00 2001 From: tyderian1978 Date: Tue, 3 Nov 2026 11:18:17 +1100 Subject: [PATCH] vmware: fetch each vCenter tag and category once per run get_vmware_object_tags() fetched every tag attached to an object, and its category when tag_name_include_category is set, from the vSphere tagging API, for every object. Most objects share a few hundred tags, so a run fetched the same tags thousands of times, one HTTPS request each. Tags and category names are now kept by id for the run, per source (tag ids are only unique within one vCenter). The tags returned are unchanged. --- module/sources/vmware/connection.py | 15 +++- tests/test_vmware_tag_cache.py | 102 ++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 tests/test_vmware_tag_cache.py diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index cde3c53..a62b7fa 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1326,11 +1326,19 @@ def get_vmware_object_tags(self, obj): log.error(f"Unable to retrieve vCenter tags for '{obj.name}': {e}") return + # vCenter tags and categories by id: most objects share a few hundred tags, so each + # is fetched once per run (per source) instead of once per object it's attached to + tag_cache = self.__dict__.setdefault("_tag_cache", dict()) + tag_category_name_cache = self.__dict__.setdefault("_tag_category_name_cache", dict()) + for tag_id in object_tag_ids: # noinspection PyBroadException try: - tag = self.tag_session.tagging.Tag.get(tag_id) # store the object + tag = tag_cache.get(tag_id) + if tag is None: + tag = self.tag_session.tagging.Tag.get(tag_id) # store the object + tag_cache[tag_id] = tag tag_name = tag.name tag_description = tag.description except Exception as e: @@ -1341,7 +1349,10 @@ def get_vmware_object_tags(self, obj): if bool(self.settings.tag_name_include_category) is True: # noinspection PyBroadException try: - category_name = self.tag_session.tagging.Category.get(tag.category_id).name + category_name = tag_category_name_cache.get(tag.category_id) + if category_name is None: + category_name = self.tag_session.tagging.Category.get(tag.category_id).name + tag_category_name_cache[tag.category_id] = category_name except Exception as e: log.debug(f"Unable to retrieve category of vCenter tag '{tag_name}': {e}") diff --git a/tests/test_vmware_tag_cache.py b/tests/test_vmware_tag_cache.py new file mode 100644 index 0000000..3b6cd56 --- /dev/null +++ b/tests/test_vmware_tag_cache.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +Most vCenter objects share a few hundred tags, so get_vmware_object_tags() fetches +each tag and category from the vSphere tagging API once per source, not once per +object it is attached to. The result must be the same as without the cache. + +vcsim serves no tag API, so the tag session is faked. +""" +from types import SimpleNamespace + +import pytest + +from module.sources.vmware import connection as vmware_connection +from module.sources.vmware.connection import VMWareHandler + + +@pytest.fixture(autouse=True) +def dynamic_id(monkeypatch): + """The vSphere automation SDK is optional and absent here, the tag call needs the type.""" + monkeypatch.setattr(vmware_connection, "DynamicID", + lambda **kwargs: SimpleNamespace(**kwargs), raising=False) + + +class _CountingTagging: + """The parts of the vSphere tagging API get_vmware_object_tags() uses, counting lookups.""" + + def __init__(self): + self.tag_lookups = 0 + self.category_lookups = 0 + tags = { + "tag-1": SimpleNamespace(name="prod", description="production", category_id="cat-1"), + "tag-2": SimpleNamespace(name="web", description="", category_id="cat-2"), + } + categories = {"cat-1": "env", "cat-2": "role"} + outer = self + + class _Tag: + @staticmethod + def get(tag_id): + outer.tag_lookups += 1 + return tags[tag_id] + + class _Category: + @staticmethod + def get(category_id): + outer.category_lookups += 1 + return SimpleNamespace(name=categories[category_id]) + + class _TagAssociation: + @staticmethod + def list_attached_tags(_dynamic_id): + return ["tag-1", "tag-2"] + + self.Tag, self.Category, self.TagAssociation = _Tag, _Category, _TagAssociation + + +def _handler(inventory, tagging, include_category): + handler = object.__new__(VMWareHandler) + handler.inventory = inventory + handler.name = "test" + handler.source_tag = "Source: test" + handler.tag_session = SimpleNamespace(tagging=tagging) + handler.settings = SimpleNamespace(tag_name_include_category=include_category) + return handler + + +def _collect(handler, moid): + obj = SimpleNamespace(name=moid, _wsdlName="VirtualMachine", _moId=moid) + return [tag.get_display_name() for tag in handler.get_vmware_object_tags(obj)] + + +@pytest.mark.parametrize("include_category, expected", [ + (False, ["prod", "web"]), + (True, ["env:prod", "role:web"]), +]) +def test_each_tag_and_category_is_fetched_once(inventory, include_category, expected): + tagging = _CountingTagging() + handler = _handler(inventory, tagging, include_category) + + results = [_collect(handler, f"vm-{n}") for n in range(5)] + + assert results == [expected] * 5, "the cached tags differ from the fetched ones" + assert tagging.tag_lookups == 2, "a tag was fetched more than once" + assert tagging.category_lookups == (2 if include_category else 0) + + +def test_cache_is_per_source(inventory): + # two sources are two vCenters: tag ids are only unique within one + tagging_a, tagging_b = _CountingTagging(), _CountingTagging() + + _collect(_handler(inventory, tagging_a, False), "vm-1") + _collect(_handler(inventory, tagging_b, False), "vm-1") + + assert (tagging_a.tag_lookups, tagging_b.tag_lookups) == (2, 2)