From db4bd7016f21aae6e0e93d1aab1ba2088b850f1e Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 8 Sep 2026 11:13:04 +0200 Subject: [PATCH] fix: Remove SpaceTokenOccupancyCache methods from ResourceManagementClient/Handler FreeDiskSpaceCommand now queries ResourceManagementDB directly instead of going through the client/handler/service layer. Added export_getSpaceTokenOccupancy to PublisherHandler as the only exposed endpoint for this table. Removed: - selectSpaceTokenOccupancyCache, deleteSpaceTokenOccupancyCache, addOrModifySpaceTokenOccupancyCache from ResourceManagementClient - _DEPRECATED_COLUMNS and _stripDeprecatedColumns from ResourceManagementHandler (only used for SpaceTokenOccupancyCache Endpoint column) - test_SpaceTokenOccupancy from integration tests --- .../Client/ResourceManagementClient.py | 83 +------------------ .../Command/FreeDiskSpaceCommand.py | 43 +++++----- .../Service/PublisherHandler.py | 23 ++++- .../Test_ResourceManagement.py | 44 +--------- 4 files changed, 50 insertions(+), 143 deletions(-) diff --git a/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py b/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py index 45029674ce2..55203b228d0 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py +++ b/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py @@ -1,7 +1,8 @@ -""" ResourceManagementClient +"""ResourceManagementClient - Client to interact with the ResourceManagement service and from it with the DB. +Client to interact with the ResourceManagement service and from it with the DB. """ + from DIRAC.Core.Base.Client import Client, createClient @@ -720,81 +721,3 @@ def addOrModifyPolicyResult( columnValues = [element, name, policyName, statusType, status, reason, dateEffective, lastCheckTime, vO] return self._getRPC().addOrModify("PolicyResult", prepareDict(columnNames, columnValues)) - - # SpaceTokenOccupancyCache Methods ........................................... - - def selectSpaceTokenOccupancyCache( - self, endpoint=None, token=None, total=None, guaranteed=None, free=None, lastCheckTime=None, meta=None - ): - """ - Gets from SpaceTokenOccupancyCache all rows that match the parameters given. - - :param endpoint: endpoint - :type endpoint: string, list - :param token: name of the token - :type token: string, list - :param total: total terabytes - :type total: integer, list - :param guaranteed: guaranteed terabytes - :type guaranteed: integer, list - :param free: free terabytes - :type free: integer, list - :param lastCheckTime: time-stamp from which the result is effective - :type lastCheckTime: datetime, list - :param dict meta: metadata for the mysql query. Currently it is being used only for column selection. - For example: meta={'columns': ['Name']} will return only the 'Name' column. - :return: S_OK() || S_ERROR() - """ - columnNames = ["Endpoint", "Token", "Total", "Guaranteed", "Free", "LastCheckTime", "Meta"] - columnValues = [endpoint, token, total, guaranteed, free, lastCheckTime, meta] - - return self._getRPC().select("SpaceTokenOccupancyCache", prepareDict(columnNames, columnValues)) - - def deleteSpaceTokenOccupancyCache( - self, endpoint=None, token=None, total=None, guaranteed=None, free=None, lastCheckTime=None - ): - """ - Deletes from SpaceTokenOccupancyCache all rows that match the parameters given. - - :param endpoint: endpoint - :type endpoint: string, list - :param token: name of the token - :type token: string, list - :param total: total terabytes - :type total: integer, list - :param guaranteed: guaranteed terabytes - :type guaranteed: integer, list - :param free: free terabytes - :type free: integer, list - :param lastCheckTime: time-stamp from which the result is effective - :type lastCheckTime: datetime, list - :return: S_OK() || S_ERROR() - """ - columnNames = ["Endpoint", "Token", "Total", "Guaranteed", "Free", "LastCheckTime"] - columnValues = [endpoint, token, total, guaranteed, free, lastCheckTime] - - return self._getRPC().delete("SpaceTokenOccupancyCache", prepareDict(columnNames, columnValues)) - - def addOrModifySpaceTokenOccupancyCache( - self, endpoint=None, token=None, total=None, guaranteed=None, free=None, lastCheckTime=None - ): - """ - Adds or updates-if-duplicated to SpaceTokenOccupancyCache. Using `site` and `token` - to query the database, decides whether to insert or update the table. - - :param endpoint: endpoint - :type endpoint: string, list - :param str token: name of the token - :param int total: total terabytes - :param int guaranteed: guaranteed terabytes - :param int free: free terabytes - :param datetime lastCheckTime: time-stamp from which the result is effective - :return: S_OK() || S_ERROR() - """ - columnNames = ["Endpoint", "Token", "Total", "Guaranteed", "Free", "LastCheckTime"] - columnValues = [endpoint, token, total, guaranteed, free, lastCheckTime] - - return self._getRPC().addOrModify("SpaceTokenOccupancyCache", prepareDict(columnNames, columnValues)) - - -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF diff --git a/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index d194657807f..c21264accb1 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -1,11 +1,9 @@ -""" FreeDiskSpaceCommand - The Command gets the free space that is left in a Storage Element - - Note: there are, still, many references to "space tokens", - for example ResourceManagementClient().selectSpaceTokenOccupancyCache(token=elementName) - This is for historical reasons, and shoud be fixed one day. - For the moment, when you see "token" or "space token" here, just read "StorageElement". - +"""FreeDiskSpaceCommand +The Command gets the free space that is left in a Storage Element +Note: there are, still, many references to "space tokens", +for example the SpaceTokenOccupancyCache table. +This is for historical reasons, and shoud be fixed one day. +For the moment, when you see "token" or "space token" here, just read "StorageElement". """ import errno @@ -18,8 +16,8 @@ from DIRAC.Core.Utilities.File import convertSizeUnits from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers from DIRAC.Resources.Storage.StorageElement import StorageElement -from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient from DIRAC.ResourceStatusSystem.Command.Command import Command +from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB from DIRAC.ResourceStatusSystem.Utilities import CSHelpers @@ -31,7 +29,7 @@ class FreeDiskSpaceCommand(Command): def __init__(self, args=None, clients=None): super().__init__(args, clients=clients) - self.rmClient = ResourceManagementClient() + self.rmDB = ResourceManagementDB() def _prepareCommand(self): """ @@ -99,15 +97,17 @@ def _storeCommand(self, results): """ # Stores in cache - res = self.rmClient.addOrModifySpaceTokenOccupancyCache( - endpoint=results["Endpoint"], - lastCheckTime=datetime.utcnow(), - free=results["Free"], - total=results["Total"], - token=results["ElementName"], + res = self.rmDB.addOrModify( + "SpaceTokenOccupancyCache", + { + "Token": results["ElementName"], + "Free": results["Free"], + "Total": results["Total"], + "LastCheckTime": datetime.utcnow(), + }, ) if not res["OK"]: - self.log.error("Error calling addOrModifySpaceTokenOccupancyCache", res["Message"]) + self.log.error("Error calling addOrModify on SpaceTokenOccupancyCache", res["Message"]) return res # Now proceed with the accounting @@ -150,7 +150,7 @@ def doCache(self): return params elementName, unit = params["Value"] - result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName) + result = self.rmDB.select("SpaceTokenOccupancyCache", {"Token": elementName}) if not result["OK"]: return result @@ -199,8 +199,9 @@ def _cleanCommand(self, toDelete=None): if not toDelete: toDelete = [] - res = self.rmClient.selectSpaceTokenOccupancyCache( - meta={"older": ["LastCheckTime", datetime.utcnow() - timedelta(hours=6)]} + res = self.rmDB.select( + "SpaceTokenOccupancyCache", + {"meta": {"older": ["LastCheckTime", datetime.utcnow() - timedelta(hours=6)]}}, ) if not res["OK"]: return res @@ -222,7 +223,7 @@ def _cleanCommand(self, toDelete=None): toDelete = [toDelete] for ep in toDelete: - res = self.rmClient.deleteSpaceTokenOccupancyCache(ep[0], ep[1]) + res = self.rmDB.delete("SpaceTokenOccupancyCache", {"Token": ep[0]}) if not res["OK"]: self.log.warn("Could not delete entry from SpaceTokenOccupancyCache", res["Message"]) diff --git a/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py b/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py index be6014ccb3b..4e5ff725fd7 100644 --- a/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py +++ b/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py @@ -4,16 +4,18 @@ This service has been built to provide the RSS web views with all the information they need. NO OTHER COMPONENT THAN Web controllers should make use of it. """ + # pylint: disable=no-self-use from datetime import datetime, timedelta # DIRAC -from DIRAC import S_OK, gConfig, S_ERROR +from DIRAC import S_ERROR, S_OK, gConfig +from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSiteCEMapping, getSites from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.SiteSEMapping import getSEHosts, getStorageElementsHosts -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites, getSiteCEMapping from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers +from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB class PublisherHandlerMixin: @@ -38,6 +40,8 @@ def initializeHandler(cls, serviceInfoDict): resourceManagementClientClass = result["Value"] cls.rmClient = resourceManagementClientClass() + cls.rmDB = ResourceManagementDB() + return S_OK() types_getSites = [] @@ -335,6 +339,21 @@ def export_getCachedDowntimes(self, element, elementType, name, severity): return result + types_selectSpaceTokenOccupancyCache = [(str, list, type(None))] + + @classmethod + def export_selectSpaceTokenOccupancyCache(cls, token): + """ + Gets space token occupancy from the SpaceTokenOccupancyCache table. + + :param token: name of the token (StorageElement) + :type token: str or list + :return: S_OK() || S_ERROR() + """ + + params = {"Token": token} if token else {} + return cls.rmDB.select("SpaceTokenOccupancyCache", params) + types_setStatus = [str] * 7 def export_setStatus(self, element, name, statusType, status, elementType, username, lastCheckTime): diff --git a/tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py b/tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py index 942e668b26e..bb4ed623f2e 100644 --- a/tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py +++ b/tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py @@ -1,8 +1,8 @@ -""" This is a test of the chain - ResourceManagementClient -> ResourceManagementHandler -> ResourceManagementDB - It supposes that the DB is present, and that the service is running +"""This is a test of the chain +ResourceManagementClient -> ResourceManagementHandler -> ResourceManagementDB +It supposes that the DB is present, and that the service is running - The DB is supposed to be empty when the test starts +The DB is supposed to be empty when the test starts """ # pylint: disable=wrong-import-position, missing-docstring @@ -263,42 +263,6 @@ def test_PolicyResult(rmClient): assert not res["Value"], res["Value"] -def test_SpaceTokenOccupancy(rmClient): - """ - SpaceTokenOccupancy table - """ - - res = rmClient.deleteSpaceTokenOccupancyCache("endpoint", "token") # just making sure it's not there (yet) - assert res["OK"] is True, res["Message"] - - # TEST addOrModifySpaceTokenOccupancy - res = rmClient.addOrModifySpaceTokenOccupancyCache( - "endpoint", "token", 500.0, 1000.0, 200.0, datetime.datetime.now() - ) - assert res["OK"] is True, res["Message"] - - res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token") - assert res["OK"] is True, res["Message"] - # check if the name that we got is equal to the previously added 'token' - assert res["Value"][0][1] == "token" - - res = rmClient.addOrModifySpaceTokenOccupancyCache("endpoint", "token", free=100.0) - assert res["OK"] is True, res["Message"] - - res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token") - # check if the result has changed - assert res["Value"][0][3] == 100.0 - - # TEST deleteSpaceTokenOccupancy - # ............................................................................... - res = rmClient.deleteSpaceTokenOccupancyCache("endpoint", "token") - assert res["OK"] is True, res["Message"] - - res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token") - assert res["OK"] is True, res["Message"] - assert not res["Value"], res["Value"] - - def test_Transfer(rmClient): """ TransferOccupancy table