Skip to content

[9.0] fix: Remove SpaceTokenOccupancyCache methods from ResourceManagementClient/Handler #8777

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fstagni wants to merge 1 commit into
base: rel-v9r0
Choose a base branch
from
Open

[9.0] fix: Remove SpaceTokenOccupancyCache methods from ResourceManagementClient/Handler #8777

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 3 additions & 80 deletions src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
""" ResourceManagementClient
"""ResourceManagementClient

Client to interact with the ResourceManagement service and from it with the DB.
Client to interact with the ResourceManagement service and from it with the DB.
"""

from DIRAC.Core.Base.Client import Client, createClient


Expand Down Expand Up @@ -720,81 +721,3 @@ def addOrModifyPolicyResult(
columnValues = [element, name, policyName, statusType, status, reason, dateEffective, lastCheckTime, vO]

return self._getRPC().addOrModify("PolicyResult", prepareDict(columnNames, columnValues))

# SpaceTokenOccupancyCache Methods ...........................................

def selectSpaceTokenOccupancyCache(
self, endpoint=None, token=None, total=None, guaranteed=None, free=None, lastCheckTime=None, meta=None
):
"""
Gets from SpaceTokenOccupancyCache all rows that match the parameters given.

:param endpoint: endpoint
:type endpoint: string, list
:param token: name of the token
:type token: string, list
:param total: total terabytes
:type total: integer, list
:param guaranteed: guaranteed terabytes
:type guaranteed: integer, list
:param free: free terabytes
:type free: integer, list
:param lastCheckTime: time-stamp from which the result is effective
:type lastCheckTime: datetime, list
:param dict meta: metadata for the mysql query. Currently it is being used only for column selection.
For example: meta={'columns': ['Name']} will return only the 'Name' column.
:return: S_OK() || S_ERROR()
"""
columnNames = ["Endpoint", "Token", "Total", "Guaranteed", "Free", "LastCheckTime", "Meta"]
columnValues = [endpoint, token, total, guaranteed, free, lastCheckTime, meta]

return self._getRPC().select("SpaceTokenOccupancyCache", prepareDict(columnNames, columnValues))

def deleteSpaceTokenOccupancyCache(
self, endpoint=None, token=None, total=None, guaranteed=None, free=None, lastCheckTime=None
):
"""
Deletes from SpaceTokenOccupancyCache all rows that match the parameters given.

:param endpoint: endpoint
:type endpoint: string, list
:param token: name of the token
:type token: string, list
:param total: total terabytes
:type total: integer, list
:param guaranteed: guaranteed terabytes
:type guaranteed: integer, list
:param free: free terabytes
:type free: integer, list
:param lastCheckTime: time-stamp from which the result is effective
:type lastCheckTime: datetime, list
:return: S_OK() || S_ERROR()
"""
columnNames = ["Endpoint", "Token", "Total", "Guaranteed", "Free", "LastCheckTime"]
columnValues = [endpoint, token, total, guaranteed, free, lastCheckTime]

return self._getRPC().delete("SpaceTokenOccupancyCache", prepareDict(columnNames, columnValues))

def addOrModifySpaceTokenOccupancyCache(
self, endpoint=None, token=None, total=None, guaranteed=None, free=None, lastCheckTime=None
):
"""
Adds or updates-if-duplicated to SpaceTokenOccupancyCache. Using `site` and `token`
to query the database, decides whether to insert or update the table.

:param endpoint: endpoint
:type endpoint: string, list
:param str token: name of the token
:param int total: total terabytes
:param int guaranteed: guaranteed terabytes
:param int free: free terabytes
:param datetime lastCheckTime: time-stamp from which the result is effective
:return: S_OK() || S_ERROR()
"""
columnNames = ["Endpoint", "Token", "Total", "Guaranteed", "Free", "LastCheckTime"]
columnValues = [endpoint, token, total, guaranteed, free, lastCheckTime]

return self._getRPC().addOrModify("SpaceTokenOccupancyCache", prepareDict(columnNames, columnValues))


# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
43 changes: 22 additions & 21 deletions src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
""" FreeDiskSpaceCommand
The Command gets the free space that is left in a Storage Element

Note: there are, still, many references to "space tokens",
for example ResourceManagementClient().selectSpaceTokenOccupancyCache(token=elementName)
This is for historical reasons, and shoud be fixed one day.
For the moment, when you see "token" or "space token" here, just read "StorageElement".

"""FreeDiskSpaceCommand
The Command gets the free space that is left in a Storage Element
Note: there are, still, many references to "space tokens",
for example the SpaceTokenOccupancyCache table.
This is for historical reasons, and shoud be fixed one day.
For the moment, when you see "token" or "space token" here, just read "StorageElement".
"""

import errno
Expand All @@ -18,8 +16,8 @@
from DIRAC.Core.Utilities.File import convertSizeUnits
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
from DIRAC.Resources.Storage.StorageElement import StorageElement
from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
from DIRAC.ResourceStatusSystem.Command.Command import Command
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
from DIRAC.ResourceStatusSystem.Utilities import CSHelpers


Expand All @@ -31,7 +29,7 @@ class FreeDiskSpaceCommand(Command):
def __init__(self, args=None, clients=None):
super().__init__(args, clients=clients)

self.rmClient = ResourceManagementClient()
self.rmDB = ResourceManagementDB()

def _prepareCommand(self):
"""
Expand Down Expand Up @@ -99,15 +97,17 @@ def _storeCommand(self, results):
"""

# Stores in cache
res = self.rmClient.addOrModifySpaceTokenOccupancyCache(
endpoint=results["Endpoint"],
lastCheckTime=datetime.utcnow(),
free=results["Free"],
total=results["Total"],
token=results["ElementName"],
res = self.rmDB.addOrModify(
"SpaceTokenOccupancyCache",
{
"Token": results["ElementName"],
"Free": results["Free"],
"Total": results["Total"],
"LastCheckTime": datetime.utcnow(),
},
)
if not res["OK"]:
self.log.error("Error calling addOrModifySpaceTokenOccupancyCache", res["Message"])
self.log.error("Error calling addOrModify on SpaceTokenOccupancyCache", res["Message"])
return res

# Now proceed with the accounting
Expand Down Expand Up @@ -150,7 +150,7 @@ def doCache(self):
return params
elementName, unit = params["Value"]

result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName)
result = self.rmDB.select("SpaceTokenOccupancyCache", {"Token": elementName})

if not result["OK"]:
return result
Expand Down Expand Up @@ -199,8 +199,9 @@ def _cleanCommand(self, toDelete=None):
if not toDelete:
toDelete = []

res = self.rmClient.selectSpaceTokenOccupancyCache(
meta={"older": ["LastCheckTime", datetime.utcnow() - timedelta(hours=6)]}
res = self.rmDB.select(
"SpaceTokenOccupancyCache",
{"meta": {"older": ["LastCheckTime", datetime.utcnow() - timedelta(hours=6)]}},
)
if not res["OK"]:
return res
Expand All @@ -222,7 +223,7 @@ def _cleanCommand(self, toDelete=None):
toDelete = [toDelete]

for ep in toDelete:
res = self.rmClient.deleteSpaceTokenOccupancyCache(ep[0], ep[1])
res = self.rmDB.delete("SpaceTokenOccupancyCache", {"Token": ep[0]})
if not res["OK"]:
self.log.warn("Could not delete entry from SpaceTokenOccupancyCache", res["Message"])

Expand Down
23 changes: 21 additions & 2 deletions src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,18 @@
This service has been built to provide the RSS web views with all the information
they need. NO OTHER COMPONENT THAN Web controllers should make use of it.
"""

# pylint: disable=no-self-use
from datetime import datetime, timedelta

# DIRAC
from DIRAC import S_OK, gConfig, S_ERROR
from DIRAC import S_ERROR, S_OK, gConfig
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSiteCEMapping, getSites
from DIRAC.Core.DISET.RequestHandler import RequestHandler
from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
from DIRAC.Core.Utilities.SiteSEMapping import getSEHosts, getStorageElementsHosts
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites, getSiteCEMapping
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB


class PublisherHandlerMixin:
Expand All @@ -38,6 +40,8 @@ def initializeHandler(cls, serviceInfoDict):
resourceManagementClientClass = result["Value"]
cls.rmClient = resourceManagementClientClass()

cls.rmDB = ResourceManagementDB()

return S_OK()

types_getSites = []
Expand Down Expand Up @@ -335,6 +339,21 @@ def export_getCachedDowntimes(self, element, elementType, name, severity):

return result

types_selectSpaceTokenOccupancyCache = [(str, list, type(None))]

@classmethod
def export_selectSpaceTokenOccupancyCache(cls, token):
"""
Gets space token occupancy from the SpaceTokenOccupancyCache table.

:param token: name of the token (StorageElement)
:type token: str or list
:return: S_OK() || S_ERROR()
"""

params = {"Token": token} if token else {}
return cls.rmDB.select("SpaceTokenOccupancyCache", params)

types_setStatus = [str] * 7

def export_setStatus(self, element, name, statusType, status, elementType, username, lastCheckTime):
Expand Down
44 changes: 4 additions & 40 deletions tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
""" This is a test of the chain
ResourceManagementClient -> ResourceManagementHandler -> ResourceManagementDB
It supposes that the DB is present, and that the service is running
"""This is a test of the chain
ResourceManagementClient -> ResourceManagementHandler -> ResourceManagementDB
It supposes that the DB is present, and that the service is running

The DB is supposed to be empty when the test starts
The DB is supposed to be empty when the test starts
"""
# pylint: disable=wrong-import-position, missing-docstring

Expand Down Expand Up @@ -263,42 +263,6 @@ def test_PolicyResult(rmClient):
assert not res["Value"], res["Value"]


def test_SpaceTokenOccupancy(rmClient):
"""
SpaceTokenOccupancy table
"""

res = rmClient.deleteSpaceTokenOccupancyCache("endpoint", "token") # just making sure it's not there (yet)
assert res["OK"] is True, res["Message"]

# TEST addOrModifySpaceTokenOccupancy
res = rmClient.addOrModifySpaceTokenOccupancyCache(
"endpoint", "token", 500.0, 1000.0, 200.0, datetime.datetime.now()
)
assert res["OK"] is True, res["Message"]

res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token")
assert res["OK"] is True, res["Message"]
# check if the name that we got is equal to the previously added 'token'
assert res["Value"][0][1] == "token"

res = rmClient.addOrModifySpaceTokenOccupancyCache("endpoint", "token", free=100.0)
assert res["OK"] is True, res["Message"]

res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token")
# check if the result has changed
assert res["Value"][0][3] == 100.0

# TEST deleteSpaceTokenOccupancy
# ...............................................................................
res = rmClient.deleteSpaceTokenOccupancyCache("endpoint", "token")
assert res["OK"] is True, res["Message"]

res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token")
assert res["OK"] is True, res["Message"]
assert not res["Value"], res["Value"]


def test_Transfer(rmClient):
"""
TransferOccupancy table
Expand Down
Loading