fix: starting removing the concept of SpaceToken (SRM)

dirac.cfg

@@ -790,7 +790,7 @@ Resources
       CheckAccess = True # Allowed for Check if no RSS enabled
       RemoveAccess = True # Allowed for Remove if no RSS enabled
       OccupancyLFN = /lhcb/storageDetails.json # Json containing occupancy details
-      SpaceReservation = LHCb-EOS # Space reservation name if any. Concept like SpaceToken
+      SpaceReservation = LHCb-EOS # Space reservation name if any.
       ArchiveTimeout = 84600 # Timeout for the FTS archiving
       BringOnlineTimeout = 84600 # Timeout for the bring online operation used by FTS
       WLCGTokenBasePath = /eos/lhcb # EXPERIMENTAL Path from which the token should be relative to
@@ -803,7 +803,6 @@ Resources
         Protocol = srm # primary protocol
         Path = /eos/lhcb/grid/prod # base path
         Access = remote
-        SpaceToken = LHCb-EOS
         WSUrl = /srm/v2/server?SFN=
         InputProtocols = file, https, root, srm, gsiftp # Allow to overwrite the list of protocols understood as input
         OutputProtocols = file, https, root, srm, gsiftp # Allow to overwrite the list of protocols that can be generated
@@ -826,7 +825,6 @@ Resources
       GFAL2_SRM2
       {
         Path = /eos/lhcb/grid/user
-        SpaceToken = LHCb_USER
       }
       # Add an extra protocol
       GFAL2_XROOT
@@ -836,7 +834,6 @@ Resources
         Protocol = root
         Path = /eos/lhcb/grid/user
         Access = remote
-        SpaceToken = LHCb-EOS
         WSUrl = /srm/v2/server?SFN=
       }
     }

docs/source/AdministratorGuide/Resources/storage.rst

@@ -31,8 +31,6 @@ DIRAC provides an abstraction of a SE interface that allows to access different
         WSUrl = /srm/managerv2?SFN=
         # Path to navigate to the VO namespace on the storage
         Path = /castor/cern.ch/grid
-        # SRM space token
-        SpaceToken = LHCb_USER
         # VO specific path definitions
         VOPath
         {
@@ -57,7 +55,7 @@ Configuration options are:
 * ``RemoveAccess``: default ``True``. Allowed for Remove if no RSS enabled
 * ``OccupancyLFN``: default (``/<vo>/occupancy.json``). LFN where the json file containing the space reporting is to be found
 * ``OccupancyPlugin``: default (``empty``). Plugin to find the occupancy of a given storage.
-* ``SpaceReservation``: just a name of a zone of the physical storage which can have some space reserved. Extends the SRM ``SpaceToken`` concept.
+* ``SpaceReservation``: just a name of a zone of the physical storage which can have some space reserved.
 * ``ArchiveTimeout``: for tape SE only. If set to a value in seconds, enables the `FTS Archive Monitoring feature <https://fts3-docs.web.cern.ch/fts3-docs/docs/archive_monitoring.html>`_
 * ``BringOnlineTimeout``: for tape SE only. If set to a value in seconds, specify the BringOnline parameter for FTS transfers. Otherwise, the default is whatever is in the ``FTS3Job`` class.
 * ``WLCGTokenBasePath``: EXPERIMENTAL Path from which the token should be relative to (only used for FTS transfers for now)
@@ -93,7 +91,6 @@ In order to factorize the configuration, it is possible to use ``BaseSE``, which
           Protocol = srm
           Path = /eos/lhcb/grid/prod
           Access = remote
-          SpaceToken = LHCb-EOS
           WSUrl = /srm/v2/server?SFN=
         }
       }
@@ -112,7 +109,6 @@ In order to factorize the configuration, it is possible to use ``BaseSE``, which
         {
           PluginName = GFAL2_SRM2
           Path = /eos/lhcb/grid/user
-          SpaceToken = LHCb_USER
         }
       }
       GFAL2_XROOT
@@ -122,7 +118,6 @@ In order to factorize the configuration, it is possible to use ``BaseSE``, which
         Protocol = root
         Path = /eos/lhcb/grid/user
         Access = remote
-        SpaceToken = LHCb-EOS
         WSUrl = /srm/v2/server?SFN=
       }
     }
@@ -144,7 +139,6 @@ This definition would be strictly equivalent to::
           Protocol = srm
           Path = /eos/lhcb/grid/prod
           Access = remote
-          SpaceToken = LHCb-EOS
           WSUrl = /srm/v2/server?SFN=
         }
       }
@@ -163,7 +157,6 @@ This definition would be strictly equivalent to::
           Protocol = srm
           Path = /eos/lhcb/grid/prod
           Access = remote
-          SpaceToken = LHCb-EOS
           WSUrl = /srm/v2/server?SFN=
         }
       }
@@ -180,7 +173,6 @@ This definition would be strictly equivalent to::
           Protocol = srm
           Path = /eos/lhcb/grid/user
           Access = remote
-          SpaceToken = LHCb_USER
           WSUrl = /srm/v2/server?SFN=
         }
       }
@@ -192,7 +184,6 @@ This definition would be strictly equivalent to::
         Protocol = root
         Path = /eos/lhcb/grid/user
         Access = remote
-        SpaceToken = LHCb-EOS
         WSUrl = /srm/v2/server?SFN=
       }
     }
@@ -261,7 +252,7 @@ For example::
 
 The LFN of this file is by default `/<vo>/occupancy.json`, but can be overwritten with the ``OccupancyLFN`` option of the SE.
 
-The ``SpaceReservation`` option allows to specify a physical zone of the storage which would have space reservation (for example ``LHCb_USER``, ``LHCb_PROD``, etc). It extends the concept of ``SpaceToken`` that SRM has. This option is only used if the StoragePlugin does not return itself a ``SpaceReservation`` value.
+The ``SpaceReservation`` option allows to specify a physical zone of the storage which would have space reservation (for example ``LHCb_USER``, ``LHCb_PROD``, etc). This option is only used if the StoragePlugin does not return itself a ``SpaceReservation`` value.
 
 The ``OccupancyPlugin`` allows to change the way space occupancy is measured. Several plugins are available (please refer to the module documentation):
 
@@ -311,7 +302,6 @@ You need to define a protocol section with SRM, specifying that a ``file`` URL c
       Path = /disk
       # This is different from the ``standard`` definition
       Access = local
-      SpaceToken = LHCb-Disk
       WSUrl = /srm/managerv2?SFN=
       # This is different from the ``standard`` definition
       OutputProtocols = file, https, gsiftp, root, srm

src/DIRAC/ConfigurationSystem/Agent/RucioSynchronizerAgent.py

@@ -1,4 +1,4 @@
-""" :mod: RucioSynchronizer
+""":mod: RucioSynchronizer
 
   Agent that synchronizes Rucio and Dirac
 
@@ -92,7 +92,6 @@ def getStorageElements(vo):
 
         mapping = {"Protocol": "scheme", "Host": "hostname", "Port": "port", "Path": "prefix"}
         for protocol in all_protocols:
-            space_token = None
             params = {
                 "hostname": None,
                 "scheme": None,
@@ -107,17 +106,15 @@ def getStorageElements(vo):
             res = se.getStorageParameters(protocol=protocol)
             if res["OK"]:
                 values = res["Value"]
-                for key in ["Protocol", "Host", "Access", "Path", "Port", "WSUrl", "SpaceToken", "WSUrl", "PluginName"]:
+                for key in ["Protocol", "Host", "Access", "Path", "Port", "WSUrl", "WSUrl", "PluginName"]:
                     value = values.get(key)
                     if key in mapping:
                         params[mapping[key]] = value
                     else:
-                        if key == "SpaceToken":
-                            space_token = value
                         if params["scheme"] == "srm" and key == "WSUrl":
                             params["extended_attributes"] = {
                                 "web_service_path": f"{value}",
-                                "space_token": space_token,
+                                "space_token": None,  # nosec B105
                             }
                     if key == "Protocol":
                         params["domains"]["lan"]["read"] = read_protocols.get(value, 0)
@@ -181,7 +178,7 @@ def configHelper(voList):
 
         if len(selectedCatalog) > 1:
             log.error(
-                "VO %s: Services/Catalogs section mis-configured." " More that one Rucio file catalog",
+                "VO %s: Services/Catalogs section mis-configured. More that one Rucio file catalog",
                 f"[VO: {vo}, Catalogs: {selectedCatalog}]",
             )
             continue

src/DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -31,11 +31,9 @@
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 
 from DIRAC.FrameworkSystem.Client.Logger import gLogger
-from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager
 from DIRAC.FrameworkSystem.Utilities.TokenManagementUtilities import getIdProviderClient
 
 from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, returnValueOrRaise
-from DIRAC.Core.Utilities.DErrno import cmpError
 
 from DIRAC.Core.Utilities.JEncode import JSerializable
 from DIRAC.Core.Utilities.TimeUtilities import DiracTime
@@ -326,31 +324,6 @@ def cancel(self, context):
         except FTS3ClientException as e:
             return S_ERROR(f"Error canceling the job {e}")
 
-    @staticmethod
-    def __fetchSpaceToken(seName, vo):
-        """Fetch the space token of storage element
-
-        :param seName: name of the storageElement
-        :param vo: vo of the job
-        :returns: space token. If there is no SpaceToken defined, returns None
-        """
-        seToken = None
-        if seName:
-            seObj = StorageElement(seName, vo=vo)
-
-            res = seObj.getStorageParameters(protocol="srm")
-            if not res["OK"]:
-                # If there is no SRM protocol, we do not specify
-                # the space token
-                if cmpError(res, errno.ENOPROTOOPT):
-                    return S_OK(None)
-
-                return res
-
-            seToken = res["Value"].get("SpaceToken")
-
-        return S_OK(seToken)
-
     @staticmethod
     def __isTapeSE(seName, vo):
         """Check whether a given SE is a tape storage
@@ -429,11 +402,6 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N
 
         nbOfHops = len(allHops)
 
-        res = self.__fetchSpaceToken(self.sourceSE, self.vo)
-        if not res["OK"]:
-            return res
-        source_spacetoken = res["Value"]
-
         failedLFNs = set()
 
         copy_pin_lifetime = None
@@ -665,7 +633,7 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N
             transfers=transfers,
             overwrite=True,
             disable_cleanup=True,
-            source_spacetoken=source_spacetoken,
+            source_spacetoken=None,
             bring_online=bring_online,
             copy_pin_lifetime=copy_pin_lifetime,
             retry=3,
@@ -809,7 +777,7 @@ def _constructStagingJob(self, pinTime, allLFNs, target_spacetoken):
             transfers=transfers,
             overwrite=True,
             disable_cleanup=True,
-            source_spacetoken=target_spacetoken,
+            source_spacetoken=None,
             bring_online=bring_online,
             copy_pin_lifetime=copy_pin_lifetime,
             retry=3,
@@ -853,23 +821,16 @@ def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protoc
         log = gLogger.getLocalSubLogger(f"submit/{self.operationID}/{self.sourceSE}_{self.targetSE}")
 
         # Construct the target SURL
-        res = self.__fetchSpaceToken(self.targetSE, self.vo)
-        if not res["OK"]:
-            return res
-        target_spacetoken = res["Value"]
-
         allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit]
 
         if self.type == "Transfer":
             res = self._constructTransferJob(
-                pinTime, allLFNs, target_spacetoken, protocols=protocols, tokensEnabled=bool(fts_access_token)
+                pinTime, allLFNs, None, protocols=protocols, tokensEnabled=bool(fts_access_token)
             )
-
         elif self.type == "Staging":
-            res = self._constructStagingJob(pinTime, allLFNs, target_spacetoken)
-        # elif self.type == 'Removal':
-        #   res = self._constructRemovalJob(context, allLFNs, failedLFNs, target_spacetoken)
-
+            res = self._constructStagingJob(pinTime, allLFNs, None)
+        else:
+            return S_ERROR("Unknown type")
         if not res["OK"]:
             return res
 

src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py

@@ -166,11 +166,6 @@ def monkeypatchForAllTest(monkeypatch):
         "_StorageFactory__generateStorageObject",
         mock_StorageFactory_generateStorageObject,
     )
-    monkeypatch.setattr(
-        DIRAC.DataManagementSystem.Client.FTS3Job.FTS3Job,
-        "_FTS3Job__fetchSpaceToken",
-        lambda _self, _seName, _vo: S_OK(),
-    )
 
     def mock_init(self, useProxy=False, vo=None):
         self.proxy = False

src/DIRAC/Resources/Storage/StorageBase.py

@@ -54,7 +54,7 @@ class StorageBase:
 
     """
 
-    PROTOCOL_PARAMETERS = ["Protocol", "Host", "Path", "Port", "SpaceToken", "WSUrl"]
+    PROTOCOL_PARAMETERS = ["Protocol", "Host", "Path", "Port", "WSUrl"]
     # Options to be prepended in the URL
     # keys are the name of the parameters in the CS
     # values are the name of the options as they appear in the URL

src/DIRAC/Resources/Storage/StorageElement.py

@@ -1,5 +1,4 @@
-""" This is the StorageElement module. It implements The StorageElementItem as well as the caching system
-"""
+"""This is the StorageElement module. It implements The StorageElementItem as well as the caching system"""
 
 # # custom duty
 
@@ -371,7 +370,6 @@ def isSameSE(self, otherSE):
         """Compares two SE together and tries to guess if the two SEs are pointing at the same
         location from the namespace point of view.
         This is primarily aimed at avoiding to overwrite a file with itself, in particular
-        where the difference is only the SRM spacetoken.
 
         Two SEs are considered to be the same if they have a couple (Host, Path) in common
         among their various protocols
@@ -405,7 +403,6 @@ def getOccupancy(self, unit="MB", **kwargs):
         It returns the Total and Free space, and a SpaceReservation.
 
         The SpaceReservation is just a name of a zone of the physical storage which can have some space reserved.
-        It corresponds to the ``SpaceToken`` concept of SRM.
         If the StorageElement definition has a ``SpaceReservation`` option in the CS, this is returned, unless
         it is overwritten by the storage plugin.
 

src/DIRAC/Resources/Storage/StorageFactory.py

@@ -1,15 +1,15 @@
-""" Storage Factory Class - creates instances of various Storage plugins from the Core DIRAC or extensions
+"""Storage Factory Class - creates instances of various Storage plugins from the Core DIRAC or extensions
 
-    This Class has three public methods:
+This Class has three public methods:
 
-    getStorageName():  Resolves links in the CS to the target SE name.
+getStorageName():  Resolves links in the CS to the target SE name.
 
-    getStorage():      This creates a single storage stub based on the parameters passed in a dictionary.
-                      This dictionary must have the following keys: 'StorageName','PluginName','Protocol'
-                      Other optional keys are 'Port','Host','Path','SpaceToken'
+getStorage():      This creates a single storage stub based on the parameters passed in a dictionary.
+                  This dictionary must have the following keys: 'StorageName','PluginName','Protocol'
+                  Other optional keys are 'Port','Host','Path'
 
-    getStorages()      This takes a DIRAC SE definition and creates storage stubs for the protocols found in the CS.
-                      By providing an optional list of protocols it is possible to limit the created stubs.
+getStorages()      This takes a DIRAC SE definition and creates storage stubs for the protocols found in the CS.
+                  By providing an optional list of protocols it is possible to limit the created stubs.
 """
 
 from DIRAC import gLogger, gConfig, S_OK, S_ERROR
@@ -367,7 +367,7 @@ def _getConfigStorageProtocolDetails(self, storageName, protocolSection, seConfi
         options = res["Value"]
 
         # We must have certain values internally even if not supplied in CS
-        protocolDict = {"Access": "", "Host": "", "Path": "", "Port": "", "Protocol": "", "SpaceToken": "", "WSUrl": ""}
+        protocolDict = {"Access": "", "Host": "", "Path": "", "Port": "", "Protocol": "", "WSUrl": ""}
         for option in options:
             configPath = cfgPath(protocolConfigPath, option)
             optionValue = gConfig.getValue(configPath, "")

src/DIRAC/Resources/Storage/test/Test_FilePlugin.py

@@ -1,5 +1,4 @@
-""" test File Plugin
-"""
+"""test File Plugin"""
 
 from unittest import mock
 import unittest
@@ -38,7 +37,6 @@ def mock_StorageFactory_getConfigStorageProtocols(storageName, derivedStorageNam
             "PluginName": "File",
             "Port": "",
             "Protocol": "file",
-            "SpaceToken": "",
             "WSUrl": "",
         }
     }

src/DIRAC/Resources/Storage/test/Test_GFAL2_XROOTStorage.py

@@ -20,7 +20,6 @@ def setUp(self):
             Path="/path",
             Host="host",
             Port="",
-            SvcClass="spaceToken",
             WSPath="wspath",
         )
 
@@ -32,13 +31,6 @@ def test_constructURLFromLFN(self):
         resource.se.vo = voName
         testLFN = f"/{voName}/path/to/filename"
 
-        # # with spaceToken
-        res = resource.constructURLFromLFN(testLFN)
-        self.assertTrue(res["OK"])
-        self.assertEqual(f"protocol://host//path{testLFN}?svcClass={self.parameterDict['SvcClass']}", res["Value"])
-
-        # # no spaceToken
-        resource.protocolParameters["SvcClass"] = ""
         res = resource.constructURLFromLFN(testLFN)
         self.assertTrue(res["OK"])
         self.assertEqual(f"protocol://host//path{testLFN}", res["Value"])