Skip to content

Commit

Permalink
feat: remove system instances, first import
Browse files Browse the repository at this point in the history
  • Loading branch information
atsareg committed Jun 12, 2024
1 parent 877f334 commit 1cad666
Show file tree
Hide file tree
Showing 22 changed files with 88 additions and 394 deletions.
3 changes: 1 addition & 2 deletions src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
getServiceSection,
getAgentSection,
getExecutorSection,
getSystemSection,
)
from DIRAC.Core.Utilities.Devloader import Devloader

Expand Down Expand Up @@ -507,7 +506,7 @@ def __addUserDataToConfiguration(self):
if self.componentType == "service":
self.__setDefaultSection(getServiceSection(self.componentName))
elif self.componentType == "tornado":
self.__setDefaultSection(getSystemSection("Tornado"))
self.__setDefaultSection("/Systems/Tornado")
elif self.componentType == "agent":
self.__setDefaultSection(getAgentSection(self.componentName))
elif self.componentType == "executor":
Expand Down
102 changes: 26 additions & 76 deletions src/DIRAC/ConfigurationSystem/Client/PathFinder.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,6 @@
from DIRAC.ConfigurationSystem.Client.Helpers import Path


def getDIRACSetup():
"""Get DIRAC default setup name
:return: str
"""
return gConfigurationData.extractOptionFromCFG("/DIRAC/Setup")


def divideFullName(entityName, componentName=None):
"""Convert component full name to tuple
Expand All @@ -32,45 +24,12 @@ def divideFullName(entityName, componentName=None):
raise RuntimeError(f"Service ({entityName}) name must be with the form system/service")


def getSystemInstance(system, setup=False):
"""Find system instance name
:param str system: system name
:param str setup: setup name
:return: str
"""
optionPath = Path.cfgPath("/DIRAC/Setups", setup or getDIRACSetup(), system)
instance = gConfigurationData.extractOptionFromCFG(optionPath)
if not instance:
raise RuntimeError(f"Option {optionPath} is not defined")
return instance


def getSystemSection(system, instance=False, setup=False):
"""Get system section
:param str system: system name
:param str instance: instance name
:param str setup: setup name
:return: str -- system section path
"""
system, _ = divideFullName(system, "_") # for backward compatibility
return Path.cfgPath(
"/Systems",
system,
instance or getSystemInstance(system, setup=setup),
)


def getComponentSection(system, component=False, setup=False, componentCategory="Services"):
def getComponentSection(system, component=False, componentCategory="Services"):
"""Function returns the path to the component.
:param str system: system name or component name prefixed by the system in which it is placed.
e.g. 'WorkloadManagement/SandboxStoreHandler'
:param str component: component name, e.g. 'SandboxStoreHandler'
:param str setup: Name of the setup.
:param str componentCategory: Category of the component, it can be:
'Agents', 'Services', 'Executors' or 'Databases'.
Expand All @@ -80,81 +39,76 @@ def getComponentSection(system, component=False, setup=False, componentCategory=
:raise RuntimeException: If in the system - the system part does not correspond to any known system in DIRAC.
Examples:
getComponentSection('WorkloadManagement/SandboxStoreHandler', setup='Production', componentCategory='Services')
getComponentSection('WorkloadManagement', 'SandboxStoreHandler', 'Production')
getComponentSection('WorkloadManagement/SandboxStoreHandler', componentCategory='Services')
getComponentSection('WorkloadManagement', 'SandboxStoreHandler')
"""
system, component = divideFullName(system, component)
return Path.cfgPath(getSystemSection(system, setup=setup), componentCategory, component)
return Path.cfgPath(f"/Systems/{system}", componentCategory, component)


def getAPISection(system, endpointName=False, setup=False):
def getAPISection(system, endpointName=False):
"""Get API section in a system
:param str system: system name
:param str endpointName: endpoint name
:return: str
"""
return getComponentSection(system, component=endpointName, setup=setup, componentCategory="APIs")
return getComponentSection(system, component=endpointName, componentCategory="APIs")


def getServiceSection(system, serviceName=False, setup=False):
def getServiceSection(system, serviceName=False):
"""Get service section in a system
:param str system: system name
:param str serviceName: service name
:param str setup: setup name
:return: str
"""
return getComponentSection(system, component=serviceName, setup=setup)
return getComponentSection(system, component=serviceName)


def getAgentSection(system, agentName=False, setup=False):
def getAgentSection(system, agentName=False):
"""Get agent section in a system
:param str system: system name
:param str agentName: agent name
:param str setup: setup name
:return: str
"""
return getComponentSection(system, component=agentName, setup=setup, componentCategory="Agents")
return getComponentSection(system, component=agentName, componentCategory="Agents")


def getExecutorSection(system, executorName=None, component=False, setup=False):
def getExecutorSection(system, executorName=None):
"""Get executor section in a system
:param str system: system name
:param str executorName: executor name
:param str setup: setup name
:return: str
"""
return getComponentSection(system, component=executorName, setup=setup, componentCategory="Executors")
return getComponentSection(system, component=executorName, componentCategory="Executors")


def getDatabaseSection(system, dbName=False, setup=False):
def getDatabaseSection(system, dbName=False):
"""Get DB section in a system
:param str system: system name
:param str dbName: DB name
:param str setup: setup name
:return: str
"""
return getComponentSection(system, component=dbName, setup=setup, componentCategory="Databases")
return getComponentSection(system, component=dbName, componentCategory="Databases")


def getSystemURLSection(system, setup=False):
def getSystemURLSection(system):
"""Get URLs section in a system
:param str system: system name
:param str setup: setup name
:return: str
"""
return Path.cfgPath(getSystemSection(system, setup=setup), "URLs")
return Path.cfgPath(f"/Systems/{system}", "URLs")


def checkComponentURL(componentURL, system=None, component=None, pathMandatory=False):
Expand Down Expand Up @@ -183,35 +137,33 @@ def checkComponentURL(componentURL, system=None, component=None, pathMandatory=F
return url.geturl()


def getSystemURLs(system, setup=False, failover=False):
def getSystemURLs(system, failover=False):
"""Generate url.
:param str system: system name or full name e.g.: Framework/ProxyManager
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:param bool failover: to add failover URLs to end of result list
:return: dict -- complete urls. e.g. [dips://some-domain:3424/Framework/Service]
"""
urlDict = {}
for service in gConfigurationData.getOptionsFromCFG(f"{getSystemSection(system, setup=setup)}/URLs") or []:
urlDict[service] = getServiceURLs(system, service, setup=setup, failover=failover)
for service in gConfigurationData.getOptionsFromCFG(f"/Systems/{system}/URLs") or []:
urlDict[service] = getServiceURLs(system, service, failover=failover)
return urlDict


def getServiceURLs(system, service=None, setup=False, failover=False):
def getServiceURLs(system, service=None, failover=False):
"""Generate url.
:param str system: system name or full name e.g.: Framework/ProxyManager
:param str service: service name, like 'ProxyManager'.
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:param bool failover: to add failover URLs to end of result list
:return: list -- complete urls. e.g. [dips://some-domain:3424/Framework/Service]
"""
system, service = divideFullName(system, service)
resList = []
mainServers = None
systemSection = getSystemSection(system, setup=setup)
systemSection = f"/Systems/{system}"

# Add failover URLs at the end of the list
failover = "Failover" if failover else ""
Expand All @@ -228,7 +180,7 @@ def getServiceURLs(system, service=None, setup=False, failover=False):
# Operations cannot be imported at the beginning because of a bootstrap problem
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations

mainServers = Operations(setup=setup).getValue("MainServers", [])
mainServers = Operations().getValue("MainServers", [])
if not mainServers:
raise Exception("No Main servers defined")

Expand Down Expand Up @@ -261,31 +213,29 @@ def useLegacyAdapter(system, service=None) -> bool:
return (value or "no").lower() in ("y", "yes", "true", "1")


def getServiceURL(system, service=None, setup=False):
def getServiceURL(system, service=None):
"""Generate url.
:param str system: system name or full name e.g.: Framework/ProxyManager
:param str service: service name, like 'ProxyManager'.
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:return: str -- complete list of urls. e.g. dips://some-domain:3424/Framework/Service, dips://..
"""
system, service = divideFullName(system, service)
urls = getServiceURLs(system, service=service, setup=setup)
urls = getServiceURLs(system, service=service)
return ",".join(urls) if urls else ""


def getServiceFailoverURL(system, service=None, setup=False):
def getServiceFailoverURL(system, service=None):
"""Get failover URLs for service
:param str system: system name or full name, like 'Framework/Service'.
:param str service: service name, like 'ProxyManager'.
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:return: str -- complete list of urls
"""
system, service = divideFullName(system, service)
systemSection = getSystemSection(system, setup=setup)
systemSection = f"/Systems/{system}"
failovers = gConfigurationData.extractOptionFromCFG(f"{systemSection}/FailoverURLs/{service}")
if not failovers:
return ""
Expand Down
5 changes: 2 additions & 3 deletions src/DIRAC/ConfigurationSystem/Client/Utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getDIRACSiteName
from DIRAC.ConfigurationSystem.Client.PathFinder import getDatabaseSection
from DIRAC.Core.Utilities.Glue2 import getGlue2CEInfo
from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemInstance


def getGridVOs():
Expand Down Expand Up @@ -616,7 +615,7 @@ def getAuthAPI():
:return: str
"""
return gConfig.getValue(f"/Systems/Framework/{getSystemInstance('Framework')}/URLs/AuthAPI")
return gConfig.getValue(f"/Systems/Framework/URLs/AuthAPI")


def getAuthorizationServerMetadata(issuer=None, ignoreErrors=False):
Expand Down Expand Up @@ -655,5 +654,5 @@ def isDownloadProxyAllowed():
:return: S_OK(bool)/S_ERROR()
"""
cs_path = f"/Systems/Framework/{getSystemInstance('Framework')}/APIs/Auth"
cs_path = f"/Systems/Framework/APIs/Auth"
return gConfig.getValue(cs_path + "/allowProxyDownload", True)
5 changes: 0 additions & 5 deletions src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,6 @@ def pathFinder(monkeypatch):
return PathFinder


def test_getDIRACSetup(pathFinder):
"""Test getDIRACSetup"""
assert pathFinder.getDIRACSetup() == "TestSetup"


@pytest.mark.parametrize(
"system, componentName, setup, componentType, result",
[
Expand Down
4 changes: 1 addition & 3 deletions src/DIRAC/Core/Base/private/ModuleLoader.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import os
from DIRAC.Core.Utilities import List
from DIRAC import gConfig, S_ERROR, S_OK, gLogger
from DIRAC.ConfigurationSystem.Client import PathFinder
from DIRAC.Core.Utilities.Extensions import extensionsByPriority, recurseImport


Expand Down Expand Up @@ -44,8 +43,7 @@ def loadModules(self, modulesList, hideExceptions=False):
# Check if it's a system name
# Look in the CS
system = modName
# Can this be generated with sectionFinder?
csPath = f"{PathFinder.getSystemSection(system)}/Executors"
csPath = f"/Systems/{system}/Executors"
gLogger.verbose(f"Exploring {csPath} to discover modules")
result = gConfig.getSections(csPath)
if result["OK"]:
Expand Down
27 changes: 4 additions & 23 deletions src/DIRAC/Core/DISET/private/BaseClient.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ def __init__(self, serviceName, **kwargs):
self.__retryCounter = 1
self.__bannedUrls = []
for initFunc in (
self.__discoverSetup,
self.__discoverVO,
self.__discoverTimeout,
self.__discoverURL,
Expand Down Expand Up @@ -117,24 +116,6 @@ def getServiceName(self):
"""
return self._serviceName

def __discoverSetup(self):
"""Discover which setup to use and stores it in self.setup
The setup is looked for:
* kwargs of the constructor (see KW_SETUP)
* the ThreadConfig
* in the CS /DIRAC/Setup
* default to 'Test'
:return: S_OK()/S_ERROR()
"""
if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]:
self.setup = str(self.kwargs[self.KW_SETUP])
else:
self.setup = self.__threadConfig.getSetup()
if not self.setup:
self.setup = gConfig.getValue("/DIRAC/Setup", "Test")
return S_OK()

def __discoverVO(self):
"""Discover which VO to use and stores it in self.vo
The VO is looked for:
Expand Down Expand Up @@ -326,16 +307,16 @@ def __findServiceURL(self):

# We extract the list of URLs from the CS (System/URLs/Component)
try:
urls = getServiceURL(self._destinationSrv, setup=self.setup)
urls = getServiceURL(self._destinationSrv)
except Exception as e:
return S_ERROR(f"Cannot get URL for {self._destinationSrv} in setup {self.setup}: {repr(e)}")
return S_ERROR(f"Cannot get URL for {self._destinationSrv} : {repr(e)}")
if not urls:
return S_ERROR(f"URL for service {self._destinationSrv} not found")

failoverUrls = []
# Try if there are some failover URLs to use as last resort
try:
failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup)
failoverUrlsStr = getServiceFailoverURL(self._destinationSrv)
if failoverUrlsStr:
failoverUrls = failoverUrlsStr.split(",")
except Exception:
Expand Down Expand Up @@ -564,7 +545,7 @@ def _proposeAction(self, transport, action):
"""
if not self.__initStatus["OK"]:
return self.__initStatus
stConnectionInfo = ((self.__URLTuple[3], self.setup, self.vo), action, self.__extraCredentials, DIRAC.version)
stConnectionInfo = ((self.__URLTuple[3], "None", self.vo), action, self.__extraCredentials, DIRAC.version)

# Send the connection info and get the answer back
retVal = transport.sendData(S_OK(BaseClient._serializeStConnectionInfo(stConnectionInfo)))
Expand Down
Loading

0 comments on commit 1cad666

Please sign in to comment.