From b58c753ff9be140d6a45953289da15dcc9ce533e Mon Sep 17 00:00:00 2001 From: ARRABITO Luisa Date: Wed, 21 Aug 2024 15:08:23 +0200 Subject: [PATCH 1/9] feat (TS): add check on Deleted status --- .../DB/TransformationDB.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index 37ec1b752cd..8b3a012a4b8 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -1138,6 +1138,9 @@ def __addMetaQuery(self, transID, queryDict, queryType, author="", connection=Fa if isinstance(parameterValue, int): parameterType = "Integer" parameterValue = str(parameterValue) + if isinstance(parameterValue, float): + parameterType = "Float" + parameterValue = str(parameterValue) if isinstance(parameterValue, dict): parameterType = "Dict" parameterValue = str(parameterValue) @@ -1617,6 +1620,20 @@ def addFile(self, fileDicts, force=False, connection=False): gLogger.info("Files to add to transformations:", filesToAdd) if filesToAdd: for transID, lfns in transFiles.items(): + # Check if file is already in the Transformation and if is in Deleted Status + res = self.getTransformationFiles( + condDict={"TransformationID": transID, "LFN": lfns}, connection=connection + ) + if not res["OK"]: + return res + fileIDs = [] + for fileDict in res["Value"]: + fileIDs.append(fileDict["FileID"]) + if fileDict["Status"] == "Deleted": + res = self.__setTransformationFileStatus(list(fileIDs), "Unused", connection=connection) + if not res["OK"]: + return res + res = self.addFilesToTransformation(transID, lfns) if not res["OK"]: gLogger.error("Failed to add files to transformation", f"{transID} {res['Message']}") @@ -1747,6 +1764,20 @@ def setMetadata(self, path, usermetadatadict): gLogger.info("Files to add to transformations:", filesToAdd) if filesToAdd: for transID, lfns in transFiles.items(): + # Check if file is already in the Transformation and if is in Deleted Status + res = self.getTransformationFiles( + condDict={"TransformationID": transID, "LFN": lfns}, connection=connection + ) + if not res["OK"]: + return res + fileIDs = [] + for fileDict in res["Value"]: + fileIDs.append(fileDict["FileID"]) + if fileDict["Status"] == "Deleted": + res = self.__setTransformationFileStatus(list(fileIDs), "Unused", connection=connection) + if not res["OK"]: + return res + res = self.addFilesToTransformation(transID, lfns) if not res["OK"]: gLogger.error("Failed to add files to transformation", f"{transID} {res['Message']}") From aca80302009294d070c17a63c6520ccb95bea176 Mon Sep 17 00:00:00 2001 From: ARRABITO Luisa Date: Wed, 21 Aug 2024 15:35:37 +0200 Subject: [PATCH 2/9] fix (TS): add missing argument --- src/DIRAC/TransformationSystem/DB/TransformationDB.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index 8b3a012a4b8..9a326140e70 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -1706,7 +1706,7 @@ def __addDirectory(self, path, force): successful.append(lfn) return {"OK": True, "Value": len(res["Value"]["Successful"]), "Successful": successful, "Failed": failed} - def setMetadata(self, path, usermetadatadict): + def setMetadata(self, path, usermetadatadict, connection=False): """ It can be applied to a file or to a directory (path). For a file, add the file to Transformations if the updated metadata dictionary passes the filter. From 97ecdcf96e12c8506296b422b1c8d5a46f8f1dcc Mon Sep 17 00:00:00 2001 From: ARRABITO Luisa Date: Wed, 21 Aug 2024 15:49:31 +0200 Subject: [PATCH 3/9] feat (TS): update DataFile status --- src/DIRAC/TransformationSystem/DB/TransformationDB.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index 9a326140e70..ccde0a9e84f 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -1633,6 +1633,9 @@ def addFile(self, fileDicts, force=False, connection=False): res = self.__setTransformationFileStatus(list(fileIDs), "Unused", connection=connection) if not res["OK"]: return res + res = self.__setDataFileStatus(list(fileIDs), "New", connection=connection) + if not res["OK"]: + return res res = self.addFilesToTransformation(transID, lfns) if not res["OK"]: @@ -1777,6 +1780,10 @@ def setMetadata(self, path, usermetadatadict, connection=False): res = self.__setTransformationFileStatus(list(fileIDs), "Unused", connection=connection) if not res["OK"]: return res + res = self.__setDataFileStatus(list(fileIDs), "New", connection=connection) + if not res["OK"]: + return res + res = self.addFilesToTransformation(transID, lfns) if not res["OK"]: From 190d5ca984b0416c8c5c8edd0bdb67d1949260c6 Mon Sep 17 00:00:00 2001 From: ARRABITO Luisa Date: Thu, 22 Aug 2024 11:31:55 +0200 Subject: [PATCH 4/9] fix (TS): use TransformationStatus --- .../TransformationSystem/DB/TransformationDB.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index ccde0a9e84f..c0c1ccc86d0 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -15,6 +15,7 @@ from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Base.DB import DB from DIRAC.Core.Utilities.DErrno import cmpError +from DIRAC.TransformationSystem.Client import TransformationStatus from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Core.Utilities.List import stringListToString, intListToString, breakListIntoChunks @@ -1629,12 +1630,11 @@ def addFile(self, fileDicts, force=False, connection=False): fileIDs = [] for fileDict in res["Value"]: fileIDs.append(fileDict["FileID"]) - if fileDict["Status"] == "Deleted": - res = self.__setTransformationFileStatus(list(fileIDs), "Unused", connection=connection) + if fileDict["Status"] == TransformationStatus.DELETED: + res = self.__setTransformationFileStatus(list(fileIDs), TransformationStatus.UNUSED, connection=connection) if not res["OK"]: return res - res = self.__setDataFileStatus(list(fileIDs), "New", connection=connection) - if not res["OK"]: + if not (res := self.__setDataFileStatus(list(fileIDs), TransformationStatus.NEW, connection=connection)["OK"]: return res res = self.addFilesToTransformation(transID, lfns) @@ -1776,11 +1776,11 @@ def setMetadata(self, path, usermetadatadict, connection=False): fileIDs = [] for fileDict in res["Value"]: fileIDs.append(fileDict["FileID"]) - if fileDict["Status"] == "Deleted": - res = self.__setTransformationFileStatus(list(fileIDs), "Unused", connection=connection) + if fileDict["Status"] == TransformationStatus.DELETED: + res = self.__setTransformationFileStatus(list(fileIDs), TransformationStatus.UNUSED, connection=connection) if not res["OK"]: return res - res = self.__setDataFileStatus(list(fileIDs), "New", connection=connection) + res = self.__setDataFileStatus(list(fileIDs), TransformationStatus.NEW, connection=connection) if not res["OK"]: return res From 7a13ae91f03c45eeb0e14d7667ed06c572b2f298 Mon Sep 17 00:00:00 2001 From: ARRABITO Luisa Date: Thu, 22 Aug 2024 11:53:38 +0200 Subject: [PATCH 5/9] fix (TS): pre-commit reformat --- .../TransformationSystem/DB/TransformationDB.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index c0c1ccc86d0..978ddd85aa5 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -1631,10 +1631,16 @@ def addFile(self, fileDicts, force=False, connection=False): for fileDict in res["Value"]: fileIDs.append(fileDict["FileID"]) if fileDict["Status"] == TransformationStatus.DELETED: - res = self.__setTransformationFileStatus(list(fileIDs), TransformationStatus.UNUSED, connection=connection) + res = self.__setTransformationFileStatus( + list(fileIDs), TransformationStatus.UNUSED, connection=connection + ) if not res["OK"]: return res - if not (res := self.__setDataFileStatus(list(fileIDs), TransformationStatus.NEW, connection=connection)["OK"]: + if not ( + res := self.__setDataFileStatus( + list(fileIDs), TransformationStatus.NEW, connection=connection + )["OK"] + ): return res res = self.addFilesToTransformation(transID, lfns) @@ -1777,14 +1783,15 @@ def setMetadata(self, path, usermetadatadict, connection=False): for fileDict in res["Value"]: fileIDs.append(fileDict["FileID"]) if fileDict["Status"] == TransformationStatus.DELETED: - res = self.__setTransformationFileStatus(list(fileIDs), TransformationStatus.UNUSED, connection=connection) + res = self.__setTransformationFileStatus( + list(fileIDs), TransformationStatus.UNUSED, connection=connection + ) if not res["OK"]: return res res = self.__setDataFileStatus(list(fileIDs), TransformationStatus.NEW, connection=connection) if not res["OK"]: return res - res = self.addFilesToTransformation(transID, lfns) if not res["OK"]: gLogger.error("Failed to add files to transformation", f"{transID} {res['Message']}") From 99a427b36b7be7c77da40396069dc610e9872593 Mon Sep 17 00:00:00 2001 From: ARRABITO Luisa Date: Thu, 22 Aug 2024 13:52:34 +0200 Subject: [PATCH 6/9] fix (TS): use transformationfiles status --- .../Client/TransformationFilesStatus.py | 2 ++ .../TransformationSystem/DB/TransformationDB.py | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/DIRAC/TransformationSystem/Client/TransformationFilesStatus.py b/src/DIRAC/TransformationSystem/Client/TransformationFilesStatus.py index 73a4731e612..d35e5e67c17 100644 --- a/src/DIRAC/TransformationSystem/Client/TransformationFilesStatus.py +++ b/src/DIRAC/TransformationSystem/Client/TransformationFilesStatus.py @@ -17,6 +17,8 @@ PROB_IN_FC = "ProbInFC" #: REMOVED = "Removed" +#: +DELETED = "Deleted" # below ones are for inherited transformations #: diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index 978ddd85aa5..fb8d45d6f5a 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -15,7 +15,7 @@ from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Base.DB import DB from DIRAC.Core.Utilities.DErrno import cmpError -from DIRAC.TransformationSystem.Client import TransformationStatus +from DIRAC.TransformationSystem.Client import TransformationFilesStatus from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Core.Utilities.List import stringListToString, intListToString, breakListIntoChunks @@ -1630,15 +1630,15 @@ def addFile(self, fileDicts, force=False, connection=False): fileIDs = [] for fileDict in res["Value"]: fileIDs.append(fileDict["FileID"]) - if fileDict["Status"] == TransformationStatus.DELETED: + if fileDict["Status"] == TransformationFilesStatus.DELETED: res = self.__setTransformationFileStatus( - list(fileIDs), TransformationStatus.UNUSED, connection=connection + list(fileIDs), TransformationFilesStatus.UNUSED, connection=connection ) if not res["OK"]: return res if not ( res := self.__setDataFileStatus( - list(fileIDs), TransformationStatus.NEW, connection=connection + list(fileIDs), "New", connection=connection )["OK"] ): return res @@ -1782,13 +1782,13 @@ def setMetadata(self, path, usermetadatadict, connection=False): fileIDs = [] for fileDict in res["Value"]: fileIDs.append(fileDict["FileID"]) - if fileDict["Status"] == TransformationStatus.DELETED: + if fileDict["Status"] == TransformationFilesStatus.DELETED: res = self.__setTransformationFileStatus( - list(fileIDs), TransformationStatus.UNUSED, connection=connection + list(fileIDs), TransformationFilesStatus.UNUSED, connection=connection ) if not res["OK"]: return res - res = self.__setDataFileStatus(list(fileIDs), TransformationStatus.NEW, connection=connection) + res = self.__setDataFileStatus(list(fileIDs), "New", connection=connection) if not res["OK"]: return res From 8ea362a139ec5baad3de05ca083b4173b89bdccb Mon Sep 17 00:00:00 2001 From: ARRABITO Luisa Date: Thu, 22 Aug 2024 13:54:10 +0200 Subject: [PATCH 7/9] fix (TS): pre-commit reformat --- src/DIRAC/TransformationSystem/DB/TransformationDB.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index fb8d45d6f5a..c19c7876c27 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -1636,11 +1636,7 @@ def addFile(self, fileDicts, force=False, connection=False): ) if not res["OK"]: return res - if not ( - res := self.__setDataFileStatus( - list(fileIDs), "New", connection=connection - )["OK"] - ): + if not (res := self.__setDataFileStatus(list(fileIDs), "New", connection=connection)["OK"]): return res res = self.addFilesToTransformation(transID, lfns) From d152f995d4d0605a4ee6364c2638482f478b7801 Mon Sep 17 00:00:00 2001 From: ARRABITO Luisa Date: Thu, 22 Aug 2024 15:50:25 +0200 Subject: [PATCH 8/9] fix (TS): remove list append --- .../TransformationSystem/DB/TransformationDB.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index c19c7876c27..713b8c87a0c 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -1627,16 +1627,18 @@ def addFile(self, fileDicts, force=False, connection=False): ) if not res["OK"]: return res - fileIDs = [] for fileDict in res["Value"]: - fileIDs.append(fileDict["FileID"]) if fileDict["Status"] == TransformationFilesStatus.DELETED: res = self.__setTransformationFileStatus( - list(fileIDs), TransformationFilesStatus.UNUSED, connection=connection + list([fileDict["FileID"]]), TransformationFilesStatus.UNUSED, connection=connection ) if not res["OK"]: return res - if not (res := self.__setDataFileStatus(list(fileIDs), "New", connection=connection)["OK"]): + if not ( + res := self.__setDataFileStatus( + list([fileDict["FileID"]]), "New", connection=connection + )["OK"] + ): return res res = self.addFilesToTransformation(transID, lfns) @@ -1775,16 +1777,14 @@ def setMetadata(self, path, usermetadatadict, connection=False): ) if not res["OK"]: return res - fileIDs = [] for fileDict in res["Value"]: - fileIDs.append(fileDict["FileID"]) if fileDict["Status"] == TransformationFilesStatus.DELETED: res = self.__setTransformationFileStatus( - list(fileIDs), TransformationFilesStatus.UNUSED, connection=connection + list([fileDict["FileID"]]), TransformationFilesStatus.UNUSED, connection=connection ) if not res["OK"]: return res - res = self.__setDataFileStatus(list(fileIDs), "New", connection=connection) + res = self.__setDataFileStatus(list([fileDict["FileID"]]), "New", connection=connection) if not res["OK"]: return res From 5ba6450ceb3bfc787e09a9f9a293a84fe1eed1c9 Mon Sep 17 00:00:00 2001 From: ARRABITO Luisa Date: Thu, 22 Aug 2024 15:58:12 +0200 Subject: [PATCH 9/9] fix (TS): allow for parameter values equal to zero --- src/DIRAC/TransformationSystem/DB/TransformationDB.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index 713b8c87a0c..3f7aa4ef2dc 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -1127,7 +1127,7 @@ def __addMetaQuery(self, transID, queryDict, queryType, author="", connection=Fa for parameterName in sorted(queryDict): parameterValue = queryDict[parameterName] - if not parameterValue: + if parameterValue is None: continue parameterType = "String" if isinstance(parameterValue, (list, tuple)):