diff --git a/src/DIRAC/TransformationSystem/Client/TransformationClient.py b/src/DIRAC/TransformationSystem/Client/TransformationClient.py index 7f1b7daa81a..e442ce65ce1 100644 --- a/src/DIRAC/TransformationSystem/Client/TransformationClient.py +++ b/src/DIRAC/TransformationSystem/Client/TransformationClient.py @@ -3,6 +3,7 @@ from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.Core.Base.Client import Client, createClient from DIRAC.Core.Utilities.List import breakListIntoChunks +from DIRAC.Core.Utilities.JEncode import decode as jdecode from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.TransformationSystem.Client import TransformationStatus from DIRAC.TransformationSystem.Client import TransformationFilesStatus @@ -176,9 +177,10 @@ def getTransformationFiles( timeStamp = "LastUpdate" if "LFN" not in condDict: - res = rpcClient.getTransformationFiles( + res = rpcClient.getTransformationFilesAsJsonString( condDict, older, newer, timeStamp, orderAttribute, offset, maxfiles, columns ) + res, _ = jdecode(res) # TransformationDB.getTransformationFiles includes a "Records"/"ParameterNames" # that we don't want to return to the client so explicitly return S_OK with the value if not res["OK"]: @@ -204,9 +206,10 @@ def getTransformationFiles( # Apply the offset to the list of LFNs condDict["LFN"] = lfnList[offsetToApply : offsetToApply + limit] # No limit and no offset as the list is limited already - res = rpcClient.getTransformationFiles( + res = rpcClient.getTransformationFilesAsJsonString( condDict, older, newer, timeStamp, orderAttribute, None, None, columns ) + res, _ = jdecode(res) if not res["OK"]: gLogger.error( "Error getting files for transformation %s (offset %d), %s" diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index aaf054156cb..6e7682a08fa 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -603,6 +603,7 @@ def getTransformationFiles( offset=None, connection=False, columns=None, + include_web_records=True, ): """Get files for the supplied transformations with support for the web standard structure""" connection = self.__getConnection(connection) @@ -640,11 +641,12 @@ def getTransformationFiles( return res resultList = [dict(zip(columns, row)) for row in res["Value"]] - webList = [[str(item) if not isinstance(item, int) else item for item in row] for row in res["Value"]] result = S_OK(resultList) - result["Records"] = webList - result["ParameterNames"] = columns + if include_web_records: + webList = [[str(item) if not isinstance(item, int) else item for item in row] for row in res["Value"]] + result["Records"] = webList + result["ParameterNames"] = columns return result def getFileSummary(self, lfns, connection=False): diff --git a/src/DIRAC/TransformationSystem/Service/TransformationManagerHandler.py b/src/DIRAC/TransformationSystem/Service/TransformationManagerHandler.py index d6dae921bc2..73828372df7 100644 --- a/src/DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +++ b/src/DIRAC/TransformationSystem/Service/TransformationManagerHandler.py @@ -1,9 +1,11 @@ """ Service for interacting with TransformationDB """ + from DIRAC import S_OK, S_ERROR from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.Security.Properties import SecurityProperty from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning +from DIRAC.Core.Utilities.JEncode import encode as jencode from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.TransformationSystem.Client import TransformationFilesStatus @@ -289,6 +291,7 @@ def export_getTransformationFiles( limit=None, offset=None, columns=None, + include_web_records=True, ): if not condDict: condDict = {} @@ -302,8 +305,22 @@ def export_getTransformationFiles( offset=offset, connection=False, columns=columns, + include_web_records=include_web_records, ) + types_getTransformationFilesAsJsonString = types_getTransformationFiles + + def export_getTransformationFilesAsJsonString(self, *args, **kwargs): + """ + DEncode cannot cope with nested structures of multiple millions items. + Encode everything as a json string, that DEncode cna then transmit faster. + + This will be the default as of v9.0 + """ + kwargs["include_web_records"] = False + res = self.export_getTransformationFiles(*args, **kwargs) + return S_OK(jencode(res)) + #################################################################### # # These are the methods to manipulate the TransformationTasks table