diff --git a/framework/BaseClasses.py b/framework/BaseClasses.py index 325bd70f4b..e1c18a6e26 100644 --- a/framework/BaseClasses.py +++ b/framework/BaseClasses.py @@ -84,7 +84,7 @@ def readXML(self,xmlNode,messageHandler,variableGroups={},globalAttributes=None) self.name = xmlNode.attrib['name'] else: self.raiseAnError(IOError,'not found name for a '+self.__class__.__name__) - self.type = xmlNode.tag + self.type = xmlNode.tag if globalAttributes is not None: self.globalAttributes = globalAttributes if 'verbosity' in xmlNode.attrib.keys() or 'verbosity' in self.globalAttributes: @@ -114,7 +114,7 @@ def replaceVariableGroups(node): self.raiseADebug('------Reading Completed for:') self.printMe() - def handleInput(self,paramInput,messageHandler,variableGroups={},globalAttributes=None): + def handleInput(self, paramInput, messageHandler, variableGroups={}, globalAttributes=None): """ provide a basic reading capability from the xml input file for what is common to all types in the simulation than calls _handleInput that needs to be overloaded and used as API. Each type supported by the simulation should have: name (xml attribute), type (xml tag), diff --git a/framework/DataObjects/DataSet.py b/framework/DataObjects/DataSet.py index a4ecdc74ff..33d12856ec 100644 --- a/framework/DataObjects/DataSet.py +++ b/framework/DataObjects/DataSet.py @@ -690,7 +690,7 @@ def sliceByIndex(self,index): # NOTE: The slice may include NaN if a variable does not have a value along a different index for this snapshot along "index" return slices - def write(self,fileName,style='netCDF',**kwargs): + def write(self, fileName, style='netCDF', **kwargs): """ Writes this dataset to disk based on the format. @ In, fileName, str, path and name of file to write @@ -703,12 +703,12 @@ def write(self,fileName,style='netCDF',**kwargs): if style.lower() == 'netcdf': self._toNetCDF(fileName,**kwargs) elif style.lower() == 'csv': - if len(self)==0: #TODO what if it's just metadata? + if len(self) == 0: self.raiseAWarning('Nothing to write to CSV! Checking metadata ...') else: #first write the CSV firstIndex = kwargs.get('firstIndex',0) - self._toCSV(fileName,start=firstIndex,**kwargs) + self._toCSV(fileName, start=firstIndex, **kwargs) # then the metaxml if len(self._meta): self._toCSVXML(fileName,**kwargs) @@ -1681,14 +1681,14 @@ def _getRealizationFromDataByValue(self, match, noMatch, tol=1e-15, unpackXArray return len(self),None return idx,self._getRealizationFromDataByIndex(idx,unpackXArray) - def _getRequestedElements(self,options): + def _getRequestedElements(self, options): """ Obtains a list of the elements to be written, based on defaults and options[what] @ In, options, dict, general list of options for writing output files @ Out, keep, list(str), list of variables that will be written to file """ if 'what' in options.keys(): - elements = options['what'].split(',') + elements = options['what'] keep = [] for entry in elements: small = entry.strip().lower() @@ -1867,7 +1867,7 @@ def _setScalingFactors(self,var=None): except Exception: self.raiseADebug('Had an issue with setting scaling factors for variable "{}". No big deal.'.format(name)) - def _toCSV(self,fileName,start=0,**kwargs): + def _toCSV(self, fileName, start=0, **kwargs): """ Writes this data object to CSV file (except the general metadata, see _toCSVXML) @ In, fileName, str, path/name to write file diff --git a/framework/OutStreams/Factory.py b/framework/OutStreams/Factory.py index 1a2a506c72..2e5544e03a 100644 --- a/framework/OutStreams/Factory.py +++ b/framework/OutStreams/Factory.py @@ -22,29 +22,22 @@ ################################################################################ from utils import utils -from .OutStreamManager import OutStreamManager -from .OutStreamPlot import OutStreamPlot -from .OutStreamPrint import OutStreamPrint -## [ Add new class here ] -################################################################################ -## Alternatively, to fully automate this file: -# from OutStreamManagers import * -################################################################################ -""" - Interface Dictionary (factory) (private) -""" -# This machinery will automatically populate the "knownTypes" given the -# imports defined above. -__base = 'OutStreamManager' -__interFaceDict = {} +from .OutStreamBase import OutStreamBase + +from .FilePrint import FilePrint +from .GeneralPlot import GeneralPlot +# from .DataMining import DataMining +# from .VDCComparison import VDCComparison -for classObj in utils.getAllSubclasses(eval(__base)): - ## As long as these subclasses follow the pattern of starting with OutStream - ## this will appropriately key them to a more user-friendly name without the - ## need for them to redudantly prepend "X" as "OutStreamX" - key = classObj.__name__.replace('OutStream','') - __interFaceDict[key] = classObj +# Interface Dictionary (factory) (private) +__base = 'OutStreamBase' +__interFaceDict = { + 'Print': FilePrint, + 'Plot': GeneralPlot, + # 'DataMining': DataMining, + # 'VDCComparison': VDCComparison, +} def knownTypes(): """ diff --git a/framework/OutStreams/OutStreamPrint.py b/framework/OutStreams/FilePrint.py similarity index 64% rename from framework/OutStreams/OutStreamPrint.py rename to framework/OutStreams/FilePrint.py index e80c1287d5..b39c6c1795 100644 --- a/framework/OutStreams/OutStreamPrint.py +++ b/framework/OutStreams/FilePrint.py @@ -16,20 +16,14 @@ @author: alfoa """ -#for future compatibility with Python 3----------------------------------------- -from __future__ import division, print_function, unicode_literals, absolute_import -#End compatibility block for Python 3------------------------------------------- -#External Modules--------------------------------------------------------------- import os -#External Modules End----------------------------------------------------------- -#Internal Modules--------------------------------------------------------------- +from utils import InputData, InputTypes import DataObjects -from .OutStreamManager import OutStreamManager +from .OutStreamBase import OutStreamBase from ClassProperty import ClassProperty -#Internal Modules End----------------------------------------------------------- -class OutStreamPrint(OutStreamManager): +class FilePrint(OutStreamBase): """ Class for managing the printing of files as an output stream. """ @@ -46,19 +40,41 @@ def availableOutStreamTypes(cls): """ A class level constant that tells developers what outstreams are available from this class - @ In, cls, the OutStreamPrint class of which this object will be a type + @ In, cls, the OutStreams class of which this object will be a type """ return cls._availableOutStreamTypes + @classmethod + def getInputSpecification(cls): + """ + Method to get a reference to a class that specifies the input data for class "cls". + @ In, cls, the class for which we are retrieving the specification + @ Out, inputSpecification, InputData.ParameterInput, class to use for specifying the input of cls. + """ + spec = OutStreamBase.getInputSpecification() + + types = InputTypes.makeEnumType('FilePrintTypes', 'FilePrintTypes', cls._availableOutStreamTypes) + spec.addSub(InputData.parameterInputFactory('type', contentType=types)) + spec.addSub(InputData.parameterInputFactory('source', contentType=InputTypes.StringListType)) + spec.addSub(InputData.parameterInputFactory('what', contentType=InputTypes.StringListType)) + spec.addSub(InputData.parameterInputFactory('filename', contentType=InputTypes.StringType)) + spec.addSub(InputData.parameterInputFactory('clusterLabel', contentType=InputTypes.StringType)) + + # these are in user manual or code, but don't appear to be used/documented ... + # spec.addSub(InputData.parameterInputFactory('target', contentType=InputTypes.StringListType)) + # spec.addSub(InputData.parameterInputFactory('directory', + # contentType=InputTypes.StringListType)) + return spec + def __init__(self): """ Constructor @ In, None @ Out, None """ - OutStreamManager.__init__(self) + OutStreamBase.__init__(self) + self.type = 'OutStreamFilePrint' self.printTag = 'OUTSTREAM PRINT' - OutStreamManager.__init__(self) self.sourceName = [] self.sourceData = None self.what = None @@ -66,6 +82,41 @@ def __init__(self): self.indexPrinted = {} # keys are filenames, which should be reset at the end of every step self.subDirectory = None # subdirectory where to store the outputs + def _handleInput(self, spec): + """ + Loads the input specs for this object. + @ In, spec, InputData.ParameterInput, input specifications + @ Out, None + """ + typ = spec.findFirst('type') + if typ is None: + self.raiseAnError(IOError, f'OutStream.Print "{self.name}" is missing the "type" node!') + self.options['type'] = typ.value + + source = spec.findFirst('source') + if source is None: + self.raiseAnError(IOError, f'OutStream.Print "{self.name}" is missing the "source" node!') + self.sourceName = source.value + + # FIXME this is a terrible name + what = spec.findFirst('what') + if what is not None: + self.what = what.value # [x.lower() for x in what.value] + + fname = spec.findFirst('filename') + if fname is not None: + self.filename = fname.value + + cluster = spec.findFirst('clusterLabel') + if cluster is not None: + self.options['clusterLabel'] = cluster.value + + # checks + if self.options['type'] == 'csv' and self.what is not None: + for target in [x.lower() for x in self.what]: + if not target.startswith(('input', 'output', 'metadata')): + self.raiseAnError(IOError, f' requests must start with "input", "output", or "metadata"! See OutStream.Print "{self.name}"') + def localGetInitParams(self): """ This method is called from the base function. It retrieves the initial @@ -78,7 +129,7 @@ def localGetInitParams(self): for index in range(len(self.sourceName)): paramDict['Source Name ' + str(index) + ' :'] = self.sourceName[index] if self.what: - for index, var in enumerate(self.what.split(',')): + for index, var in enumerate(self.what): paramDict['Variable Name ' + str(index) + ' :'] = var return paramDict @@ -91,35 +142,7 @@ def initialize(self, inDict): @ Out, None """ # the linking to the source is performed in the base class initialize method - OutStreamManager.initialize(self, inDict) - - def localReadXML(self, xmlNode): - """ - This Function is called from the base class, It reads the parameters that - belong to a plot block (outputs by filling data structure (self)) - @ In, xmlNode, xml.etree.ElementTree.Element, Xml element node - @ Out, None - """ - self.type = 'OutStreamPrint' - for subnode in xmlNode: - if subnode.tag not in ['type','source','what','filename','target','clusterLabel','directory']: - self.raiseAnError(IOError, ' Print Outstream object ' + str(self.name) + ' contains the following unknown node: ' + str(subnode.tag)) - if subnode.tag == 'source': - self.sourceName = subnode.text.split(',') - elif subnode.tag == 'filename': - self.filename = subnode.text - else: - self.options[subnode.tag] = subnode.text - if 'type' not in self.options.keys(): - self.raiseAnError(IOError, 'type tag not present in Print block called ' + self.name) - if self.options['type'] not in self.availableOutStreamTypes: - self.raiseAnError(TypeError, 'Print type ' + self.options['type'] + ' not available yet. ') - if 'what' in self.options.keys(): - self.what = self.options['what'] - if self.options['type'] == 'csv': - for elm in self.what.lower().split(","): - if not elm.startswith("input") and not elm.startswith("output") and not elm.startswith("metadata"): - self.raiseAnError(IOError, 'Not recognized request in "what" node <'+elm.strip()+'>. The request must begin with one of "input", "output" or "metadata" or it could be "all" for ROMs!') + OutStreamBase.initialize(self, inDict) def addOutput(self): """ diff --git a/framework/OutStreams/OutStreamPlot.py b/framework/OutStreams/GeneralPlot.py similarity index 93% rename from framework/OutStreams/OutStreamPlot.py rename to framework/OutStreams/GeneralPlot.py index 6cfae7e0e4..a947e5bd2a 100644 --- a/framework/OutStreams/OutStreamPlot.py +++ b/framework/OutStreams/GeneralPlot.py @@ -16,15 +16,11 @@ @author: alfoa """ -## for future compatibility with Python 3--------------------------------------- -from __future__ import division, print_function, unicode_literals, absolute_import -## End compatibility block for Python 3----------------------------------------- ## External Modules------------------------------------------------------------- import numpy as np import ast import copy -#from scipy.interpolate import Rbf, griddata import numpy.ma as ma import platform import os @@ -32,15 +28,14 @@ import matplotlib from mpl_toolkits.mplot3d import Axes3D from collections import defaultdict -## Maybe necessary -# import _tkinter ## External Modules End--------------------------------------------------------- ## Internal Modules------------------------------------------------------------- -from utils import utils +from utils.InputData import parameterInputFactory as PIF +from utils import utils, mathUtils, InputTypes from utils import mathUtils from utils.cached_ndarray import c1darray -from .OutStreamManager import OutStreamManager +from .OutStreamBase import OutStreamBase from ClassProperty import ClassProperty ## Internal Modules End--------------------------------------------------------- @@ -51,7 +46,7 @@ import matplotlib.pyplot as plt -class OutStreamPlot(OutStreamManager): +class GeneralPlot(OutStreamBase): """ OutStream of type Plot """ @@ -92,6 +87,54 @@ def availableInterpolators(cls): """ return cls._availableInterpolators + @classmethod + def getInputSpecification(cls): + """ + Method to get a reference to a class that specifies the input data for class "cls". + @ In, cls, the class for which we are retrieving the specification + @ Out, inputSpecification, InputData.ParameterInput, class to use for specifying the input of cls. + """ + spec = OutStreamBase.getInputSpecification() + # TODO this is waaaaay to much to convert right now + # For now, accept a blank plotting check and sort it out later. + spec.strictMode = False + return spec + ################################################################### + # TODO here's a good start, but skipping for now: + # spec.addParam('interactive', param_type=InputTypes.BoolType) + # spec.addParam('overwrite', param_type=InputTypes.BoolType) + # spec.addSub(PIF('filename', contentType=InputTypes.StringType)) + + # xyz = InputTypes.makeEnumType('PlotXYZ', 'PlotXYZ', ['x', 'y', 'z']) + + # action = PIF('actions') + # hows = InputTypes.makeEnumType('GeneralPlotHow', 'GeneralPlotHow', + # ['screen', 'pdf', 'png', 'eps', 'pgf', 'ps', 'gif', 'svg', 'jpeg', 'raw', 'bmp', 'tiff', 'svgz']) + # action.addSub(PIF('how', contentType=hows)) + + # title = PIF('title') + # title.addSub(PIF('text', contentType=InputTypes.StringType)) + # # kwargs can be anything, so just turn strict mode off for it + # title.addSub(PIF('kwargs', strictMode=False)) + # action.addSub(title) + + # labelFormat = PIF('labelFormat') + # labelFormat.addSub(PIF('axis', contentType=xyz)) + # sciPlain = InputTypes.makeEnumType('SciNot', 'SciNot', ['sci', 'scientific', 'plain']) + # labelFormat.addSub(PIF('style', contentType=sciPlain)) + # labelFormat.addSub(PIF('scilimits', contentType=InputTypes.StringType)) + # labelFormat.addSub(PIF('useOffset', contentType=InputTypes.FloatType)) + # action.addSub(labelFormat) + + # figProp TODO WORKING XXX + # TODO + # spec.addSub(action) + + # settings = parameterInputFactory('plotSettings') + # TODO + # spec.addSub(settings) + # return spec + #################################### END draft def __init__(self): """ @@ -100,7 +143,7 @@ def __init__(self): @ In, None @ Out, None """ - OutStreamManager.__init__(self) + OutStreamBase.__init__(self) self.printTag = 'OUTSTREAM PLOT' ## default plot is 2D @@ -146,6 +189,108 @@ def __init__(self): self.mixtureMeans = None self.mixtureCovars = None + # TODO started, but didn't finish due to time constraints + # this should be a good start for _handleInput in the future. + # def _handleInput(self, spec): + # """ + # Loads the input specs for this object. + # @ In, spec, InputData.ParameterInput, input specifications + # @ Out, None + # """ + # if 'dim' in spec.parameterValues: + # self.raiseAnError(IOError,"the 'dim' attribute has been deprecated. This warning became an error in January 2017") + # foundPlot = False + # for subnode in spec.subparts: + # # if actions, read actions block + # if subnode.getName() == 'filename': + # self.filename = subnode.value + # if subnode.getName() == 'actions': + # self.__readPlotActions(subnode) + # if subnode.getName() == 'plotSettings': + # self.options[subnode.getName()] = {} + # self.options[subnode.getName()]['plot'] = [] + # for subsub in subnode.subparts: + # if subsub.getName() == 'gridSpace': + # # if self.dim == 3: self.raiseAnError(IOError, 'SubPlot option can not be used with 3-dimensional plots!') + # self.options[subnode.getName()][subsub.getName()] = subsub.value + # elif subsub.getName() == 'plot': + # tempDict = {} + # foundPlot = True + # for subsubsub in subsub.subparts: + # if subsubsub.getName() == 'gridLocation': + # tempDict[subsubsub.getName()] = {} + # for subsubsubsub in subsubsub.subparts: + # tempDict[subsubsub.getName()][subsubsubsub.getName()] = subsubsubsub.value + # elif subsubsub.getName() == 'range': + # tempDict[subsubsub.getName()] = {} + # for subsubsubsub in subsubsub.subparts: + # tempDict[subsubsub.getName()][subsubsubsub.getName()] = subsubsubsub.value + # elif subsubsub.getName() != 'kwargs': + # tempDict[subsubsub.getName()] = subsubsub.value + # else: + # tempDict['attributes'] = {} + # for sss in subsubsub.subparts: + # tempDict['attributes'][sss.getName()] = sss.value + # self.options[subnode.getName()][subsub.getName()].append(tempDict) + # elif subsub.getName() == 'legend': + # self.options[subnode.getName()][subsub.getName()] = {} + # for legendChild in subsub.subparts: + # self.options[subnode.getName()][subsub.getName()][legendChild.getName()] = utils.tryParse(legendChild.value) + # else: + # self.options[subnode.getName()][subsub.getName()] = subsub.value + # # TODO WORKING XXX + # if subnode.getName() == 'title': + # self.options[subnode.getName()] = {} + # for subsub in subnode: + # self.options[subnode.getName()][subsub.getName()] = subsub.text.strip() + # if 'text' not in self.options[subnode.getName()].keys(): + # self.options[subnode.getName()]['text' ] = xmlNode.attrib['name'] + # if 'location' not in self.options[subnode.getName()].keys(): + # self.options[subnode.getName()]['location'] = 'center' + # ## is this 'figureProperties' valid? + # if subnode.getName() == 'figureProperties': + # self.options[subnode.getName()] = {} + # for subsub in subnode: + # self.options[subnode.getName()][subsub.getName()] = subsub.text.strip() + # self.type = 'OutStreamPlot' + + # if not 'plotSettings' in self.options.keys(): + # self.raiseAnError(IOError, 'For plot named ' + self.name + ' the plotSettings block is required.') + + # if not foundPlot: + # self.raiseAnError(IOError, 'For plot named' + self.name + ', No plot section has been found in the plotSettings block!') + + # self.outStreamTypes = [] + # xyz, xy = sorted(['x','y','z']), sorted(['x','y']) + # for pltIndex in range(len(self.options['plotSettings']['plot'])): + # if not 'type' in self.options['plotSettings']['plot'][pltIndex].keys(): + # self.raiseAnError(IOError, 'For plot named' + self.name + ', No plot type keyword has been found in the plotSettings/plot block!') + # else: + # # check the dimension and check the consistency + # if set(xyz) < set(self.options['plotSettings']['plot'][pltIndex].keys()): + # dim = 3 + # elif set(xy) < set(self.options['plotSettings']['plot'][pltIndex].keys()): + # dim = 2 if self.options['plotSettings']['plot'][pltIndex]['type'] != 'histogram' else 3 + # elif set(['x']) < set(self.options['plotSettings']['plot'][pltIndex].keys()) and self.options['plotSettings']['plot'][pltIndex]['type'] == 'histogram': + # dim = 2 + # else: + # self.raiseAnError(IOError, 'Wrong dimensionality or axis specification for plot '+self.name+'.') + # if self.dim is not None and self.dim != dim: + # self.raiseAnError(IOError, 'The OutStream Plot '+self.name+' combines 2D and 3D plots. This is not supported!') + # self.dim = dim + # if self.availableOutStreamTypes[self.dim].count(self.options['plotSettings']['plot'][pltIndex]['type']) == 0: + # self.raiseAMessage('For plot named' + self.name + ', type ' + self.options['plotSettings']['plot'][pltIndex]['type'] + ' is not among pre-defined plots! \n The OutstreamSystem will try to construct a call on the fly!', 'ExceptedError') + # self.outStreamTypes.append(self.options['plotSettings']['plot'][pltIndex]['type']) + # self.raiseADebug('matplotlib version is ' + str(matplotlib.__version__)) + + # if self.dim not in [2, 3]: + # self.raiseAnError(TypeError, 'This Plot interface is able to handle 2D-3D plot only') + + # if 'gridSpace' in self.options['plotSettings'].keys(): + # grid = list(map(int, self.options['plotSettings']['gridSpace'].split(' '))) + # self.gridSpace = matplotlib.gridspec.GridSpec(grid[0], grid[1]) + + ##################### # PRIVATE METHODS # ##################### @@ -447,7 +592,7 @@ def __executeActions(self): #plt.figure().gca(projection = '3d').xaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter()) #plt.figure().gca(projection = '3d').zaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter()) self.plt3D.ticklabel_format(**{'style':'sci', 'scilimits':(0,1), 'useOffset':False, 'axis':'both'}) - if 'title' not in self.options.keys(): + if 'title' not in self.options.keys(): if self.dim == 2: plt.title(self.name, fontdict = {'verticalalignment':'baseline', 'horizontalalignment':'center'}) if self.dim == 3: @@ -800,7 +945,7 @@ def initialize(self, inDict): self.mixtureCovars.append(self.options['plotSettings']['plot'][pltIndex]['attributes']['mixtureCovars'].split(',')) self.numberAggregatedOS = len(self.options['plotSettings']['plot']) # initialize here the base class - OutStreamManager.initialize(self, inDict) + OutStreamBase.initialize(self, inDict) # execute actions (we execute the actions here also because we can perform a check at runtime!! self.__executeActions() @@ -811,8 +956,10 @@ def localReadXML(self, xmlNode): @ In, xmlNode, xml.etree.ElementTree.Element, Xml element node @ Out, None """ - if 'dim' in xmlNode.attrib.keys(): + if 'dim' in xmlNode.attrib: self.raiseAnError(IOError,"the 'dim' attribute has been deprecated. This warning became an error in January 2017") + if 'overwrite' in xmlNode.attrib: + self.overwrite = utils.stringIsTrue(xmlNode.attrib['overwrite']) foundPlot = False for subnode in xmlNode: # if actions, read actions block diff --git a/framework/OutStreams/OutStreamManager.py b/framework/OutStreams/OutStreamBase.py similarity index 80% rename from framework/OutStreams/OutStreamManager.py rename to framework/OutStreams/OutStreamBase.py index fcd2c54179..33ae8d59fc 100644 --- a/framework/OutStreams/OutStreamManager.py +++ b/framework/OutStreams/OutStreamBase.py @@ -28,15 +28,26 @@ from BaseClasses import BaseType import DataObjects import Models -from utils import utils +from utils import utils, InputData, InputTypes #Internal Modules End----------------------------------------------------------- -class OutStreamManager(BaseType): +class OutStreamBase(BaseType): """ OUTSTREAM CLASS This class is a general base class for outstream action classes For example, a matplotlib interface class or Print class, etc. """ + @classmethod + def getInputSpecification(cls): + """ + Method to get a reference to a class that specifies the input data for class "cls". + @ In, cls, the class for which we are retrieving the specification + @ Out, inputSpecification, InputData.ParameterInput, class to use for specifying the input of cls. + """ + spec = BaseType.getInputSpecification() + spec.addParam('dir', param_type=InputTypes.StringType, required=False) + return spec + def __init__(self): """ Init of Base class @@ -69,25 +80,33 @@ def __init__(self): def _readMoreXML(self, xmlNode): """ - Function to read the portion of the xml input that belongs to this - specialized class and initialize some stuff based on the inputs received - @ In, xmlNode, xml.etree.ElementTree.Element, Xml element node + Function to read the portion of the input that belongs to this + specialized class and initialize based on the inputs received + @ In, xmlNode, xml.etree.ElementTree.Element, xml element node @ Out, None - The text is supposed to contain the info where and which variable to change. - In case of a code the syntax is specified by the code interface itself """ - if 'overwrite' in xmlNode.attrib.keys(): - if utils.stringIsTrue(xmlNode.attrib['overwrite']): - self.overwrite = True - else: - self.overwrite = False - if 'dir' in xmlNode.attrib: - self.subDirectory = xmlNode.attrib['dir'] - if '~' in self.subDirectory: - self.subDirectory= os.path.expanduser(self.subDirectory) - - self.localReadXML(xmlNode) + ## general options for all OutStreams + spec = self.getInputSpecification()() + spec.parseNode(xmlNode) + subDir = spec.parameterValues.get('dir', None) + if subDir: + subDir = os.path.expanduser(subDir) + self.subDirectory = subDir + ## pass remaining to inheritors + # if unconverted, use the old xml reading + if 'localReadXML' in dir(self): + self.localReadXML(xmlNode) + # otherwise it has _handleInput (and it should) and use input specs + else: + self._handleInput(spec) + def _handleInput(self, spec): + """ + Loads the input specs for this object. + @ In, spec, InputData.ParameterInput, input specifications + @ Out, None + """ + pass def getInitParams(self): """ diff --git a/framework/OutStreams/__init__.py b/framework/OutStreams/__init__.py index 4a272b21cb..0673c3f50a 100644 --- a/framework/OutStreams/__init__.py +++ b/framework/OutStreams/__init__.py @@ -25,13 +25,10 @@ ## These lines ensure that we do not have to do something like: ## 'from OutStreamManagers.OutStreamPlot import OutStreamPlot' outside ## of this submodule -from .OutStreamManager import OutStreamManager -from .OutStreamPlot import OutStreamPlot -from .OutStreamPrint import OutStreamPrint +from .OutStreamBase import OutStreamBase +from .FilePrint import FilePrint +from .GeneralPlot import GeneralPlot as Plot from .Factory import knownTypes from .Factory import returnInstance from .Factory import returnClass - -# We should not really need this as we do not use wildcard imports -__all__ = ['OutStreamManager','OutStreamPlot','OutStreamPrint'] diff --git a/framework/Simulation.py b/framework/Simulation.py index 3a90fedce4..de36ef1358 100644 --- a/framework/Simulation.py +++ b/framework/Simulation.py @@ -269,8 +269,7 @@ def __init__(self,frameworkDir,verbosity='all',interactive=Interaction.No): self.functionsDict = {} self.filesDict = {} # for each file returns an instance of a Files class self.metricsDict = {} - self.OutStreamManagerPlotDict = {} - self.OutStreamManagerPrintDict = {} + self.outStreamsDict = {} self.stepSequenceList = [] #the list of step of the simulation #list of supported queue-ing software: @@ -296,9 +295,7 @@ def __init__(self,frameworkDir,verbosity='all',interactive=Interaction.No): self.addWhatDict['Functions' ] = Functions self.addWhatDict['Files' ] = Files self.addWhatDict['Metrics' ] = Metrics - self.addWhatDict['OutStreams' ] = {} - self.addWhatDict['OutStreams' ]['Plot' ] = OutStreams - self.addWhatDict['OutStreams' ]['Print'] = OutStreams + self.addWhatDict['OutStreams' ] = OutStreams #Mapping between an entity type and the dictionary containing the instances for the simulation @@ -314,9 +311,7 @@ def __init__(self,frameworkDir,verbosity='all',interactive=Interaction.No): self.whichDict['Databases' ] = self.dataBasesDict self.whichDict['Functions' ] = self.functionsDict self.whichDict['Metrics' ] = self.metricsDict - self.whichDict['OutStreams'] = {} - self.whichDict['OutStreams']['Plot' ] = self.OutStreamManagerPlotDict - self.whichDict['OutStreams']['Print'] = self.OutStreamManagerPrintDict + self.whichDict['OutStreams' ] = self.outStreamsDict # The QApplication ## The benefit of this enumerated type is that anything other than @@ -425,17 +420,16 @@ def XMLread(self,xmlNode,runInfoSkip = set(),xmlFilename=None): varGroups={} # read other nodes for child in xmlNode: - if child.tag=='VariableGroups': + if child.tag == 'VariableGroups': continue #we did these before the for loop if child.tag in self.whichDict: self.raiseADebug('-'*2+' Reading the block: {0:15}'.format(str(child.tag))+2*'-') Class = child.tag - if len(child.attrib.keys()) == 0: + if len(child.attrib) == 0: globalAttributes = {} else: globalAttributes = child.attrib - #if 'verbosity' in globalAttributes.keys(): self.verbosity = globalAttributes['verbosity'] - if Class not in ['RunInfo','OutStreams'] and "returnInputParameter" in self.addWhatDict[Class].__dict__: + if Class not in ['RunInfo'] and "returnInputParameter" in self.addWhatDict[Class].__dict__: paramInput = self.addWhatDict[Class].returnInputParameter() paramInput.parseNode(child) for childChild in paramInput.subparts: @@ -457,26 +451,14 @@ def XMLread(self,xmlNode,runInfoSkip = set(),xmlFilename=None): #place the instance in the proper dictionary (self.whichDict[Type]) under his name as key, #the type is the general class (sampler, data, etc) while childChild.tag is the sub type #if name not in self.whichDict[Class].keys(): self.whichDict[Class][name] = self.addWhatDict[Class].returnInstance(childChild.tag,self) - if Class != 'OutStreams': - if name not in self.whichDict[Class].keys(): - if "needsRunInfo" in self.addWhatDict[Class].__dict__: - self.whichDict[Class][name] = self.addWhatDict[Class].returnInstance(childChild.tag,self.runInfoDict,self) - else: - self.whichDict[Class][name] = self.addWhatDict[Class].returnInstance(childChild.tag,self) + if name not in self.whichDict[Class]: + if "needsRunInfo" in self.addWhatDict[Class].__dict__: + self.whichDict[Class][name] = self.addWhatDict[Class].returnInstance(childChild.tag,self.runInfoDict,self) else: - self.raiseAnError(IOError,'Redundant naming in the input for class '+Class+' and name '+name) + self.whichDict[Class][name] = self.addWhatDict[Class].returnInstance(childChild.tag,self) else: - if name not in self.whichDict[Class][subType].keys(): - self.whichDict[Class][subType][name] = self.addWhatDict[Class][subType].returnInstance(childChild.tag,self) - else: - self.raiseAnError(IOError,'Redundant naming in the input for class '+Class+' and sub Type'+subType+' and name '+name) - #now we can read the info for this object - #if globalAttributes and 'verbosity' in globalAttributes.keys(): localVerbosity = globalAttributes['verbosity'] - #else : localVerbosity = self.verbosity - if Class != 'OutStreams': - self.whichDict[Class][name].readXML(childChild, self.messageHandler, varGroups, globalAttributes=globalAttributes) - else: - self.whichDict[Class][subType][name].readXML(childChild, self.messageHandler, globalAttributes=globalAttributes) + self.raiseAnError(IOError,'Redundant naming in the input for class '+Class+' and name '+name) + self.whichDict[Class][name].readXML(childChild, self.messageHandler, varGroups, globalAttributes=globalAttributes) else: self.raiseAnError(IOError,'not found name attribute for one "{}": {}'.format(Class,subType)) else: @@ -537,34 +519,20 @@ def checkStep(self,stepInstance,stepName): @ In, stepName, string, the name of the step to check @ Out, None """ - for [role,myClass,objectType,name] in stepInstance.parList: - if myClass!= 'Step' and myClass not in list(self.whichDict.keys()): - self.raiseAnError(IOError,'For step named '+stepName+' the role '+role+' has been assigned to an unknown class type '+myClass) - if myClass != 'OutStreams': - if name not in list(self.whichDict[myClass].keys()): - self.raiseADebug('name:',name) - self.raiseADebug('myClass:',myClass) - self.raiseADebug('list:',list(self.whichDict[myClass].keys())) - self.raiseADebug('whichDict[myClass]',self.whichDict[myClass]) - self.raiseAnError(IOError,'In step '+stepName+' the class '+myClass+' named '+name+' supposed to be used for the role '+role+' has not been found') - else: - if objectType not in self.whichDict[myClass].keys(): - self.raiseAnError(IOError,'In step "{}" class "{}" the type "{}" is not recognized!'.format(stepName,myClass,objectType)) - if name not in self.whichDict[myClass][objectType].keys(): - self.raiseADebug('name: '+name) - self.raiseADebug('list: '+str(list(self.whichDict[myClass][objectType].keys()))) - self.raiseADebug(str(self.whichDict[myClass][objectType])) - self.raiseAnError(IOError,'In step '+stepName+' the class '+myClass+' named '+name+' supposed to be used for the role '+role+' has not been found') - + for [role, myClass, objectType, name] in stepInstance.parList: + if myClass != 'Step' and myClass not in list(self.whichDict.keys()): + self.raiseAnError(IOError, f'For step named "{stepName}" the role "{role}" has been ' + + f'assigned to an unknown class type "{myClass}"!') + if name not in self.whichDict[myClass]: + self.raiseADebug('name:',name) + self.raiseADebug('myClass:',myClass) + self.raiseADebug('list:',list(self.whichDict[myClass].keys())) + self.raiseADebug('whichDict[myClass]',self.whichDict[myClass]) + self.raiseAnError(IOError, f'In step "{stepName}" the class "{myClass}" named "{name}" ' + + f'supposed to be used for the role "{role}" has not been found!') if myClass != 'Files': # check if object type is consistent - if myClass != 'OutStreams': - objtype = self.whichDict[myClass][name].type - else: - objtype = self.whichDict[myClass][objectType][name].type - if objectType != objtype.replace("OutStream",""): - objtype = self.whichDict[myClass][name].type - #self.raiseAnError(IOError,'In step '+stepName+' the class '+myClass+' named '+name+' used for role '+role+' has mismatching type. Type is "'+objtype.replace("OutStream","")+'" != inputted one "'+objectType+'"!') + objtype = self.whichDict[myClass][name].type def __readRunInfo(self,xmlNode,runInfoSkip,xmlFilename): """ @@ -729,8 +697,7 @@ def __prntDict(Dict,msg): #msg=__prntDict(self.testsDict,msg) msg=__prntDict(self.filesDict,msg) msg=__prntDict(self.dataBasesDict,msg) - msg=__prntDict(self.OutStreamManagerPlotDict,msg) - msg=__prntDict(self.OutStreamManagerPrintDict,msg) + msg=__prntDict(self.outStreamsDict,msg) msg=__prntDict(self.addWhatDict,msg) msg=__prntDict(self.whichDict,msg) self.raiseADebug(msg) @@ -763,10 +730,7 @@ def run(self): for [key,b,c,d] in stepInstance.parList: #Only for input and output we allow more than one object passed to the step, so for those we build a list if key == 'Input' or key == 'Output': - if b == 'OutStreams': - stepInputDict[key].append(self.whichDict[b][c][d]) - else: - stepInputDict[key].append(self.whichDict[b][d]) + stepInputDict[key].append(self.whichDict[b][d]) else: stepInputDict[key] = self.whichDict[b][d] #add the global objects diff --git a/framework/Steps.py b/framework/Steps.py index 076971aeda..6e153065bf 100644 --- a/framework/Steps.py +++ b/framework/Steps.py @@ -42,7 +42,7 @@ from utils import utils from utils import InputData, InputTypes import Models -from OutStreams import OutStreamManager +from OutStreams import OutStreamBase from DataObjects import DataObject #Internal Modules End-------------------------------------------------------------------------------- @@ -448,7 +448,7 @@ def _localInitializeStep(self,inDictionary): #if type(inDictionary['Output'][i]).__name__ not in ['str','bytes','unicode']: if 'HDF5' in inDictionary['Output'][i].type: inDictionary['Output'][i].initialize(self.name) - elif inDictionary['Output'][i].type in ['OutStreamPlot','OutStreamPrint']: + elif isinstance(inDictionary['Output'][i], OutStreamBase): inDictionary['Output'][i].initialize(inDictionary) self.raiseADebug('for the role Output the item of class {0:15} and name {1:15} has been initialized'.format(inDictionary['Output'][i].type,inDictionary['Output'][i].name)) self._registerMetadata(inDictionary) @@ -488,8 +488,8 @@ def _localTakeAstepRun(self,inDictionary): if finishedJob.getReturnCode() == 0: # if the return code is > 0 => means the system code crashed... we do not want to make the statistics poor => we discard this run for output in outputs: - if output.type not in ['OutStreamPlot','OutStreamPrint']: - model.collectOutput(finishedJob,output) + if not isinstance(output, OutStreamBase): + model.collectOutput(finishedJob, output) else: output.addOutput() else: @@ -603,7 +603,7 @@ def _localInitializeStep(self,inDictionary): self._outputDictCollectionLambda = [] # set up output collection lambdas for outIndex, output in enumerate(inDictionary['Output']): - if output.type not in ['OutStreamPlot','OutStreamPrint']: + if not isinstance(output, OutStreamBase): if 'SolutionExport' in inDictionary.keys() and output.name == inDictionary['SolutionExport'].name: self._outputCollectionLambda.append((lambda x:None, outIndex)) self._outputDictCollectionLambda.append((lambda x:None, outIndex)) @@ -856,7 +856,7 @@ def __getOutputs(self, inDictionary): """ outputs = [] for out in inDictionary['Output']: - if not isinstance(out,OutStreamManager): + if not isinstance(out, OutStreamBase): outputs.append(out) return outputs @@ -961,9 +961,9 @@ def _localInitializeStep(self,inDictionary): filename = os.path.join(self.fromDirectory, inInput.name) inInput.load(filename, style='csv') - #Initialize all the OutStreamPrint and OutStreamPlot outputs + #Initialize all the OutStreams for output in inDictionary['Output']: - if type(output).__name__ in ['OutStreamPrint','OutStreamPlot']: + if isinstance(output, OutStreamBase): output.initialize(inDictionary) self.raiseADebug('for the role Output the item of class {0:15} and name {1:15} has been initialized'.format(output.type,output.name)) # register metadata @@ -1045,7 +1045,7 @@ def _localTakeAstepRun(self,inDictionary): self.raiseAnError(IOError,"Unknown action type "+self.actionType[i]) for output in inDictionary['Output']: - if output.type in ['OutStreamPrint','OutStreamPlot']: + if isinstance(output, OutStreamBase): output.addOutput() def _localGetInitParams(self): diff --git a/framework/utils/InputData.py b/framework/utils/InputData.py index a71ca485ef..e98ee94670 100644 --- a/framework/utils/InputData.py +++ b/framework/utils/InputData.py @@ -275,9 +275,10 @@ def addSub(cls, sub, quantity=Quantity.zero_to_infinity): """ cls.subs[sub] = None subsSet = cls._subDict.get(sub.getName(), set()) - if (len(subsSet) == 1 and next(iter(subsSet))._checkCanRead is None) or \ - (len(subsSet) > 0 and sub._checkCanRead is not None): - print("ERROR adding checked and unchecked to", sub.getName()," in ", + if __debug__: + if (len(subsSet) == 1 and next(iter(subsSet))._checkCanRead is None) or \ + (len(subsSet) > 0 and sub._checkCanRead is not None): + print("INPUT SPEC ERROR adding checked and unchecked to", sub.getName()," in ", cls.getName()+" len "+str(len(subsSet))) subsSet.add(sub) cls._subDict[sub.getName()] = subsSet @@ -353,7 +354,7 @@ def setContentType(cls, contentType): """ cls.contentType = contentType - def parseNode(self,node, errorList = None): + def parseNode(self, node, errorList=None): """ Parses the xml node and puts the results in self.parameterValues and self.subparts and self.value @@ -426,7 +427,7 @@ def handleError(s): self.subparts.append(subInstance) elif self.strictMode: allowed = [s.getName() for s in subs] - handleError('no class to handle '+childName+' tried '+str(subsSet)+" allowed:"+str(allowed)) #Extra if debugging: + ' keys: '+str(set(self._subDict.keys()))+ str({k: [j.getName() for j in self._subDict[k]] for k in self._subDict.keys()})) + handleError(f'Unrecognized input: "{childName}"! Allowed: "{allowed}", tried "{subsSet}"') if self.strictMode: nodeNames = set([child.tag for child in node]) if nodeNames != subNames: diff --git a/plugins/HERON b/plugins/HERON index 53cbd51dc8..102e84a8da 160000 --- a/plugins/HERON +++ b/plugins/HERON @@ -1 +1 @@ -Subproject commit 53cbd51dc837258b7dcc4cb20305570cc6bff9d5 +Subproject commit 102e84a8dad209b801974e531ab6f67a81f7174c diff --git a/tests/framework/TestXSD/TestStrictCheck.py b/tests/framework/TestXSD/TestStrictCheck.py index c58e40f99f..e8f30bbfa8 100644 --- a/tests/framework/TestXSD/TestStrictCheck.py +++ b/tests/framework/TestXSD/TestStrictCheck.py @@ -60,7 +60,7 @@ def checkAnswer(expected, actual): checkAnswer('Required parameter required_string not in inner', errors[0]) checkAnswer('no_such_element not in attributes and strict mode on in inner', errors[1]) -checkAnswer("no class to handle no_such_sub tried set() allowed:['sub_1', 'sub_2', 'sub_3', 'sub_bool']", errors[2]) +checkAnswer('Unrecognized input: "no_such_sub"! Allowed: "[\'sub_1\', \'sub_2\', \'sub_3\', \'sub_bool\']", tried "set()"', errors[2]) print("passes",passFails[0],"fails",passFails[1]) sys.exit(passFails[1]) diff --git a/tests/framework/test_output.xml b/tests/framework/test_output.xml index 54801da83b..7357bab816 100644 --- a/tests/framework/test_output.xml +++ b/tests/framework/test_output.xml @@ -96,7 +96,7 @@ csv PointSets - + line @@ -684,7 +684,7 @@ test fullprint_Pointset 2DHistoryPlot2 - test + testPlot plot2 plot3 plot4 diff --git a/tests/framework/tests b/tests/framework/tests index 21cf738526..e8b2e5804c 100644 --- a/tests/framework/tests +++ b/tests/framework/tests @@ -41,7 +41,7 @@ #REQUIREMENT_TEST R-RA-3 type = 'RavenFramework' input = 'test_output.xml' - output = 'output_check/1-plot7_stem.png output_check/1-2DHistoryPlot_histogram.png output_check/1-test_line-line.png output_check/1-plot2_line-line.png output_check/1-plot4_scatter-scatter.png output_check/1-plot3_scatter-scatter.png output_check/1-plot9_stem.png output_check/1-plot8_stem.png output_check/1-plot6_histogram.png output_check/1-plot5_histogram.png output_check/1-plot11_step.png output_check/1-plot10_stem.png output_check/1-plot14_tri-surface-tri-surface.png output_check/1-plot13_surface-surface.png output_check/1-plot12_pseudocolor.png output_check/1-plot17_contour3D-filledContour3D.png output_check/1-plot16_contour-filledContour.png output_check/1-plot15_wireframe-wireframe.png output_check/1-plot18_scatter.png output_check/1-plot19_scatter.png output_check/fullprint_Pointset.csv' + output = 'output_check/1-plot7_stem.png output_check/1-2DHistoryPlot_histogram.png output_check/1-testPlot_line-line.png output_check/1-plot2_line-line.png output_check/1-plot4_scatter-scatter.png output_check/1-plot3_scatter-scatter.png output_check/1-plot9_stem.png output_check/1-plot8_stem.png output_check/1-plot6_histogram.png output_check/1-plot5_histogram.png output_check/1-plot11_step.png output_check/1-plot10_stem.png output_check/1-plot14_tri-surface-tri-surface.png output_check/1-plot13_surface-surface.png output_check/1-plot12_pseudocolor.png output_check/1-plot17_contour3D-filledContour3D.png output_check/1-plot16_contour-filledContour.png output_check/1-plot15_wireframe-wireframe.png output_check/1-plot18_scatter.png output_check/1-plot19_scatter.png output_check/fullprint_Pointset.csv' csv = 'output_check/fullprint_HistorySet_2.csv output_check/test_4.csv output_check/testprint_selective_hist1_4.csv output_check/testprint_selective_Pointset.csv' UnorderedXml = 'output_check/fullprint_HistorySet.xml output_check/fullprint_Pointset.xml' remove_unicode_identifier = true diff --git a/tests/framework/unit_tests/DataObjects/TestDataSet.py b/tests/framework/unit_tests/DataObjects/TestDataSet.py index b0126bc6d7..a9c68cebc9 100644 --- a/tests/framework/unit_tests/DataObjects/TestDataSet.py +++ b/tests/framework/unit_tests/DataObjects/TestDataSet.py @@ -572,7 +572,7 @@ def formatRealization(rlz): # to CSV ## test writing to file csvname = 'DataSetUnitTest' -data.write(csvname,style='CSV',**{'what':'a,b,c,x,y,z,RAVEN_sample_ID,prefix'}) +data.write(csvname,style='CSV',**{'what':'a,b,c,x,y,z,RAVEN_sample_ID,prefix'.split(',')}) ## test metadata written correct = ['', ' ', diff --git a/tests/framework/unit_tests/DataObjects/TestHistorySet.py b/tests/framework/unit_tests/DataObjects/TestHistorySet.py index dd3c5560ad..32bcce2f99 100644 --- a/tests/framework/unit_tests/DataObjects/TestHistorySet.py +++ b/tests/framework/unit_tests/DataObjects/TestHistorySet.py @@ -522,7 +522,7 @@ def formatRealization(rlz): # to CSV ## test writing to file csvname = 'HistorySetUnitTest' -data.write(csvname,style='CSV',**{'what':'a,b,c,x,y,z,RAVEN_sample_ID,prefix'}) +data.write(csvname,style='CSV',**{'what':'a,b,c,x,y,z,RAVEN_sample_ID,prefix'.split(',')}) ## test metadata written correct = ['', ' ', @@ -733,7 +733,7 @@ def formatRealization(rlz): data.addRealization(rlz1) data.addRealization(rlz2) csvname = 'HSVectorMetaUnitTest' -data.write(csvname,style='CSV',**{'what':'a,b,c,y,RAVEN_sample_ID,prefix,vectorMeta'}) +data.write(csvname,style='CSV',**{'what':'a,b,c,y,RAVEN_sample_ID,prefix,vectorMeta'.split(',')}) ## test metadata written correct = ['', ' ', @@ -780,7 +780,7 @@ def formatRealization(rlz): os.remove(csvname+'_1.csv') csvname = 'HSVectorMetaUnitTest' -dataCSV.write(csvname,style='CSV',**{'what':'a,b,c,y,RAVEN_sample_ID,prefix,vectorMeta'}) +dataCSV.write(csvname,style='CSV',**{'what':'a,b,c,y,RAVEN_sample_ID,prefix,vectorMeta'.split(',')}) # read in XML lines = open(csvname+'.xml','r').readlines() # remove line endings @@ -803,7 +803,7 @@ def formatRealization(rlz): os.remove(csvname+'_0.csv') os.remove(csvname+'_1.csv') csvname = 'HSVectorMetaUnitTest' -dataCSV.write(csvname,style='CSV',**{'what':'a,b,c,y,RAVEN_sample_ID,prefix,vectorMeta'}) +dataCSV.write(csvname,style='CSV',**{'what':'a,b,c,y,RAVEN_sample_ID,prefix,vectorMeta'.split(',')}) # read in XML lines = open(csvname+'.xml','r').readlines() # remove line endings diff --git a/tests/framework/unit_tests/DataObjects/TestPointSet.py b/tests/framework/unit_tests/DataObjects/TestPointSet.py index 40ecc5e3b8..96fa227219 100644 --- a/tests/framework/unit_tests/DataObjects/TestPointSet.py +++ b/tests/framework/unit_tests/DataObjects/TestPointSet.py @@ -480,7 +480,7 @@ def formatRealization(rlz): # to CSV ## test writing to file csvname = 'PointSetUnitTest' -data.write(csvname,style='CSV',**{'what':'a,b,c,x,y,z,RAVEN_sample_ID,prefix'}) +data.write(csvname,style='CSV',**{'what':'a,b,c,x,y,z,RAVEN_sample_ID,prefix'.split(',')}) ## test metadata written correct = ['', ' ',