diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 06e2aa5..cadab31 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,6 +66,7 @@ repos: - id: mixed-line-ending - id: name-tests-test args: [--pytest-test-first] + exclude: tests/helpers - id: no-commit-to-branch stages: - pre-push diff --git a/Makefile b/Makefile index 81fc923..f2e0bba 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ pydocstyle: lint: black isort pydocstyle mypy: - mypy . + mypy src gen_classes: cd src && xsdata generate -c $(CONFIG) -p greenbutton_objects.data.espi https://www.naesb.org/espi.xsd diff --git a/pyproject.toml b/pyproject.toml index 02c26be..020e7d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,10 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3" ] -keywords = ["feed", "reader", "tutorial"] dependencies = [ "xsdata" ] +keywords = ["feed", "reader"] requires-python = ">=3.9" [project.optional-dependencies] diff --git a/tests/test_parse/__init__.py b/src/greenbutton_objects/atom/__init__.py similarity index 100% rename from tests/test_parse/__init__.py rename to src/greenbutton_objects/atom/__init__.py diff --git a/src/greenbutton_objects/atom/href_tree.py b/src/greenbutton_objects/atom/href_tree.py new file mode 100644 index 0000000..ff6f853 --- /dev/null +++ b/src/greenbutton_objects/atom/href_tree.py @@ -0,0 +1,97 @@ +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from greenbutton_objects.data.atom import ContentType, EntryType, Feed + + +@dataclass +class HRefTreeNode: + uri: str + parent: Optional[str] = None + contentType: type = type(None) + content: List[ContentType] = field(default_factory=list) + children: List[str] = field(default_factory=list) + related: List[str] = field(default_factory=list) + title: str = "" + + +class HRefForest: + def __init__(self) -> None: + self.forest: Dict[str, HRefTreeNode] = {} + + def __ensure_container(self, uri: str) -> None: + if uri not in self.forest: + self.forest[uri] = HRefTreeNode(uri) + + def __link_parents(self) -> "HRefForest": + for node in self.forest.values(): + if node.parent: + parent_node = self.forest.get(node.parent) + if parent_node: + parent_node.children.append(node.uri) + return self + + def __ensure_containers(self) -> "HRefForest": + for key in list(self.forest.keys()): + node = self.forest[key] + if node.parent: + self.__ensure_container(node.parent) + for related_uri in node.related: + self.__ensure_container(related_uri) + return self + + def __add_nodes(self, feed: Feed) -> "HRefForest": + def entry_content_type(entry: EntryType) -> type: + if entry.content and entry.content[0].content: + content_type = type(entry.content[0].content[0]) + else: + content_type = type(None) + return content_type + + for entry in feed.entry: + related = [] + parent = None + uri = "" + + content_type = entry_content_type(entry) + + for link in entry.link: + # Skip links without URIs + if not link.href: + continue + if link.rel == "self": + uri = link.href + elif link.rel == "related": + related.append(link.href) + elif link.rel == "up": + parent = link.href + + title = self.get_entry_title(entry) + + self.forest[uri] = HRefTreeNode( + uri=uri, + title=title, + parent=parent, + related=related, + contentType=content_type, + content=entry.content, + ) + + return self + + @staticmethod + def get_entry_title(entry: EntryType) -> str: + if entry.title: + title_parts = [] + for text in entry.title: + if len(text.content) > 0: + title_parts.append(text.content[0]) + return "".join(title_parts) # type: ignore + else: + return "" + + def build(self, feed: Feed) -> "HRefForest": + return self.__add_nodes(feed).__ensure_containers().__link_parents() + + def root_nodes(self) -> List[str]: + return [node.uri for node in self.forest.values() if node.parent is None] diff --git a/src/greenbutton_objects/atom/object_tree.py b/src/greenbutton_objects/atom/object_tree.py new file mode 100644 index 0000000..805fc52 --- /dev/null +++ b/src/greenbutton_objects/atom/object_tree.py @@ -0,0 +1,104 @@ +from dataclasses import dataclass, field +from itertools import chain +from typing import Any, Dict, Iterable, List, Optional, Sequence, Union + +from greenbutton_objects.atom.href_tree import HRefForest, HRefTreeNode +from greenbutton_objects.util import get_first + + +@dataclass +class EntryNode: + title: str + uri: str + content: Sequence[object] + + parent: Optional["EntryNode"] = None + content_type: type = type(None) + children_type: type = type(None) + children: List["EntryNode"] = field(default_factory=list) + # TODO: Should related be a list or dict keyed by type? + related: List["EntryNode"] = field(default_factory=list) + + def infer_children_type(self) -> None: + # We are making a big assumption here that all children are of the same type. + # This is to make things simpler for the consumer who's using the library + if self.children and self.children[0].content: + content_ = self.children[0].content[0] + if content_.content: # type: ignore + self.children_type = content_.content[0].__class__ # type: ignore + else: + self.children_type = type(None) + + def first_content(self) -> Any: + first_node = get_first(self.content) + first_node_content = first_node.content # type: ignore + return get_first(first_node_content) + + def get_related_of_type(self, elements_type: type) -> Iterable["EntryNode"]: + containers = [obj for obj in self.related if obj.children_type is elements_type] + if containers: + elements: Iterable[EntryNode] = chain.from_iterable( + container.children for container in containers + ) + else: + elements = [obj for obj in self.related if obj.content_type is elements_type] + return elements + + def safe_get_content(self, content_type: type) -> Union[Any, None]: + obj = get_first(self.get_related_of_type(content_type)) + return obj.first_content() if obj else None + + +class ObjectForest: + def __init__( + self, + ) -> None: + self.__roots: List[EntryNode] = [] + + def build(self, href_forest: HRefForest) -> "ObjectForest": + node_cache: Dict[str, EntryNode] = {} + + def add_node(href_node: HRefTreeNode) -> EntryNode: + entry_node = EntryNode( + title=href_node.title, + uri=href_node.uri, + content=href_node.content, + content_type=href_node.contentType, + ) + node_cache[href_node.uri] = entry_node + return entry_node + + def build_tree(node_uri: str) -> EntryNode: + if node_uri not in node_cache: + href_node = href_forest.forest[node_uri] + entry_node = add_node(href_node) + + # Children + entry_node.children = [build_tree(child) for child in href_node.children] + for child in entry_node.children: + child.parent = entry_node + entry_node.infer_children_type() + + # Relatives + entry_node.related = [build_tree(child) for child in href_node.related] + + return node_cache[node_uri] + + self.__roots = [build_tree(uri) for uri in href_forest.root_nodes()] + + return self + + @staticmethod + def get_elements_by_type(elements_type: type, source: list[EntryNode]) -> Iterable[EntryNode]: + containers = [obj for obj in source if obj.children_type is elements_type] + if containers: + elements: Iterable[EntryNode] = chain.from_iterable( + container.children for container in containers + ) + else: + elements = [obj for obj in source if obj.content_type is elements_type] + return elements + + @property + def roots(self) -> List[EntryNode]: + return self.__roots diff --git a/src/greenbutton_objects/enums.py b/src/greenbutton_objects/enums.py deleted file mode 100644 index 74058eb..0000000 --- a/src/greenbutton_objects/enums.py +++ /dev/null @@ -1,221 +0,0 @@ -from enum import Enum - - -class AccumulationBehaviourType(Enum): - notApplicable = 0 - bulkQuantity = 1 - cumulative = 3 - deltaData = 4 - indicating = 6 - summation = 9 - instantaneous = 12 - - -class CommodityType(Enum): - notApplicable = 0 - electricity = 1 - air = 4 - naturalGas = 7 - propane = 8 - potableWater = 9 - steam = 10 - wastewater = 11 - heatingFluid = 12 - coolingFluid = 13 - - -class ConsumptionTierType(Enum): - notApplicable = 0 - blockTier1 = 1 - blockTier2 = 2 - blockTier3 = 3 - blockTier4 = 4 - blockTier5 = 5 - blockTier6 = 6 - blockTier7 = 7 - blockTier8 = 8 - blockTier9 = 9 - blockTier10 = 10 - blockTier11 = 11 - blockTier12 = 12 - blockTier13 = 13 - blockTier14 = 14 - blockTier15 = 15 - blockTier16 = 16 - - -class CurrencyCode(Enum): - na = 0 - aus = 36 - cad = 124 - usd = 840 - eur = 978 - - @property - def symbol(self): - if self in [CurrencyCode.aus, CurrencyCode.cad, CurrencyCode.usd]: - return "$" - elif self is CurrencyCode.eur: - return "€" - else: - return "¤" - - -class DataQualifierType(Enum): - notApplicable = 0 - average = 2 - maximum = 8 - minimum = 9 - normal = 12 - - -class FlowDirectionType(Enum): - notApplicable = 0 - forward = 1 - reverse = 19 - - -class KindType(Enum): - notApplicable = 0 - currency = 3 - current = 4 - currentAngle = 5 - date = 7 - demand = 8 - energy = 12 - frequency = 15 - power = 37 - powerFactor = 38 - quantityPower = 40 - voltage = 54 - voltageAngle = 55 - distortionPowerFactor = 64 - volumetricFlow = 155 - - -class PhaseCode(Enum): - notApplicable = 0 - c = 32 - ca = 40 - b = 64 - bc = 66 - a = 128 - an = 129 - ab = 132 - abc = 224 - s2 = 256 - s2n = 257 - s1 = 512 - s1n = 513 - s1s2 = 768 - s1s2n = 769 - - -class QualityOfReading(Enum): - valid = 0 - manuallyEdited = 7 - estimatedUsingReferenceDay = 8 - estimatedUsingLinearInterpolation = 9 - questionable = 10 - derived = 11 - projected = 12 - mixed = 13 - raw = 14 - normalizedForWeather = 15 - other = 16 - validated = 17 - verified = 18 - - -class ServiceKind(Enum): - electricity = 0 - naturalGas = 1 - water = 2 - pressure = 4 - heat = 5 - cold = 6 - communication = 7 - time = 8 - - -class TimeAttributeType(Enum): - notApplicable = 0 - tenMinutes = 1 - fifteenMinutes = 2 - twentyFourHours = 4 - thirtyMinutes = 5 - sixtyMinutes = 7 - daily = 11 - monthly = 13 - present = 15 - previous = 16 - weekly = 24 - forTheSpecifiedPeriod = 32 - daily30MinuteFixedBlock = 79 - - -class UomType(Enum): - notApplicable = 0 - amps = 5 - volts = 29 - joules = 31 - hertz = 33 - watts = 38 - cubicMeters = 42 - voltAmps = 61 - voltAmpsReactive = 63 - cosine = 65 - voltsSquared = 67 - ampsSquared = 69 - voltAmpHours = 71 - wattHours = 72 - voltAmpReactiveHours = 73 - ampHours = 106 - cubicFeet = 119 - cubicFeetPerHour = 122 - cubicMetersPerHour = 125 - usGallons = 128 - usGallonsPerHour = 129 - imperialGallons = 130 - imperialGallonsPerHour = 131 - britishThermalUnits = 132 - britishThermalUnitsPerHour = 133 - liters = 134 - litersPerHour = 137 - gaugePascals = 140 - absolutePascals = 155 - therms = 169 - - -UOM_SYMBOLS = { - UomType.notApplicable: "", - UomType.amps: "A", - UomType.volts: "V", - UomType.joules: "J", - UomType.hertz: "Hz", - UomType.watts: "W", - UomType.cubicMeters: "m³", - UomType.voltAmps: "VA", - UomType.voltAmpsReactive: "VAr", - UomType.cosine: "cos", - UomType.voltsSquared: "V²", - UomType.ampsSquared: "A²", - UomType.voltAmpHours: "VAh", - UomType.wattHours: "Wh", - UomType.voltAmpReactiveHours: "VArh", - UomType.ampHours: "Ah", - UomType.cubicFeet: "ft³", - UomType.cubicFeetPerHour: "ft³/h", - UomType.cubicMetersPerHour: "m³/h", - UomType.usGallons: "US gal", - UomType.usGallonsPerHour: "US gal/h", - UomType.imperialGallons: "IMP gal", - UomType.imperialGallonsPerHour: "IMP gal/h", - UomType.britishThermalUnits: "BTU", - UomType.britishThermalUnitsPerHour: "BTU/h", - UomType.liters: "L", - UomType.litersPerHour: "L/h", - UomType.gaugePascals: "Pag", - UomType.absolutePascals: "Pa", - UomType.therms: "thm", -} diff --git a/src/greenbutton_objects/feed/__init__.py b/src/greenbutton_objects/feed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/greenbutton_objects/feed/feed.py b/src/greenbutton_objects/feed/feed.py new file mode 100644 index 0000000..b25034f --- /dev/null +++ b/src/greenbutton_objects/feed/feed.py @@ -0,0 +1,113 @@ +from typing import Any, List, TypeVar + +import greenbutton_objects.orig_objects as ob +from greenbutton_objects.atom.object_tree import ObjectForest +from greenbutton_objects.data import espi +from greenbutton_objects.util import get_first, get_value + +T = TypeVar("T") + + +class ObjectFeed: + def __init__(self) -> None: + self.usage_points: List[ob.UsagePoint] = [] + + def build(self, entry_forest: ObjectForest) -> "ObjectFeed": + usage_points = ObjectForest.get_elements_by_type(espi.UsagePoint, entry_forest.roots) + + for up_node in usage_points: + # TODO: We are assuming that there is only one UsagePoint object + # within the usage point node + up = up_node.first_content() + + local_time_params = up_node.safe_get_content(espi.LocalTimeParameters) + electric_power_usage_summary = up_node.safe_get_content(espi.ElectricPowerUsageSummary) + + meter_readings: List[ob.MeterReading] = [] + for mr_node in up_node.get_related_of_type(espi.MeterReading): + reading_type: espi.ReadingType | None = mr_node.safe_get_content(espi.ReadingType) + + interval_block_nodes = mr_node.get_related_of_type(espi.IntervalBlock) + interval_blocks: List[ob.IntervalBlock] = [] + + combined_readings: List[ob.IntervalReading] = [] + for interval_block_node in interval_block_nodes: + interval_block_content = get_first(interval_block_node.content) + + for interval_block in interval_block_content.content: # type: ignore + block = ob.IntervalBlock( + uri=interval_block_node.uri, interval=interval_block.interval + ) + + self.process_readings(block, interval_block) + interval_blocks.append(block) + combined_readings.extend(block.readings) + + reading = ob.MeterReading( + title=mr_node.title, + uri=mr_node.uri, + reading_type=reading_type if reading_type else espi.ReadingType(), + intervalBlock=tuple(interval_blocks), + readings=tuple(combined_readings), + ) + + for ib in reading.intervalBlock: + ib.compute_multiplier(reading.reading_type) + reading.patch() + + meter_readings.append(reading) + + service_kind = up.service_category.kind + if service_kind == "" or service_kind is None: + # TODO: We are forcing natural gas here. But we should either use + # hint given to us, apply heuristic rules or throw error + service_kind = ob.ServiceKind.GAS + for mr in meter_readings: + mr.reading_type.uom = ob.UnitSymbol.THERM.value + mr.reading_type.power_of_ten_multiplier = espi.UnitMultiplierKindValue.VALUE_MINUS_3 + for ib in mr.intervalBlock: + ib.compute_multiplier(mr.reading_type) + + else: + service_kind = ob.ServiceKind(service_kind.value) + + self.usage_points.append( + ob.UsagePoint( + title=up_node.title, + service_kind=service_kind, + local_time_parameters=local_time_params, + electric_power_usage_summary=electric_power_usage_summary, + meter_readings=tuple(meter_readings), + uri=up_node.uri, + ) + ) + + return self + + def process_readings(self, block: ob.IntervalBlock, interval_block: espi.IntervalBlock) -> None: + # TODO: If quality of reading is not in readings it might be in the + # top level description + readings = [] + for interval_reading in interval_block.interval_reading: + reading_values = self.filter_allowed(interval_reading) + if interval_reading.reading_quality: + quality = interval_reading.reading_quality[0].quality + quality_of_reading = get_value( + quality, + missing_val=ob.QualityOfReading.MISSING, + src_type=espi.QualityOfReadingValue, + dest_type=ob.QualityOfReading, + ) + reading_values["quality_of_reading"] = quality_of_reading + reading_values["raw_value"] = interval_reading.value + readings.append(ob.IntervalReading(**reading_values, parent=block)) + block.readings = readings + + allowed_reading_keys = ("consumption_tier", "cost", "cpp", "time_period", "tou") + + def filter_allowed(self, interval_reading: espi.IntervalReading) -> dict[str, Any]: + reading_values = {} + for k, v in interval_reading.__dict__.items(): + if k in self.allowed_reading_keys and v is not None: + reading_values[k] = v + return reading_values diff --git a/src/greenbutton_objects/objects.py b/src/greenbutton_objects/objects.py deleted file mode 100644 index cde506a..0000000 --- a/src/greenbutton_objects/objects.py +++ /dev/null @@ -1,112 +0,0 @@ -import datetime -import functools - -from greenbutton_objects import enums, utils - - -@functools.total_ordering -class DateTimeInterval: - def __init__(self, entity): - self.start = utils.getEntity( - entity, - "espi:start", - lambda e: datetime.datetime.fromtimestamp(int(e.text)).astimezone(datetime.timezone.utc), - ) - self.duration = utils.getEntity( - entity, "espi:duration", lambda e: datetime.timedelta(seconds=int(e.text)) - ) - - def __repr__(self): - return "" % (self.start, self.duration) - - def __eq__(self, other): - if not isinstance(other, DateTimeInterval): - return False - return (self.start, self.duration) == (other.start, other.duration) - - def __lt__(self, other): - if not isinstance(other, DateTimeInterval): - return False - return (self.start, self.duration) < (other.start, other.duration) - - -@functools.total_ordering -class IntervalReading: - def __init__(self, entity, parent): - self.intervalBlock = parent - self.cost = utils.getEntity(entity, "espi:cost", lambda e: int(e.text) / 100000.0) - self.timePeriod = utils.getEntity(entity, "espi:timePeriod", lambda e: DateTimeInterval(e)) - self._value = utils.getEntity(entity, "espi:value", lambda e: int(e.text)) - - self.readingQualities = set([ - ReadingQuality(rq, self) for rq in entity.findall("espi:ReadingQuality", utils.ns) - ]) - - def __repr__(self): - return "" % ( - self.timePeriod.start, - self.timePeriod.duration, - self.value, - self.value_symbol, - ) - - def __eq__(self, other): - if not isinstance(other, IntervalReading): - return False - return (self.timePeriod, self.value) == (other.timePeriod, other.value) - - def __lt__(self, other): - if not isinstance(other, IntervalReading): - return False - return (self.timePeriod, self.value) < (other.timePeriod, other.value) - - @property - def value(self): - if ( - self.intervalBlock is not None - and self.intervalBlock.meterReading is not None - and self.intervalBlock.meterReading.readingType is not None - and self.intervalBlock.meterReading.readingType.powerOfTenMultiplier is not None - ): - multiplier = 10**self.intervalBlock.meterReading.readingType.powerOfTenMultiplier - else: - multiplier = 1 - return self._value * multiplier - - @property - def cost_units(self): - if ( - self.intervalBlock is not None - and self.intervalBlock.meterReading is not None - and self.intervalBlock.meterReading.readingType is not None - and self.intervalBlock.meterReading.readingType.currency is not None - ): - return self.intervalBlock.meterReading.readingType.currency - else: - return enums.CurrencyCode.na - - @property - def cost_symbol(self): - return self.cost_units.symbol - - @property - def value_units(self): - if ( - self.intervalBlock is not None - and self.intervalBlock.meterReading is not None - and self.intervalBlock.meterReading.readingType is not None - and self.intervalBlock.meterReading.readingType.uom is not None - ): - return self.intervalBlock.meterReading.readingType.uom - else: - return enums.UomType.notApplicable - - @property - def value_symbol(self): - return enums.UOM_SYMBOLS[self.value_units] - - -class ReadingQuality: - def __init__(self, entity, parent): - self.intervalReading = parent - self.quality = utils.getEntity(entity, "espi:quality", lambda e: enums.QualityOfReading(int(e.text))) diff --git a/src/greenbutton_objects/orig_objects/__init__.py b/src/greenbutton_objects/orig_objects/__init__.py new file mode 100644 index 0000000..1cc17d9 --- /dev/null +++ b/src/greenbutton_objects/orig_objects/__init__.py @@ -0,0 +1,12 @@ +from .enums import QualityOfReading, ServiceKind, UnitSymbol +from .objects import IntervalBlock, IntervalReading, MeterReading, UsagePoint + +__all__ = [ + "UsagePoint", + "ServiceKind", + "QualityOfReading", + "UnitSymbol", + "IntervalBlock", + "MeterReading", + "IntervalReading", +] diff --git a/src/greenbutton_objects/orig_objects/enums.py b/src/greenbutton_objects/orig_objects/enums.py new file mode 100644 index 0000000..2018923 --- /dev/null +++ b/src/greenbutton_objects/orig_objects/enums.py @@ -0,0 +1,408 @@ +""" +These are enum objects that don't have descriptive values in the original +XSD file, these types are created by LLM using the following prompt: + +``` +Transform this into into a Enum class and a dictionary + +Dictionary should contain mapping between numeric enum value and the text description. +Enum class should be equivalent to the original class but have descrptive abreveations +for the values instead of simple VALUE_0 + +Include references to the original class. + +Also add a enum constant that represents a missing value +``` + + +TODO: Would be interesting to put it into CI/CD + + +""" + +from enum import Enum + + +class ServiceKind(Enum): + """ + Enum representing the type of service. + Equivalent to the original ServiceKindValue class. + """ + + ELECTRICITY = 0 + GAS = 1 + WATER = 2 + TIME = 3 + HEAT = 4 + REFUSE = 5 + SEWERAGE = 6 + RATES = 7 + TV_LICENSE = 8 + INTERNET = 9 + MISSING = -1 + + +SERVICE_KIND_DESCRIPTIONS = { + ServiceKind.ELECTRICITY.value: "Electricity service.", + ServiceKind.GAS.value: "Gas service.", + ServiceKind.WATER.value: "Water service.", + ServiceKind.TIME.value: "Time service.", + ServiceKind.HEAT.value: "Heat service.", + ServiceKind.REFUSE.value: "Refuse (waster) service.", + ServiceKind.SEWERAGE.value: "Sewerage service.", + ServiceKind.RATES.value: "Rates (e.g. tax, charge, toll, duty, tariff, etc.) service.", + ServiceKind.TV_LICENSE.value: "TV license service.", + ServiceKind.INTERNET.value: "Internet service.", + ServiceKind.MISSING.value: "Missing service kind value.", +} + + +class QualityOfReading(Enum): + """ + Enum representing the quality of a reading. + Equivalent to the original QualityOfReadingValue class. + """ + + VALIDATED = 0 + HUMAN_APPROVED = 7 + MACHINE_COMPUTED = 8 + INTERPOLATED = 9 + FAILED_CHECKS = 10 + CALCULATED = 11 + FORECASTED = 12 + MIXED_QUALITY = 13 + UNVALIDATED = 14 + WEATHER_ADJUSTED = 15 + OTHER = 16 + APPROVED_EDITED = 17 + FAILED_BUT_ACTUAL = 18 + BILLING_ACCEPTABLE = 19 + MISSING = -1 + + +QUALITY_OF_READING_DESCRIPTIONS = { + QualityOfReading.VALIDATED.value: "data that has gone through all required validation checks " + "and either passed them all or has been verified", + QualityOfReading.HUMAN_APPROVED.value: "Replaced or approved by a human", + QualityOfReading.MACHINE_COMPUTED.value: "data value was replaced by a machine computed value based on " + "analysis of historical data using the same type of measurement.", + QualityOfReading.INTERPOLATED.value: "data value was computed using linear " + "interpolation based on the readings before and after it", + QualityOfReading.FAILED_CHECKS.value: "data that has failed one or more checks", + QualityOfReading.CALCULATED.value: "data that has been calculated " + "(using logic or mathematical operations)", + QualityOfReading.FORECASTED.value: "data that has been calculated " + "as a projection or forecast of future readings", + QualityOfReading.MIXED_QUALITY.value: "indicates that the quality of " + "this reading has mixed characteristics", + QualityOfReading.UNVALIDATED.value: "data that has not gone through the validation", + QualityOfReading.WEATHER_ADJUSTED.value: "the values have been adjusted " "to account for weather", + QualityOfReading.OTHER.value: "specifies that a characteristic applies other " "than those defined", + QualityOfReading.APPROVED_EDITED.value: "data that has been validated and " + "possibly edited and/or estimated in accordance with approved procedures", + QualityOfReading.FAILED_BUT_ACTUAL.value: "data that failed at least one of " + "the required validation checks but was determined to represent actual usage", + QualityOfReading.BILLING_ACCEPTABLE.value: "data that is valid and acceptable " "for billing purposes", + QualityOfReading.MISSING.value: "Missing quality of reading value.", +} + + +class UnitSymbol(Enum): + """ + Enum representing the unit symbol values. + Equivalent to the original UnitSymbol class. + """ + + VA = 61 + W = 38 + VAR = 63 + VAH = 71 + WH = 72 + VARH = 73 + V = 29 + OHM = 30 + A = 5 + FARAD = 25 + HENRY = 28 + DEG_C = 23 + S = 27 + MIN = 159 + H = 160 + DEG = 9 + RAD = 10 + J = 31 + N = 32 + SIEMENS = 53 + NONE = 0 + HZ = 33 + G = 3 + PA = 39 + M = 2 + M2 = 41 + M3 = 42 + A2 = 69 + A2H = 105 + A2S = 70 + AH = 106 + A_PER_A = 152 + A_PER_M = 103 + AS = 68 + B_SPL = 79 + BM = 113 + BQ = 22 + BTU = 132 + BTU_PER_H = 133 + CD = 8 + CHAR = 76 + HZ_PER_S = 75 + CODE = 114 + COS_PHI = 65 + COUNT = 111 + FT3 = 119 + FT3_COMPENSATED = 120 + FT3_COMPENSATED_PER_H = 123 + GM2 = 78 + G_PER_G = 144 + GY = 21 + HZ_PER_HZ = 150 + CHAR_PER_S = 77 + IMPERIAL_GAL = 130 + IMPERIAL_GAL_PER_H = 131 + J_PER_K = 51 + J_PER_KG = 165 + K = 6 + KAT = 158 + KG_M = 47 + G_PER_M3 = 48 + L = 134 + L_COMPENSATED = 157 + L_COMPENSATED_PER_H = 138 + L_PER_H = 137 + L_PER_L = 143 + L_PER_S = 82 + L_UNCOMPENSATED = 156 + L_UNCOMPENSATED_PER_H = 139 + LM = 35 + LX = 34 + M2_PER_S = 49 + M3_COMPENSATED = 167 + M3_COMPENSATED_PER_H = 126 + M3_PER_H = 125 + M3_PER_S = 45 + M3_UNCOMPENSATED = 166 + M3_UNCOMPENSATED_PER_H = 127 + ME_CODE = 118 + MOL = 7 + MOL_PER_KG = 147 + MOL_PER_M3 = 145 + MOL_PER_MOL = 146 + CURRENCY = 80 + M_PER_M = 148 + M_PER_M3 = 46 + M_PER_S = 43 + M_PER_S2 = 44 + OHM_M = 102 + PA_A = 155 + PA_G = 140 + PSI_A = 141 + PSI_G = 142 + Q = 100 + Q45 = 161 + Q45H = 163 + Q60 = 162 + Q60H = 164 + QH = 101 + RAD_PER_S = 54 + REV = 154 + REV_PER_S = 4 + S_PER_S = 149 + SR = 11 + STATUS = 109 + SV = 24 + T = 37 + THERM = 169 + TIMESTAMP = 108 + US_GAL = 128 + US_GAL_PER_H = 129 + V2 = 67 + V2H = 104 + VAH_PER_REV = 117 + VARH_PER_REV = 116 + V_PER_HZ = 74 + V_PER_V = 151 + VS = 66 + WB = 36 + WH_PER_M3 = 107 + WH_PER_REV = 115 + W_PER_M_K = 50 + W_PER_S = 81 + W_PER_VA = 153 + W_PER_W = 168 + MISSING = -1 + + +UNIT_SYMBOL_DESCRIPTIONS = { + UnitSymbol.VA.value: "Apparent power, Volt Ampere (See also real power and " "reactive power.), VA", + UnitSymbol.W.value: "Real power, Watt. By definition, one Watt equals one " + "Joule per second. Electrical power may have real and reactive " + "components. The real portion of electrical power (I²R) or VIcos?, is " + "expressed in Watts. (See also apparent power and reactive power.), W", + UnitSymbol.VAR.value: "Reactive power, Volt Ampere reactive. The " + '"reactive" or "imaginary" component of electrical power (VISin?). ' + "(See also real power and apparent power)., VAr", + UnitSymbol.VAH.value: "Apparent energy, Volt Ampere hours, VAh", + UnitSymbol.WH.value: "Real energy, Watt hours, Wh", + UnitSymbol.VARH.value: "Reactive energy, Volt Ampere reactive hours, VArh", + UnitSymbol.V.value: "Electric potential, Volt (W/A), V", + UnitSymbol.OHM.value: "Electric resistance, Ohm (V/A), O", + UnitSymbol.A.value: "Current, ampere, A", + UnitSymbol.FARAD.value: "Electric capacitance, Farad (C/V), °C", + UnitSymbol.HENRY.value: "Electric inductance, Henry (Wb/A), H", + UnitSymbol.DEG_C.value: "Relative temperature in degrees Celsius. In the SI " + "unit system the symbol is ºC. Electric charge is measured in coulomb " + "that has the unit symbol C. To distinguish degree Celsius from " + "coulomb the symbol used in the UML is degC. Reason for not using ºC " + "is the special character º is difficult to manage in software.", + UnitSymbol.S.value: "Time, seconds, s", + UnitSymbol.MIN.value: "Time, minute = s * 60, min", + UnitSymbol.H.value: "Time, hour = minute * 60, h", + UnitSymbol.DEG.value: "Plane angle, degrees, deg", + UnitSymbol.RAD.value: "Plane angle, Radian (m/m), rad", + UnitSymbol.J.value: "Energy joule, (N·m = C·V = W·s), J", + UnitSymbol.N.value: "Force newton, (kg m/s²), N", + UnitSymbol.SIEMENS.value: "Electric conductance, Siemens (A / V = 1 / O), S", + UnitSymbol.NONE.value: "N/A, None", + UnitSymbol.HZ.value: "Frequency hertz, (1/s), Hz", + UnitSymbol.G.value: "Mass in gram, g", + UnitSymbol.PA.value: "Pressure, Pascal (N/m²)(Note: the absolute or " + "relative measurement of pressure is implied with this entry. See " + "below for more explicit forms.), Pa", + UnitSymbol.M.value: "Length, meter, m", + UnitSymbol.M2.value: "Area, square meter, m²", + UnitSymbol.M3.value: "Volume, cubic meter, m³", + UnitSymbol.A2.value: "Amps squared, amp squared, A2", + UnitSymbol.A2H.value: "ampere-squared, Ampere-squared hour, A²h", + UnitSymbol.A2S.value: "Amps squared time, square amp second, A²s", + UnitSymbol.AH.value: "Ampere-hours, Ampere-hours, Ah", + UnitSymbol.A_PER_A.value: "Current, Ratio of Amperages, A/A", + UnitSymbol.A_PER_M.value: "A/m, magnetic field strength, Ampere per metre, A/m", + UnitSymbol.AS.value: "Amp seconds, amp seconds, As", + UnitSymbol.B_SPL.value: "Sound pressure level, Bel, acoustic, Combine with " + '\'multiplier prefix "d" to form decibels of Sound Pressure Level db' + "(SPL), B (SPL)", + UnitSymbol.BM.value: "Signal Strength, Bel-mW, normalized to 1mW. Note: to " + 'form "dBm" combine "Bm" with multiplier "d". Bm', + UnitSymbol.BQ.value: "Radioactivity, Becquerel (1/s), Bq", + UnitSymbol.BTU.value: "Energy, British Thermal Units, BTU", + UnitSymbol.BTU_PER_H.value: "Power, BTU per hour, BTU/h", + UnitSymbol.CD.value: "Luminous intensity, candela, cd", + UnitSymbol.CHAR.value: "Number of characters, characters, char", + UnitSymbol.HZ_PER_S.value: "Rate of change of frequency, hertz per second, Hz/s", + UnitSymbol.CODE.value: "Application Value, encoded value, code", + UnitSymbol.COS_PHI.value: "Power factor, Dimensionless, cos?", + UnitSymbol.COUNT.value: "Amount of substance, counter value, count", + UnitSymbol.FT3.value: "Volume, cubic feet, ft³", + UnitSymbol.FT3_COMPENSATED.value: "Volume, cubic feet, ft³(compensated)", + UnitSymbol.FT3_COMPENSATED_PER_H.value: "Volumetric flow rate, " + "compensated cubic feet per hour, ft³(compensated)/h", + UnitSymbol.GM2.value: "Turbine inertia, gram·meter2 (Combine with " + 'multiplier prefix "k" to form kg·m2.), gm²', + UnitSymbol.G_PER_G.value: "Concentration, The ratio of the mass of a " + "solute divided by the mass of the solution., g/g", + UnitSymbol.GY.value: "Absorbed dose, Gray (J/kg), GY", + UnitSymbol.HZ_PER_HZ.value: "Frequency, Rate of frequency change, Hz/Hz", + UnitSymbol.CHAR_PER_S.value: "Data rate, characters per second, char/s", + UnitSymbol.IMPERIAL_GAL.value: "Volume, imperial gallons, ImperialGal", + UnitSymbol.IMPERIAL_GAL_PER_H.value: "Volumetric flow rate, Imperial " "gallons per hour, ImperialGal/h", + UnitSymbol.J_PER_K.value: "Heat capacity, Joule/Kelvin, J/K", + UnitSymbol.J_PER_KG.value: "Specific energy, Joules / kg, J/kg", + UnitSymbol.K.value: "Temperature, Kelvin, K", + UnitSymbol.KAT.value: "Catalytic activity, katal = mol / s, kat", + UnitSymbol.KG_M.value: "Moment of mass ,kilogram meter (kg·m), M", + UnitSymbol.G_PER_M3.value: "Density, gram/cubic meter (combine with prefix " + 'multiplier "k" to form kg/ m³), g/m³', + UnitSymbol.L.value: "Volume, litre = dm3 = m3/1000., L", + UnitSymbol.L_COMPENSATED.value: "Volume, litre, with the value " + "compensated for weather effects, L(compensated)", + UnitSymbol.L_COMPENSATED_PER_H.value: "Volumetric flow rate, litres " + "(compensated) per hour, L(compensated)/h", + UnitSymbol.L_PER_H.value: "Volumetric flow rate, litres per hour, L/h", + UnitSymbol.L_PER_L.value: "Concentration, The ratio of the volume of a " + "solute divided by the volume of the solution., L/L", + UnitSymbol.L_PER_S.value: "Volumetric flow rate, Volumetric flow rate, L/s", + UnitSymbol.L_UNCOMPENSATED.value: "Volume, litre, with the value " + "uncompensated for weather effects., L(uncompensated)", + UnitSymbol.L_UNCOMPENSATED_PER_H.value: "Volumetric flow rate, litres " + "(uncompensated) per hour, L(uncompensated)/h", + UnitSymbol.LM.value: "Luminous flux, lumen (cd sr), Lm", + UnitSymbol.LX.value: "Illuminance lux, (lm/m²), L(uncompensated)/h", + UnitSymbol.M2_PER_S.value: "Viscosity, meter squared / second, m²/s", + UnitSymbol.M3_COMPENSATED.value: "Volume, cubic meter, with the value " + "compensated for weather effects., m3(compensated)", + UnitSymbol.M3_COMPENSATED_PER_H.value: "Volumetric flow rate, " + "compensated cubic meters per hour, ³(compensated)/h", + UnitSymbol.M3_PER_H.value: "Volumetric flow rate, cubic meters per hour, m³/h", + UnitSymbol.M3_PER_S.value: "m3PerSec, cubic meters per second, m³/s", + UnitSymbol.M3_UNCOMPENSATED.value: "m3uncompensated, cubic meter, with " + "the value uncompensated for weather effects., m3(uncompensated)", + UnitSymbol.M3_UNCOMPENSATED_PER_H.value: "Volumetric flow rate, " + "uncompensated cubic meters per hour, m³(uncompensated)/h", + UnitSymbol.ME_CODE.value: "EndDeviceEvent, value to be interpreted as a " "EndDeviceEventCode, meCode", + UnitSymbol.MOL.value: "Amount of substance, mole, mol", + UnitSymbol.MOL_PER_KG.value: "Concentration, Molality, the amount of " + "solute in moles and the amount of solvent in kilograms., mol/kg", + UnitSymbol.MOL_PER_M3.value: "Concentration, The amount of substance " + "concentration, (c), the amount of solute in moles divided by the " + "volume of solution in m³., mol/m³", + UnitSymbol.MOL_PER_MOL.value: "Concentration, Molar fraction (x), the " + "ratio of the molar amount of a solute divided by the molar amount of " + "the solution., mol/mol", + UnitSymbol.CURRENCY.value: "Monetary unit, Generic money (Note: Specific " + "monetary units are identified by the currency class)., ¤", + UnitSymbol.M_PER_M.value: "Length, Ratio of length, m/m", + UnitSymbol.M_PER_M3.value: "Fuel efficiency, meters per cubic meter, m/m³", + UnitSymbol.M_PER_S.value: "Velocity, meters per second, m/s", + UnitSymbol.M_PER_S2.value: "Acceleration, meters per second squared, m/s²", + UnitSymbol.OHM_M.value: "Resistivity, ohm meter, Ω·m", + UnitSymbol.PA_A.value: "Pressure, Pascal absolute, PaA", + UnitSymbol.PA_G.value: "Pressure, Pascal gauge, PaG", + UnitSymbol.PSI_A.value: "Pressure, Pounds per square inch absolute, psiA", + UnitSymbol.PSI_G.value: "Pressure, Pounds per square inch gauge, psiG", + UnitSymbol.Q.value: "Quantity power, Q, Q", + UnitSymbol.Q45.value: "Quantity power, Q measured at 45°, Q45", + UnitSymbol.Q45H.value: "Quantity energy, Q measured at 45°, Q45h", + UnitSymbol.Q60.value: "Quantity power, Q measured at 60°, Q60", + UnitSymbol.Q60H.value: "Quantity energy, Qh measured at 60°, Q60h", + UnitSymbol.QH.value: "Quantity energy, Qh, Qh", + UnitSymbol.RAD_PER_S.value: "Angular velocity, radians per second, rad/s", + UnitSymbol.REV.value: "Amount of rotation, Revolutions, rev", + UnitSymbol.REV_PER_S.value: "Rotational speed, Revolutions per second, rev/s", + UnitSymbol.S_PER_S.value: "Time, Ratio of time (can be combined with a " + "multiplier prefix to show rates such as a clock drift rate, e.g. " + '"µs/s"), s/s', + UnitSymbol.SR.value: "Solid angle, Steradian, sr", + UnitSymbol.STATUS.value: 'State, "1" = "true", "live", "on", "high", ' + '"set"; "0" = "false", "dead", "off", "low", "cleared". Note: A ' + "Boolean value is preferred but other values may be supported, status", + UnitSymbol.SV.value: "Dose equivalent, Sievert, Sv", + UnitSymbol.T.value: "Magnetic flux density, Tesla, T", + UnitSymbol.THERM.value: "Energy, Therm, therm", + UnitSymbol.TIMESTAMP.value: "Timestamp, time and date per ISO 8601 " "format, timeStamp", + UnitSymbol.US_GAL.value: "Volume, US gallons, USGal", + UnitSymbol.US_GAL_PER_H.value: "Volumetric flow rate, US gallons per " "hour, USGal/h", + UnitSymbol.V2.value: "Volts squared, Volt squared, V²", + UnitSymbol.V2H.value: "Volt-squared hour, Volt-squared-hours, V²h", + UnitSymbol.VAH_PER_REV.value: "Apparent energy metering constant, VAh per " "revolution, VAh/rev", + UnitSymbol.VARH_PER_REV.value: "Reactive energy metering constant, VArh " "per revolution, VArh/rev", + UnitSymbol.V_PER_HZ.value: "Magnetic flux, Volts per Hertz, V/Hz", + UnitSymbol.V_PER_V.value: "Voltage, Ratio of voltages (e.g. mV/V), V/V", + UnitSymbol.VS.value: "Volt seconds, Volt seconds, Vs", + UnitSymbol.WB.value: "Magnetic flux, Weber, Wb", + UnitSymbol.WH_PER_M3.value: "Energy per volume, Watt-hours per cubic " "meter, Wh/m³", + UnitSymbol.WH_PER_REV.value: "Active energy metering constant, Wh per " "revolution, Wh/rev", + UnitSymbol.W_PER_M_K.value: "Thermal conductivity, Watt per meter Kelvin, " "W/(m·K)", + UnitSymbol.W_PER_S.value: "Ramp rate, Watts per second, W/s", + UnitSymbol.W_PER_VA.value: "Power Factor, PF, W/VA", + UnitSymbol.W_PER_W.value: "Signal Strength, Ratio of power, W/W", + UnitSymbol.MISSING.value: "Unknown unit value, Unknown, unknown", +} diff --git a/src/greenbutton_objects/orig_objects/objects.py b/src/greenbutton_objects/orig_objects/objects.py new file mode 100644 index 0000000..3e7d1c0 --- /dev/null +++ b/src/greenbutton_objects/orig_objects/objects.py @@ -0,0 +1,172 @@ +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from decimal import Decimal +from typing import List, Optional, Tuple + +import greenbutton_objects.data.espi as espi +from greenbutton_objects.orig_objects import ServiceKind +from greenbutton_objects.orig_objects.enums import UNIT_SYMBOL_DESCRIPTIONS, QualityOfReading, UnitSymbol +from greenbutton_objects.util import get_value + + +@dataclass +class DateTimeInterval: + start: datetime + duration: timedelta + + +@dataclass(order=True) +class IntervalReading: + """Specific value measured by a meter or other asset. + + Each Reading is associated with a specific ReadingType. + + :ivar cost: [correction] Specifies a cost associated with this reading, + in hundred-thousandths of + the currency specified in the ReadingType for this reading. + (e.g., 840 = USD, US dollar) + NaN means there was no cost received from the feed + :ivar quality_of_reading: One or more quality of reading values for the + current interval reading. + :ivar time_period: The date time and duration of a reading. If not specified, + readings for each “intervalLength” in ReadingType are present. + :ivar raw_value: [correction] Value in units specified by ReadingType (not scaled) + :ivar consumption_tier: [extension] Code for consumption tier associated with reading. + :ivar tou: [extension] Code for the TOU type of reading. + :ivar cpp: [extension] Critical peak period (CPP) bucket the reading + value is attributed to. Value + 0 means not applicable. Even though CPP is usually considered a specialized + form of time of use 'tou', this attribute is defined explicitly for flexibility. + :ivar parent: Reference to the parent interval block + """ + + time_period: DateTimeInterval + raw_value: float + + consumption_tier: Optional[int] = None + tou: Optional[int] = None + cpp: int = 0 + + parent: Optional["IntervalBlock"] = None + cost: float = float("NaN") + quality_of_reading: QualityOfReading = QualityOfReading.MISSING + reading_type: Optional[espi.ReadingType] = None + + __cached_value: Optional[float] = None + + @property + def start(self) -> datetime: + if self.time_period is None: + return datetime.utcfromtimestamp(0) + # The type checking is ignored for the lines below + # because these conditions should never happen + # unfortunately some providers produce feeds that are not compatible + # with the defined XSD schema + if type(self.time_period.start) is str: # type: ignore + time_start = datetime.utcfromtimestamp(int(Decimal(self.time_period.start))) # type: ignore + elif type(self.time_period.start) is int: # type: ignore + time_start = datetime.utcfromtimestamp(self.time_period.start) # type: ignore + else: + time_start = self.time_period.start + return time_start + + @property + def value(self) -> float: + if ( + self.__cached_value is None + and self.reading_type is not None + and self.parent is not None + and self.parent.multiplier is not None + ): + self.__cached_value = self.raw_value * self.parent.multiplier + else: + raise ValueError("Cannot auto scale raw_value. Not enough data") + return self.__cached_value + + +@dataclass +class IntervalBlock: + uri: str + interval: DateTimeInterval + readings: List[IntervalReading] = field(default_factory=list) + + multiplier: Optional[float] = None + reading_power_of_ten: Optional[float] = None + + def compute_multiplier(self, reading_type: espi.ReadingType) -> None: + reading_power_ten = get_value( + reading_type.power_of_ten_multiplier, + src_type=espi.UnitMultiplierKindValue, + dest_type=espi.UnitMultiplierKindValue, + missing_val=espi.UnitMultiplierKindValue.VALUE_0, + ) + self.reading_power_of_ten = reading_power_ten.value + self.multiplier = 10.0**self.reading_power_of_ten + + +@dataclass +class MeterReading: + title: str + uri: str + reading_type: espi.ReadingType + readings: Tuple[IntervalReading, ...] = field(default_factory=tuple) + intervalBlock: Tuple[IntervalBlock, ...] = field(default_factory=tuple) + + __uom_symbol = None + __uom_description = None + + def patch(self) -> None: + for r in self.readings: + r.reading_type = self.reading_type + + @property + def uom_symbol(self) -> str: + if self.__uom_symbol is None: + self.__set_unit_of_measure() + return self.__uom_symbol # type: ignore + + @property + def uom_description(self) -> str: + if self.__uom_description is None: + self.__set_unit_of_measure() + return self.__uom_description # type: ignore + + def __set_unit_of_measure(self) -> None: + uom_value = get_value( + self.reading_type.uom, + src_type=espi.UnitSymbolKindValue, + dest_type=UnitSymbol, + missing_val=UnitSymbol.MISSING, + ) + uom_strs = UNIT_SYMBOL_DESCRIPTIONS[uom_value.value].split(",") + self.__uom_description = uom_strs[0] + self.__uom_symbol = uom_strs[-1] + + +@dataclass +class UsagePoint: + """ + Logical point on a network at which consumption or production is either + physically measured (e.g., metered) or estimated (e.g., unmetered streetlights). + + :ivar title: The title of the usage point. + :ivar uri: The URI that identifies this usage point in the atom feed + + :ivar meter_readings: Collected meter readings. + :ivar service_kind: Type of service (Electricity, Natural Gas, etc.) + :ivar electric_power_usage_summary: (deprecated) summary of electricity usage + :ivar local_time_parameters: Timestamp adjustment rules + :ivar status: Specifies the current status of this usage + point. Valid values include: 0 = off 1 =on, -1 = unknown + + + """ + + title: str + uri: str + + meter_readings: Tuple[MeterReading, ...] + service_kind: ServiceKind + electric_power_usage_summary: Optional[espi.ElectricPowerUsageSummary] = None + local_time_parameters: Optional[espi.TimeConfiguration] = None + status: int = -1 diff --git a/src/greenbutton_objects/parse.py b/src/greenbutton_objects/parse.py index 2c9af0a..2c655ac 100755 --- a/src/greenbutton_objects/parse.py +++ b/src/greenbutton_objects/parse.py @@ -1,69 +1,31 @@ #!/usr/bin/python -import sys -import xml.etree.ElementTree as ET +from xsdata.formats.dataclass.context import XmlContext +from xsdata.formats.dataclass.parsers import XmlParser +from xsdata.formats.dataclass.parsers.config import ParserConfig -from greenbutton_objects import resources, utils +import greenbutton_objects.data.atom as atom +from greenbutton_objects.atom.href_tree import HRefForest +from greenbutton_objects.atom.object_tree import ObjectForest +from greenbutton_objects.feed.feed import ObjectFeed -def parse_feed(filename): - tree = ET.parse(filename) +def parse_feed(filename: str) -> ObjectFeed: + data = parse_xml(filename) - usagePoints = [] - for entry in tree.getroot().findall("atom:entry/atom:content/espi:UsagePoint/../..", utils.ns): - up = resources.UsagePoint(entry) - usagePoints.append(up) + href_forest = HRefForest().build(data) + entry_forest = ObjectForest().build(href_forest) + object_feed = ObjectFeed().build(entry_forest) - meterReadings = [] - for entry in tree.getroot().findall("atom:entry/atom:content/espi:MeterReading/../..", utils.ns): - mr = resources.MeterReading(entry, usagePoints=usagePoints) - meterReadings.append(mr) + return object_feed - readingTypes = [] - for entry in tree.getroot().findall("atom:entry/atom:content/espi:ReadingType/../..", utils.ns): - rt = resources.ReadingType(entry, meterReadings=meterReadings) - readingTypes.append(rt) - intervalBlocks = [] - for entry in tree.getroot().findall("atom:entry/atom:content/espi:IntervalBlock/../..", utils.ns): - ib = resources.IntervalBlock(entry, meterReadings=meterReadings) - intervalBlocks.append(ib) +def parse_xml(filename: str) -> atom.Feed: + data = get_xml_parser().parse(filename, clazz=atom.Feed) + return data - return usagePoints - -def parse_feed_representation(usage_points) -> str: - """ - Return a string representation of the test_parse result. - - The representation includes the Usage Points, Meter Readings, and - Interval Readings. - """ - result = [] - for up in usage_points: - result.append("UsagePoint (%s) %s %s:" % (up.title, up.serviceCategory.name, up.status)) - for mr in up.meterReadings: - result.append(" Meter Reading (%s) %s:" % (mr.title, mr.readingType.uom.name)) - result.append("\n") - for ir in mr.intervalReadings: - result.append( - " %s, %s: %s %s" - % ( - ir.timePeriod.start, - ir.timePeriod.duration, - ir.value, - ir.value_symbol, - ) - ) - if ir.cost is not None: - result.append("(%s%s)" % (ir.cost_symbol, ir.cost)) - if len(ir.readingQualities) > 0: - result.append("[%s]" % ", ".join([rq.quality.name for rq in ir.readingQualities])) - result.append("\n\n") - return "".join(result) - - -if __name__ == "__main__": - usage_points = parse_feed(sys.argv[1]) - representation = parse_feed_representation(usage_points) - print(representation) +def get_xml_parser() -> XmlParser: + config = ParserConfig(fail_on_unknown_properties=False) + context = XmlContext() + return XmlParser(context=context, config=config) diff --git a/src/greenbutton_objects/resources.py b/src/greenbutton_objects/resources.py deleted file mode 100644 index 0f16b02..0000000 --- a/src/greenbutton_objects/resources.py +++ /dev/null @@ -1,143 +0,0 @@ -import bisect -import functools - -from greenbutton_objects import enums, objects, utils - - -class Resource: - def __init__(self, entry): - self.link_self = utils.getLink(entry, "self") - self.link_up = utils.getLink(entry, "up") - self.link_related = utils.getLink(entry, "related", True) - self.title = utils.getEntity(entry, "atom:title", lambda e: e.text) - - def __repr__(self): - return "<%s (%s)>" % (self.__class__.__name__, self.title or self.link_self) - - def isParentOf(self, other): - return other.link_self in self.link_related or other.link_up in self.link_related - - -class UsagePoint(Resource): - def __init__(self, entry, meterReadings=[]): - super(UsagePoint, self).__init__(entry) - obj = entry.find("./atom:content/espi:UsagePoint", utils.ns) - self.roleFlags = utils.getEntity(obj, "espi:roleFlags", lambda e: int(e.text, 16)) - self.status = utils.getEntity(obj, "espi:status", lambda e: int(e.text)) - self.serviceCategory = utils.getEntity( - obj, - "./espi:ServiceCategory/espi:kind", - lambda e: enums.ServiceKind(int(e.text)), - ) - - self.meterReadings = set() - for mr in meterReadings: - if self.isParentOf(mr): - self.addMeterReading(mr) - - def addMeterReading(self, meterReading): - assert self.isParentOf(meterReading) - self.meterReadings.add(meterReading) - meterReading.usagePoint = self - - -class MeterReading(Resource): - def __init__(self, entry, usagePoints=[], readingTypes=[], intervalBlocks=[]): - super(MeterReading, self).__init__(entry) - - self.usagePoint = None - self.readingType = None - self.intervalBlocks = [] - for up in usagePoints: - if up.isParentOf(self): - up.addMeterReading(self) - for rt in readingTypes: - if self.isParentOf(rt): - self.setReadingType(rt) - for ib in intervalBlocks: - if self.isParentOf(ib): - self.addIntervalBlock(ib) - - @property - def intervalReadings(self): - for ib in self.intervalBlocks: - for ir in ib.intervalReadings: - yield ir - - def setReadingType(self, readingType): - assert self.isParentOf(readingType) - assert self.readingType is None or self.readingType.link_self == readingType.link_self - self.readingType = readingType - readingType.meterReading = self - - def addIntervalBlock(self, intervalBlock): - assert self.isParentOf(intervalBlock) - bisect.insort(self.intervalBlocks, intervalBlock) - intervalBlock.meterReading = self - - -class ReadingType(Resource): - def __init__(self, entry, meterReadings=[]): - super(ReadingType, self).__init__(entry) - self.meterReading = None - - obj = entry.find("./atom:content/espi:ReadingType", utils.ns) - self.accumulationBehaviour = utils.getEntity( - obj, - "espi:accumulationBehaviour", - lambda e: enums.AccumulationBehaviourType(int(e.text)), - ) - self.commodity = utils.getEntity(obj, "espi:commodity", lambda e: enums.CommodityType(int(e.text))) - self.consumptionTier = utils.getEntity( - obj, - "espi:consumptionTier", - lambda e: enums.ConsumptionTierType(int(e.text)), - ) - self.currency = utils.getEntity(obj, "espi:currency", lambda e: enums.CurrencyCode(int(e.text))) - self.dataQualifier = utils.getEntity( - obj, "espi:dataQualifier", lambda e: enums.DataQualifierType(int(e.text)) - ) - self.defaultQuality = utils.getEntity( - obj, "espi:defaultQuality", lambda e: enums.QualityOfReading(int(e.text)) - ) - self.flowDirection = utils.getEntity( - obj, "espi:flowDirection", lambda e: enums.FlowDirectionType(int(e.text)) - ) - self.intervalLength = utils.getEntity(obj, "espi:intervalLength", lambda e: int(e.text)) - self.kind = utils.getEntity(obj, "espi:kind", lambda e: enums.KindType(int(e.text))) - self.phase = utils.getEntity(obj, "espi:phase", lambda e: enums.PhaseCode(int(e.text))) - self.powerOfTenMultiplier = utils.getEntity(obj, "espi:powerOfTenMultiplier", lambda e: int(e.text)) - self.timeAttribute = utils.getEntity( - obj, "espi:timeAttribute", lambda e: enums.TimeAttributeType(int(e.text)) - ) - self.tou = utils.getEntity(obj, "espi:tou", lambda e: enums.TOUType(int(e.text))) - self.uom = utils.getEntity(obj, "espi:uom", lambda e: enums.UomType(int(e.text))) - - for mr in meterReadings: - if mr.isParentOf(self): - mr.setReadingType(self) - - -@functools.total_ordering -class IntervalBlock(Resource): - def __init__(self, entry, meterReadings=[]): - super(IntervalBlock, self).__init__(entry) - self.meterReading = None - - obj = entry.find("./atom:content/espi:IntervalBlock", utils.ns) - self.interval = utils.getEntity(obj, "espi:interval", lambda e: objects.DateTimeInterval(e)) - self.intervalReadings = sorted([ - objects.IntervalReading(ir, self) for ir in obj.findall("espi:IntervalReading", utils.ns) - ]) - - for mr in meterReadings: - if mr.isParentOf(self): - mr.addIntervalBlock(self) - - def __eq__(self, other): - if not isinstance(other, IntervalBlock): - return False - return self.link_self == other.link_self - - def __lt__(self, other): - return self.interval < other.interval diff --git a/src/greenbutton_objects/util.py b/src/greenbutton_objects/util.py new file mode 100644 index 0000000..a5af2e5 --- /dev/null +++ b/src/greenbutton_objects/util.py @@ -0,0 +1,23 @@ +from typing import Iterable, Type, TypeVar, Union + +T = TypeVar("T") +S = TypeVar("S") + + +def get_first(iterable: Iterable[T]) -> Union[T, None]: + for item in iterable: + return item + return None + + +def get_value(source: S, missing_val: T, dest_type: Type[T], src_type: Type[S]) -> T: + # TODO: See if we can get a generic type for dest_type and src_type + # generic type is something that has .value attribute + # also we should get a generic type of dest value which is something that can + # take and int or src_type as an argument for creation + value: T = dest_type(missing_val.value) # type: ignore + if isinstance(source, int): + value = dest_type(source) # type: ignore + elif isinstance(source, src_type): + value = dest_type(source.value) # type: ignore + return value diff --git a/src/greenbutton_objects/utils.py b/src/greenbutton_objects/utils.py deleted file mode 100644 index 0a15a8d..0000000 --- a/src/greenbutton_objects/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -ns = {"atom": "http://www.w3.org/2005/Atom", "espi": "http://naesb.org/espi"} - - -def getEntity(source, target, accessor=None, multiple=False): - """ - Extracts the named entity from the source XML tree. - - `accessor` is a function of one argument; if provided and the - target entity is found, the target will be passed into `accessor` - and its result will be returned. - If `multiple` is true, the result will be all entities that match - (i.e. the function will use `findall` instead of `find`). - """ - if multiple: - es = source.findall(target, ns) - if accessor: - return [accessor(e) for e in es] - else: - return es - else: - e = source.find(target, ns) - if e is not None and accessor is not None: - return accessor(e) - else: - return e - - -def getLink(source, relation, multiple=False): - """ - Shorthand for pulling a link with the given "rel" attribute from - the source. - """ - return getEntity( - source, - './atom:link[@rel="%s"]' % relation, - lambda e: e.attrib["href"], - multiple, - ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9f5e38a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +import pathlib + +import pytest + +_ROOT_DIR = pathlib.Path(__file__).parent + + +@pytest.fixture() +def data_dir(): + return _ROOT_DIR / "data" diff --git a/tests/helpers/feed_repr.py b/tests/helpers/feed_repr.py new file mode 100644 index 0000000..f7b3271 --- /dev/null +++ b/tests/helpers/feed_repr.py @@ -0,0 +1,44 @@ +from datetime import timedelta +from math import isnan + +from greenbutton_objects import orig_objects as ob +from greenbutton_objects.feed.feed import ObjectFeed + + +def parse_feed_representation(feed: ObjectFeed) -> str: + """ + Return a string representation of the test_parse result. + + The representation includes the Usage Points, Meter Readings, and + Interval Readings. + """ + result = [] + + # Get all usage point containers and elements + for up in feed.usage_points: + result.append("UsagePoint (%s) Service: %s (%s) " % (up.title, up.service_kind.name, up.status)) + + # Get meter readings containers and elements + for mr in up.meter_readings: + result.append("Meter Reading (%s) %s:" % (mr.title, mr.uom_description)) + result.append("\n") + + for reading in mr.readings: + result.append( + " %s, %s: %g%s" + % ( + reading.start.strftime("%Y-%m-%d %H:%M:%S+00:00"), + str(timedelta(seconds=reading.time_period.duration)), # type: ignore + reading.value, + mr.uom_symbol, + ) + ) + if not isnan(reading.cost): + result.append("(%s%s)" % ("$", reading.cost / 100000)) + # TODO: Hard-coding $ for now as current code does + # not handle currency correctly + if reading.quality_of_reading != ob.QualityOfReading.MISSING: + result.append(" [%s]" % reading.quality_of_reading.name) + result.append("\n\n") + + return "".join(result) diff --git a/tests/test_digested.py b/tests/test_digested.py new file mode 100644 index 0000000..06ee7a7 --- /dev/null +++ b/tests/test_digested.py @@ -0,0 +1,139 @@ +from math import isnan + +from greenbutton_objects import parse +from greenbutton_objects.data.espi import LocalTimeParameters +from greenbutton_objects.orig_objects import QualityOfReading, ServiceKind +from greenbutton_objects.orig_objects.objects import UsagePoint + +from .helpers.feed_repr import parse_feed_representation + + +def test_electric_containerized(data_dir): + """ + Very quick test that runs only one of of the files from each source + """ + + data_file = data_dir / "abridged" / "electric_containerized.xml" + + usage_points = parse.parse_feed(str(data_file)).usage_points + + assert type(usage_points[0]) is UsagePoint + assert len(usage_points) == 1 + + up = usage_points[0] + assert up.title == "Coastal Multi-Family 12hr" + assert up.service_kind == ServiceKind.ELECTRICITY + assert "Customer/3/UsagePoint/1" in up.uri + + assert up.electric_power_usage_summary.quality_of_reading.value == QualityOfReading.UNVALIDATED.value + + assert up.local_time_parameters is not None + + lcl_time_params: LocalTimeParameters = up.local_time_parameters + assert lcl_time_params.dst_offset == 3600 + assert lcl_time_params.tz_offset == -8 * 60 * 60 + + mr = up.meter_readings[0] + assert mr.title == "Hourly Electricity Consumption" + assert "Point/1/MeterReading/0" in mr.uri + assert len(mr.intervalBlock) == 2 + assert len(mr.readings) == 8 + + assert mr.readings[0].value == 450 + assert isnan(mr.readings[0].cost) + assert mr.readings[0].quality_of_reading == QualityOfReading.MISSING + + iblock = mr.intervalBlock[0] + assert len(iblock.readings) == 4 + + assert mr.readings[0].parent == iblock + + +def test_gas_containerized(data_dir): + """ + Very quick test that runs only one of of the files from each source + """ + + data_file = data_dir / "abridged" / "gas_containerized.xml" + + usage_points = parse.parse_feed(str(data_file)).usage_points + + assert type(usage_points[0]) is UsagePoint + assert len(usage_points) == 1 + + up = usage_points[0] + assert up.title == "101 DOG ST BOBTOWN MA US 12345-9032" + assert up.service_kind == ServiceKind.GAS + assert "User/1111111/UsagePoint/01" in up.uri + + assert up.local_time_parameters is not None + + lcl_time_params: LocalTimeParameters = up.local_time_parameters + assert lcl_time_params.dst_offset == 3600 + assert lcl_time_params.tz_offset == 5 * 60 * 60 + + mr = up.meter_readings[0] + assert mr.title == "" + assert "Point/01/MeterReading/01" in mr.uri + assert len(mr.intervalBlock) == 3 + assert len(mr.readings) == 3 + + # Not this is not normalized + assert mr.readings[0].value == 12.000 + assert mr.readings[0].cost == 2806000 + assert mr.readings[0].quality_of_reading == QualityOfReading.VALIDATED + + iblock = mr.intervalBlock[0] + assert len(iblock.readings) == 1 + + assert mr.readings[0].parent == iblock + + +def test_gas_direct(data_dir): + """ + Very quick test that runs only one of of the files from each source + """ + + data_file = data_dir / "abridged" / "gas_direct.xml" + + usage_points = parse.parse_feed(str(data_file)).usage_points + + assert type(usage_points[0]) is UsagePoint + assert len(usage_points) == 1 + + up = usage_points[0] + assert up.title == "1 MAIN ST, ANYTOWN ME 12345" + assert up.service_kind == ServiceKind.GAS + assert "90/UsagePoint/NET_USAGE" in up.uri + + assert up.local_time_parameters is None + + mr = up.meter_readings[0] + assert mr.title == "" + assert "Point/NET_USAGE/MeterReading/1" in mr.uri + assert len(mr.intervalBlock) == 1 + assert len(mr.readings) == 5 + + # Not this is not normalized + assert mr.readings[0].value == 37.000 + assert mr.readings[0].cost == 5100000 + assert mr.readings[0].quality_of_reading == QualityOfReading.MISSING + + iblock = mr.intervalBlock[0] + assert len(iblock.readings) == 5 + + assert mr.readings[0].parent == iblock + + +def test_gas_direct_pb(data_dir): + """ + Very quick test that runs only one of of the files from each source + """ + + data_file = data_dir / "natural_gas" / "ngma_gas_provider_2024-07-16.xml" + + atom_forest = parse.parse_feed(str(data_file)) + repr = parse_feed_representation(atom_forest) + assert ( + repr.split("\n")[3] == " 2024-07-16 18:26:24+00:00, 30 days, 0:00:00: 15 therm($33.06) [VALIDATED]" + ) diff --git a/tests/test_parse/test_parse.py b/tests/test_parse.py similarity index 78% rename from tests/test_parse/test_parse.py rename to tests/test_parse.py index be75ee4..1e571b7 100644 --- a/tests/test_parse/test_parse.py +++ b/tests/test_parse.py @@ -14,6 +14,8 @@ import pytest from greenbutton_objects import parse +from .helpers.feed_repr import parse_feed_representation + _ROOT_DIR = pathlib.Path(__file__).parent @@ -22,8 +24,8 @@ def _save_representation(test_xml_path: Path, test_output_path: Path) -> None: Parse an XML feed, generate a representation, and save it to a text file. """ - parsed_feed = parse.parse_feed(test_xml_path) - representation = parse.parse_feed_representation(parsed_feed) + parsed_feed = parse.parse_feed(str(test_xml_path)) + representation = parse_feed_representation(parsed_feed) with open(test_output_path, "w") as f: f.write(representation) @@ -51,7 +53,7 @@ def check_file(data_file: Path): Compares the string form of a parsed XML to a saved text file. """ atom_forest = parse.parse_feed(str(data_file)) - parsed_feed_representation = parse.parse_feed_representation(atom_forest) + parsed_feed_representation = parse_feed_representation(atom_forest) expected_result_file_name = ( data_file.parent.parent.parent / "expected_results" @@ -95,15 +97,39 @@ def test_parse_natural_gas_feed(data_file_name): @pytest.mark.parametrize("energy_source", ["electricity", "natural_gas"]) -def test_quick(energy_source): +def test_quick(data_dir, energy_source): """ - Very quick test that runs only one of of the files from each source + Very quick test that runs only one of the files from each source """ - files = (_ROOT_DIR / "data" / energy_source).iterdir() + + files = (data_dir / energy_source).iterdir() + data_file_name = next(files) + check_file(data_file_name) + data_file_name = next(files) + check_file(data_file_name) + + +@pytest.mark.parametrize( + "energy_source", + [ + "abridged", + ], +) +def test_abridged(data_dir, energy_source): + """ + Very quick test that runs only one of the files from each source + """ + + files = (data_dir / energy_source).iterdir() + data_file_name = next(files) + check_file(data_file_name) + data_file_name = next(files) + check_file(data_file_name) data_file_name = next(files) check_file(data_file_name) if __name__ == "__main__": + save_expected_results("abridged") save_expected_results("electricity") save_expected_results("natural_gas")