From 89fdb3eeba3ae69e5b39f0c0cf9b242ce82c2666 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 22 Oct 2024 16:02:39 -0600 Subject: [PATCH 1/7] Improve InvalidNumber comparison for array data --- pyomo/repn/util.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index 32ec99dac0f..dc134cb0718 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -157,19 +157,37 @@ def _op(self, op, *args): return InvalidNumber(self.value, causes) def __eq__(self, other): - return self._cmp(operator.eq, other) + ans = self._cmp(operator.eq, other) + try: + return bool(ans) + except ValueError: + # ValueError can be raised by numpy.ndarray when two arrays + # are returned. In that case, ndarray returns a new ndarray + # of bool values. We will fall back on using `all` to + # reduce it to a single bool. + try: + return all(ans) + except: + pass + raise def __lt__(self, other): - return self._cmp(operator.lt, other) + # Note that as < is ambiguous for arrays, we will attempt to + # cast the result to bool and if it was an array, allow the + # exception to propagate + return bool(self._cmp(operator.lt, other)) def __gt__(self, other): - return self._cmp(operator.gt, other) + # See the comment in __lt__() on the use of bool() + return bool(self._cmp(operator.gt, other)) def __le__(self, other): - return self._cmp(operator.le, other) + # See the comment in __lt__() on the use of bool() + return bool(self._cmp(operator.le, other)) def __ge__(self, other): - return self._cmp(operator.ge, other) + # See the comment in __lt__() on the use of bool() + return bool(self._cmp(operator.ge, other)) def _error(self, msg): causes = list(filter(None, self.causes)) From 57f3873faece13deb884dc449d85764fa9d508cb Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 22 Oct 2024 16:03:38 -0600 Subject: [PATCH 2/7] Always raise exception casting InvalidNumber to str, unless processing an exception --- pyomo/repn/util.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index dc134cb0718..719e59ea585 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -197,14 +197,21 @@ def _error(self, msg): raise InvalidValueError(msg) def __str__(self): - # We will support simple conversion of InvalidNumber to strings - # (for reporting purposes) + # We want attempts to convert InvalidNumber to a string + # representation to raise a InvalidValueError, unless we are in + # the middle of processing an exception. In that case, it is + # very likely that an exception handler is generating an error + # message. We will play nice and return a reasonable string. + if sys.exc_info()[1] is None: + self._error(f'Cannot emit {self._str()} in compiled representation') + else: + return self._str() + + def _str(self): return f'InvalidNumber({self.value!r})' def __repr__(self): - # We want attempts to convert InvalidNumber to a string - # representation to raise a InvalidValueError. - self._error(f'Cannot emit {str(self)} in compiled representation') + return str(self) def __format__(self, format_spec): # FIXME: We want to move to where converting InvalidNumber to From e8b1a6b4ca4613ece2d25aa91530b2df609bf8ba Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 22 Oct 2024 16:15:52 -0600 Subject: [PATCH 3/7] Staandardize printing of InvalidNumber/nan warnings --- pyomo/repn/ampl.py | 12 ++++++++---- pyomo/repn/linear.py | 14 ++++++++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/pyomo/repn/ampl.py b/pyomo/repn/ampl.py index c6056bd9592..5bac9f2e470 100644 --- a/pyomo/repn/ampl.py +++ b/pyomo/repn/ampl.py @@ -170,6 +170,10 @@ def _strip_template_comments(vars_, base_): vars_[k] = '\n'.join(v_lines) +def _inv2str(val): + return f"{val._str() if hasattr(val, '_str') else val}" + + # The "standard" text mode template is the debugging template with the # comments removed class TextNLTemplate(TextNLDebugTemplate): @@ -539,7 +543,7 @@ def handle_product_node(visitor, node, arg1, arg2): _prod = mult * arg2[1] if _prod: deprecation_warning( - f"Encountered {mult}*{str(arg2[1])} in expression tree. " + f"Encountered {mult}*{_inv2str(arg2[1])} in expression tree. " "Mapping the NaN result to 0 for compatibility " "with the nl_v1 writer. In the future, this NaN " "will be preserved/emitted to comply with IEEE-754.", @@ -568,7 +572,7 @@ def handle_product_node(visitor, node, arg1, arg2): _prod = mult * arg2[1] if _prod: deprecation_warning( - f"Encountered {str(mult)}*{arg2[1]} in expression tree. " + f"Encountered {_inv2str(mult)}*{arg2[1]} in expression tree. " "Mapping the NaN result to 0 for compatibility " "with the nl_v1 writer. In the future, this NaN " "will be preserved/emitted to comply with IEEE-754.", @@ -976,7 +980,7 @@ def _before_monomial(visitor, child): arg2 = visitor.fixed_vars[_id] if arg2 != arg2: deprecation_warning( - f"Encountered {arg1}*{arg2} in expression tree. " + f"Encountered {arg1}*{_inv2str(arg2)} in expression tree. " "Mapping the NaN result to 0 for compatibility " "with the nl_v1 writer. In the future, this NaN " "will be preserved/emitted to comply with IEEE-754.", @@ -1016,7 +1020,7 @@ def _before_linear(visitor, child): arg2 = visitor.check_constant(arg2.value, arg2) if arg2 != arg2: deprecation_warning( - f"Encountered {arg1}*{str(arg2.value)} in expression " + f"Encountered {arg1}*{_inv2str(arg2)} in expression " "tree. Mapping the NaN result to 0 for compatibility " "with the nl_v1 writer. In the future, this NaN " "will be preserved/emitted to comply with IEEE-754.", diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 029fe892b62..1d3ed8d3956 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -61,6 +61,10 @@ _GENERAL = ExprType.GENERAL +def _inv2str(val): + return f"{val._str() if hasattr(val, '_str') else val}" + + def _merge_dict(dest_dict, mult, src_dict): if mult == 1: for vid, coef in src_dict.items(): @@ -206,8 +210,10 @@ def _handle_product_constant_constant(visitor, node, arg1, arg2): ans = arg1[1] * arg2[1] if ans != ans: if not arg1[1] or not arg2[1]: + a = _inv2str(arg1[1]) + b = _inv2str(arg2[1]) deprecation_warning( - f"Encountered {str(arg1[1])}*{str(arg2[1])} in expression tree. " + f"Encountered {a}*{b} in expression tree. " "Mapping the NaN result to 0 for compatibility " "with the lp_v1 writer. In the future, this NaN " "will be preserved/emitted to comply with IEEE-754.", @@ -599,7 +605,7 @@ def _before_monomial(visitor, child): arg2 = visitor.check_constant(arg2.value, arg2) if arg2 != arg2: deprecation_warning( - f"Encountered {arg1}*{str(arg2.value)} in expression " + f"Encountered {arg1}*{_inv2str(arg2)} in expression " "tree. Mapping the NaN result to 0 for compatibility " "with the lp_v1 writer. In the future, this NaN " "will be preserved/emitted to comply with IEEE-754.", @@ -633,7 +639,7 @@ def _before_linear(visitor, child): arg2 = visitor.check_constant(arg2.value, arg2) if arg2 != arg2: deprecation_warning( - f"Encountered {arg1}*{str(arg2.value)} in expression " + f"Encountered {arg1}*{_inv2str(arg2)} in expression " "tree. Mapping the NaN result to 0 for compatibility " "with the lp_v1 writer. In the future, this NaN " "will be preserved/emitted to comply with IEEE-754.", @@ -813,7 +819,7 @@ def finalizeResult(self, result): c != c for c in ans.linear.values() ): deprecation_warning( - f"Encountered {str(mult)}*nan in expression tree. " + f"Encountered {mult}*nan in expression tree. " "Mapping the NaN result to 0 for compatibility " "with the lp_v1 writer. In the future, this NaN " "will be preserved/emitted to comply with IEEE-754.", From 4defde4a5f34cc79ef92ee11bcee38b8d38fc82d Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 22 Oct 2024 16:16:48 -0600 Subject: [PATCH 4/7] Update tests to not compare InvalidNumbers using strings --- pyomo/repn/tests/ampl/test_nlv2.py | 22 +++++++++--------- pyomo/repn/tests/test_linear.py | 36 ++++++++++++++++-------------- pyomo/repn/tests/test_util.py | 4 ++-- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/pyomo/repn/tests/ampl/test_nlv2.py b/pyomo/repn/tests/ampl/test_nlv2.py index 35030caeb52..43327dd9ae7 100644 --- a/pyomo/repn/tests/ampl/test_nlv2.py +++ b/pyomo/repn/tests/ampl/test_nlv2.py @@ -48,7 +48,7 @@ ) import pyomo.environ as pyo -_invalid_1j = r'InvalidNumber\((\([-+0-9.e]+\+)?1j\)?\)' +nan = float('nan') class INFO(object): @@ -171,7 +171,7 @@ def test_errors_divide_by_0(self): ) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertEqual(str(repn.const), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -186,7 +186,7 @@ def test_errors_divide_by_0(self): ) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertEqual(str(repn.const), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -201,7 +201,7 @@ def test_errors_divide_by_0(self): ) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertEqual(str(repn.const), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -216,7 +216,7 @@ def test_errors_divide_by_0(self): ) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertEqual(str(repn.const), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -231,7 +231,7 @@ def test_errors_divide_by_0(self): ) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertEqual(str(repn.const), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -424,7 +424,7 @@ def test_errors_negative_frac_pow(self): ) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertRegex(str(repn.const), _invalid_1j) + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(1j)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -440,7 +440,7 @@ def test_errors_negative_frac_pow(self): ) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertRegex(str(repn.const), _invalid_1j) + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(1j)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -460,7 +460,7 @@ def test_errors_unary_func(self): ) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertEqual(str(repn.const), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -484,7 +484,7 @@ def test_errors_propagate_nan(self): ) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertEqual(str(repn.const), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -494,7 +494,7 @@ def test_errors_propagate_nan(self): repn = info.visitor.walk_expression((expr, None, None, 1)) self.assertEqual(repn.nl, None) self.assertEqual(repn.mult, 1) - self.assertEqual(str(repn.const), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.const, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) diff --git a/pyomo/repn/tests/test_linear.py b/pyomo/repn/tests/test_linear.py index d88ddf96baa..3d5048ce653 100644 --- a/pyomo/repn/tests/test_linear.py +++ b/pyomo/repn/tests/test_linear.py @@ -149,7 +149,7 @@ def test_scalars(self): self.assertEqual(cfg.var_map, {}) self.assertEqual(cfg.var_order, {}) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.constant, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -203,7 +203,7 @@ def test_scalars(self): self.assertEqual(cfg.var_map, {}) self.assertEqual(cfg.var_order, {}) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.constant, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -214,7 +214,7 @@ def test_scalars(self): self.assertEqual(cfg.var_map, {}) self.assertEqual(cfg.var_order, {}) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(1j)') + self.assertEqual(repn.constant, InvalidNumber(1j)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -253,7 +253,7 @@ def test_npv(self): self.assertEqual(cfg.var_map, {}) self.assertEqual(cfg.var_order, {}) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.constant, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -488,7 +488,7 @@ def test_monomial(self): self.assertEqual(cfg.var_map, {}) self.assertEqual(cfg.var_order, {}) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.constant, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -498,7 +498,7 @@ def test_monomial(self): self.assertEqual(cfg.var_map, {}) self.assertEqual(cfg.var_order, {}) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.constant, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -508,7 +508,7 @@ def test_monomial(self): self.assertEqual(cfg.var_map, {}) self.assertEqual(cfg.var_order, {}) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.constant, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -551,7 +551,7 @@ def test_monomial(self): self.assertEqual(cfg.var_map, {}) self.assertEqual(cfg.var_order, {}) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.constant, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -765,7 +765,8 @@ def test_linear(self): with LoggingIntercept() as LOG: repn = LinearRepnVisitor(*cfg).walk_expression(e) self.assertIn( - "DEPRECATED: Encountered 0*nan in expression tree.", LOG.getvalue() + "DEPRECATED: Encountered 0*InvalidNumber(nan) in expression tree.", + LOG.getvalue(), ) self.assertEqual(cfg.subexpr, {}) @@ -1447,9 +1448,8 @@ def test_errors_propagate_nan(self): "\texpression: (x + 1)/p\n", ) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') - self.assertEqual(len(repn.linear), 1) - self.assertEqual(str(repn.linear[id(m.x)]), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.constant, InvalidNumber(nan)) + self.assertStructuredAlmostEqual(repn.linear, {id(m.x): InvalidNumber(nan)}) self.assertEqual(repn.nonlinear, None) expr = m.y + m.x + m.z + ((3 * m.x) / m.p) / m.y @@ -1464,16 +1464,16 @@ def test_errors_propagate_nan(self): ) self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 1) - self.assertEqual(len(repn.linear), 2) - self.assertEqual(repn.linear[id(m.z)], 1) - self.assertEqual(str(repn.linear[id(m.x)]), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual( + repn.linear, {id(m.z): 1, id(m.x): InvalidNumber(nan)} + ) self.assertEqual(repn.nonlinear, None) m.y.fix(None) expr = log(m.y) + 3 repn = LinearRepnVisitor(*cfg).walk_expression(expr) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertStructuredAlmostEqual(repn.constant, InvalidNumber(nan)) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) @@ -1631,7 +1631,9 @@ def test_nonnumeric(self): self.assertEqual(cfg.var_map, {}) self.assertEqual(cfg.var_order, {}) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(array([3, 4]))') + self.assertStructuredAlmostEqual( + repn.constant, InvalidNumber(numpy.array([3, 4])) + ) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) diff --git a/pyomo/repn/tests/test_util.py b/pyomo/repn/tests/test_util.py index e0fea0fb45c..fc9d86f966f 100644 --- a/pyomo/repn/tests/test_util.py +++ b/pyomo/repn/tests/test_util.py @@ -242,7 +242,7 @@ def test_apply_operation(self): pyomo.repn.util.HALT_ON_EVALUATION_ERROR = False with LoggingIntercept() as LOG: val = apply_node_operation(div, [1, 0]) - self.assertEqual(str(val), "InvalidNumber(nan)") + self.assertStructuredAlmostEqual(val, InvalidNumber(float('nan'))) self.assertEqual( LOG.getvalue(), "Exception encountered evaluating expression 'div(1, 0)'\n" @@ -293,7 +293,7 @@ class Visitor(object): pyomo.repn.util.HALT_ON_EVALUATION_ERROR = False with LoggingIntercept() as LOG: val = complex_number_error(1j, visitor, exp) - self.assertEqual(str(val), "InvalidNumber(1j)") + self.assertEqual(val, InvalidNumber(1j)) self.assertEqual( LOG.getvalue(), "Complex number returned from expression\n" From cd87b3c00a98eccb2f6db5b578bc5fac2a2f0309 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 31 Oct 2024 12:10:09 -0600 Subject: [PATCH 5/7] Restore original contribution_guide URL --- doc/OnlineDocs/{howto => }/contribution_guide.rst | 0 doc/OnlineDocs/howto/index.rst | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename doc/OnlineDocs/{howto => }/contribution_guide.rst (100%) diff --git a/doc/OnlineDocs/howto/contribution_guide.rst b/doc/OnlineDocs/contribution_guide.rst similarity index 100% rename from doc/OnlineDocs/howto/contribution_guide.rst rename to doc/OnlineDocs/contribution_guide.rst diff --git a/doc/OnlineDocs/howto/index.rst b/doc/OnlineDocs/howto/index.rst index 1dd04dd056c..9f700bff4e8 100644 --- a/doc/OnlineDocs/howto/index.rst +++ b/doc/OnlineDocs/howto/index.rst @@ -9,4 +9,4 @@ How-To Guides solver_recipes abstract_models/index.rst debugging - contribution_guide + ../contribution_guide From 2086d3eab86d543082312456e077277b2113c424 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 31 Oct 2024 12:10:45 -0600 Subject: [PATCH 6/7] Update PyomoDOE documentation URL --- pyomo/contrib/doe/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/doe/__init__.py b/pyomo/contrib/doe/__init__.py index 154b52124d3..14589244135 100644 --- a/pyomo/contrib/doe/__init__.py +++ b/pyomo/contrib/doe/__init__.py @@ -17,9 +17,9 @@ deprecation_message = ( "Pyomo.DoE has been refactored. The current interface utilizes Experiment " "objects that label unknown parameters, experiment inputs, experiment outputs " - "and measurement error. This avoids string-based naming which is fragile. For " - "instructions to use the new interface, please see the Pyomo.DoE under the contributed " - "packages documentation at `https://pyomo.readthedocs.io/en/latest/contributed_packages/doe/doe.html`" + "and measurement error. This avoids fragile string-based naming. For " + "instructions on using the new interface, please see the Pyomo.DoE documentation " + "`https://pyomo.readthedocs.io/en/latest/explanation/analysis/doe/doe.html`" ) From b5844b92a9509ac8e8323bcc2e45154ecedeaa47 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 31 Oct 2024 12:12:48 -0600 Subject: [PATCH 7/7] Tracking move of contribution_guide --- doc/OnlineDocs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/OnlineDocs/index.rst b/doc/OnlineDocs/index.rst index e496824188c..0d7ccc41592 100644 --- a/doc/OnlineDocs/index.rst +++ b/doc/OnlineDocs/index.rst @@ -92,7 +92,7 @@ Contributing to Pyomo --------------------- Interested in contributing code or documentation to the project? Check out our -:doc:`Contribution Guide ` +:doc:`Contribution Guide ` Related Packages ----------------