diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 8ba04eec466..03894a1cb20 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -519,7 +519,7 @@ jobs: $BARON_DIR = "${env:TPL_DIR}/baron" echo "$BARON_DIR" | ` Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - $URL = "https://www.minlp.com/downloads/xecs/baron/current/" + $URL = "https://minlp.com/downloads/xecs/baron/current/" if ( "${{matrix.TARGET}}" -eq "win" ) { $INSTALLER = "${env:DOWNLOAD_DIR}/baron_install.exe" $URL += "baron-win64.exe" diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 0bfd12b998d..cc9760cbe5d 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -562,7 +562,7 @@ jobs: $BARON_DIR = "${env:TPL_DIR}/baron" echo "$BARON_DIR" | ` Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - $URL = "https://www.minlp.com/downloads/xecs/baron/current/" + $URL = "https://minlp.com/downloads/xecs/baron/current/" if ( "${{matrix.TARGET}}" -eq "win" ) { $INSTALLER = "${env:DOWNLOAD_DIR}/baron_install.exe" $URL += "baron-win64.exe" diff --git a/.jenkins.sh b/.jenkins.sh index 51dd92ba123..8771427805d 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -208,13 +208,28 @@ if test -z "$MODE" -o "$MODE" == test; then coverage report -i || exit 1 coverage xml -i || exit 1 export OS=`uname` - if test -n "$CODECOV_TOKEN"; then - CODECOV_JOB_NAME=`echo ${JOB_NAME} | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/'`.$BUILD_NUMBER.$python + if test -z "$PYOMO_SOURCE_SHA"; then + PYOMO_SOURCE_SHA=$GIT_COMMIT + fi + if test -n "$CODECOV_TOKEN" -a -n "$PYOMO_SOURCE_SHA"; then + CODECOV_JOB_NAME=$(echo ${JOB_NAME} \ + | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/').$BUILD_NUMBER.$python if test -z "$CODECOV_REPO_OWNER"; then - CODECOV_REPO_OWNER="pyomo" + if test -n "$PYOMO_SOURCE_REPO"; then + CODECOV_REPO_OWNER=$(echo "$PYOMO_SOURCE_REPO" | cut -d '/' -f 4) + elif test -n "$GIT_URL"; then + CODECOV_REPO_OWNER=$(echo "$GIT_URL" | cut -d '/' -f 4) + else + CODECOV_REPO_OWNER="" + fi fi - if test -z "CODECOV_SOURCE_BRANCH"; then - CODECOV_SOURCE_BRANCH="main" + if test -z "$CODECOV_SOURCE_BRANCH"; then + CODECOV_SOURCE_BRANCH=$(git branch -av --contains "$PYOMO_SOURCE_SHA" \ + | grep "${PYOMO_SOURCE_SHA:0:7}" | grep "/origin/" \ + | cut -d '/' -f 3 | cut -d' ' -f 1) + if test -z "$CODECOV_SOURCE_BRANCH"; then + CODECOV_SOURCE_BRANCH=main + fi fi i=0 while /bin/true; do diff --git a/examples/pyomo/tutorials/set.dat b/examples/pyomo/tutorials/set.dat index ab0d00b43cc..e2ad04122d8 100644 --- a/examples/pyomo/tutorials/set.dat +++ b/examples/pyomo/tutorials/set.dat @@ -16,3 +16,6 @@ set S[5] := 2 3; set T[2] := 1 3; set T[5] := 2 3; + +set X[2] := 1; +set X[5] := 2 3; \ No newline at end of file diff --git a/examples/pyomo/tutorials/set.out b/examples/pyomo/tutorials/set.out index 818977f6155..dd1ef2d4335 100644 --- a/examples/pyomo/tutorials/set.out +++ b/examples/pyomo/tutorials/set.out @@ -1,4 +1,4 @@ -23 Set Declarations +24 Set Declarations A : Size=1, Index=None, Ordered=Insertion Key : Dimen : Domain : Size : Members None : 1 : Any : 3 : {1, 2, 3} @@ -89,5 +89,9 @@ 2 : 1 : Any : 5 : {1, 3, 5, 7, 9} 3 : 1 : Any : 5 : {1, 4, 7, 10, 13} 4 : 1 : Any : 5 : {1, 5, 9, 13, 17} + X : Size=2, Index=B, Ordered=Insertion + Key : Dimen : Domain : Size : Members + 2 : 1 : S[2] : 1 : {1,} + 5 : 1 : S[5] : 2 : {2, 3} -23 Declarations: A B C D E F G H Hsub I J K K_2 L M N O P R S T U V +24 Declarations: A B C D E F G H Hsub I J K K_2 L M N O P R S X T U V diff --git a/examples/pyomo/tutorials/set.py b/examples/pyomo/tutorials/set.py index a14301484c9..c1ea60b48ad 100644 --- a/examples/pyomo/tutorials/set.py +++ b/examples/pyomo/tutorials/set.py @@ -171,6 +171,13 @@ def P_init(model, i, j): # model.S = Set(model.B, within=model.A) +# +# Validation of a set array can also be linked to another set array. If so, the +# elements under each index must also be found under the corresponding index in +# the validation set array: +# +model.X = Set(model.B, within=model.S) + # # Validation of set arrays can also be performed with the _validate_ option. diff --git a/pyomo/contrib/appsi/solvers/highs.py b/pyomo/contrib/appsi/solvers/highs.py index c948444839d..57a7b1eac72 100644 --- a/pyomo/contrib/appsi/solvers/highs.py +++ b/pyomo/contrib/appsi/solvers/highs.py @@ -481,7 +481,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): indices_to_remove.append(con_ndx) self._mutable_helpers.pop(con, None) self._solver_model.deleteRows( - len(indices_to_remove), np.array(indices_to_remove) + len(indices_to_remove), np.sort(np.array(indices_to_remove)) ) con_ndx = 0 new_con_map = dict() diff --git a/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py b/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py index b26f45ff2cc..4d8251e0de9 100644 --- a/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py +++ b/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py @@ -80,6 +80,43 @@ def test_mutable_params_with_remove_vars(self): res = opt.solve(m) self.assertAlmostEqual(res.best_feasible_objective, -9) + def test_fix_and_unfix(self): + # Tests issue https://github.com/Pyomo/pyomo/issues/3127 + + m = pe.ConcreteModel() + m.x = pe.Var(domain=pe.Binary) + m.y = pe.Var(domain=pe.Binary) + m.fx = pe.Var(domain=pe.NonNegativeReals) + m.fy = pe.Var(domain=pe.NonNegativeReals) + m.c1 = pe.Constraint(expr=m.fx <= m.x) + m.c2 = pe.Constraint(expr=m.fy <= m.y) + m.c3 = pe.Constraint(expr=m.x + m.y <= 1) + + m.obj = pe.Objective(expr=m.fx * 0.5 + m.fy * 0.4, sense=pe.maximize) + + opt = Highs() + + # solution 1 has m.x == 1 and m.y == 0 + r = opt.solve(m) + self.assertAlmostEqual(m.fx.value, 1, places=5) + self.assertAlmostEqual(m.fy.value, 0, places=5) + self.assertAlmostEqual(r.best_feasible_objective, 0.5, places=5) + + # solution 2 has m.x == 0 and m.y == 1 + m.y.fix(1) + r = opt.solve(m) + self.assertAlmostEqual(m.fx.value, 0, places=5) + self.assertAlmostEqual(m.fy.value, 1, places=5) + self.assertAlmostEqual(r.best_feasible_objective, 0.4, places=5) + + # solution 3 should be equal solution 1 + m.y.unfix() + m.x.fix(1) + r = opt.solve(m) + self.assertAlmostEqual(m.fx.value, 1, places=5) + self.assertAlmostEqual(m.fy.value, 0, places=5) + self.assertAlmostEqual(r.best_feasible_objective, 0.5, places=5) + def test_capture_highs_output(self): # tests issue #3003 # diff --git a/pyomo/contrib/incidence_analysis/scc_solver.py b/pyomo/contrib/incidence_analysis/scc_solver.py index 378647c190c..db201dccb0a 100644 --- a/pyomo/contrib/incidence_analysis/scc_solver.py +++ b/pyomo/contrib/incidence_analysis/scc_solver.py @@ -66,7 +66,14 @@ def generate_strongly_connected_components( ) ) - assert len(variables) == len(constraints) + if len(variables) != len(constraints): + nvar = len(variables) + ncon = len(constraints) + raise RuntimeError( + "generate_strongly_connected_components only supports systems with the" + f" same numbers of variables and equality constraints. Got {nvar}" + f" variables and {ncon} constraints." + ) if igraph is None: igraph = IncidenceGraphInterface() @@ -78,6 +85,8 @@ def generate_strongly_connected_components( subsets, include_fixed=include_fixed ): # TODO: How does len scale for reference-to-list? + # If this assert fails, it may be due to a bug in block_triangularize + # or generate_subsystem_block. assert len(block.vars) == len(block.cons) yield (block, inputs) diff --git a/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py b/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py index b75f93e4a12..ef4853d7e9a 100644 --- a/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py +++ b/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py @@ -501,5 +501,22 @@ def test_with_inequalities(self): self.assertEqual(m.x[3].value, 1.0) +@unittest.skipUnless(scipy_available, "SciPy is not available") +@unittest.skipUnless(networkx_available, "NetworkX is not available") +class TestExceptions(unittest.TestCase): + def test_nonsquare_system(self): + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2], initialize=1) + m.eq = pyo.Constraint(expr=m.x[1] + m.x[2] == 1) + + msg = "Got 2 variables and 1 constraints" + with self.assertRaisesRegex(RuntimeError, msg): + list( + generate_strongly_connected_components( + constraints=[m.eq], variables=[m.x[1], m.x[2]] + ) + ) + + if __name__ == "__main__": unittest.main() diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index f7efec4d6e7..f2954750a16 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -42,6 +42,7 @@ from pyomo.contrib.pyros.util import get_vars_from_component from pyomo.contrib.pyros.util import identify_objective_functions from pyomo.common.collections import Bunch +from pyomo.repn.plugins import nl_writer as pyomo_nl_writer import time import math from pyomo.contrib.pyros.util import time_code @@ -68,7 +69,7 @@ from pyomo.common.dependencies import numpy as np, numpy_available from pyomo.common.dependencies import scipy as sp, scipy_available from pyomo.environ import maximize as pyo_max -from pyomo.common.errors import ApplicationError +from pyomo.common.errors import ApplicationError, InfeasibleConstraintException from pyomo.opt import ( SolverResults, SolverStatus, @@ -4616,6 +4617,76 @@ def test_discrete_separation_subsolver_error(self): ), ) + @unittest.skipUnless(ipopt_available, "IPOPT is not available.") + def test_pyros_nl_writer_tol(self): + """ + Test PyROS subsolver call routine behavior + with respect to the NL writer tolerance is as + expected. + """ + m = ConcreteModel() + m.q = Param(initialize=1, mutable=True) + m.x1 = Var(initialize=1, bounds=(0, 1)) + m.x2 = Var(initialize=2, bounds=(0, m.q)) + m.obj = Objective(expr=m.x1 + m.x2) + + # fixed just inside the PyROS-specified NL writer tolerance. + m.x1.fix(m.x1.upper + 9.9e-5) + + current_nl_writer_tol = pyomo_nl_writer.TOL + ipopt_solver = SolverFactory("ipopt") + pyros_solver = SolverFactory("pyros") + + pyros_solver.solve( + model=m, + first_stage_variables=[m.x1], + second_stage_variables=[m.x2], + uncertain_params=[m.q], + uncertainty_set=BoxSet([[0, 1]]), + local_solver=ipopt_solver, + global_solver=ipopt_solver, + decision_rule_order=0, + solve_master_globally=False, + bypass_global_separation=True, + ) + + self.assertEqual( + pyomo_nl_writer.TOL, + current_nl_writer_tol, + msg="Pyomo NL writer tolerance not restored as expected.", + ) + + # fixed just outside the PyROS-specified NL writer tolerance. + # this should be exceptional. + m.x1.fix(m.x1.upper + 1.01e-4) + + err_msg = ( + "model contains a trivially infeasible variable.*x1" + ".*fixed.*outside bounds" + ) + with self.assertRaisesRegex(InfeasibleConstraintException, err_msg): + pyros_solver.solve( + model=m, + first_stage_variables=[m.x1], + second_stage_variables=[m.x2], + uncertain_params=[m.q], + uncertainty_set=BoxSet([[0, 1]]), + local_solver=ipopt_solver, + global_solver=ipopt_solver, + decision_rule_order=0, + solve_master_globally=False, + bypass_global_separation=True, + ) + + self.assertEqual( + pyomo_nl_writer.TOL, + current_nl_writer_tol, + msg=( + "Pyomo NL writer tolerance not restored as expected " + "after exceptional test." + ), + ) + @unittest.skipUnless( baron_license_is_valid, "Global NLP solver is not available and licensed." ) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 3b0187af7dd..ecabca8f115 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -38,6 +38,7 @@ from pyomo.core.expr import value from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression from pyomo.repn.standard_repn import generate_standard_repn +from pyomo.repn.plugins import nl_writer as pyomo_nl_writer from pyomo.core.expr.visitor import ( identify_variables, identify_mutable_parameters, @@ -377,7 +378,14 @@ def revert_solver_max_time_adjustment( elif isinstance(solver, SolverFactory.get_class("baron")): options_key = "MaxTime" elif isinstance(solver, SolverFactory.get_class("ipopt")): - options_key = "max_cpu_time" + options_key = ( + # IPOPT 3.14.0+ added support for specifying + # wall time limit explicitly; this is preferred + # over CPU time limit + "max_wall_time" + if solver.version() >= (3, 14, 0, 0) + else "max_cpu_time" + ) elif isinstance(solver, SolverFactory.get_class("scip")): options_key = "limits/time" else: @@ -1809,6 +1817,16 @@ def call_solver(model, solver, config, timing_obj, timer_name, err_msg): timing_obj.start_timer(timer_name) tt_timer.tic(msg=None) + # tentative: reduce risk of InfeasibleConstraintException + # occurring due to discrepancies between Pyomo NL writer + # tolerance and (default) subordinate solver (e.g. IPOPT) + # feasibility tolerances. + # e.g., a Var fixed outside bounds beyond the Pyomo NL writer + # tolerance, but still within the default IPOPT feasibility + # tolerance + current_nl_writer_tol = pyomo_nl_writer.TOL + pyomo_nl_writer.TOL = 1e-4 + try: results = solver.solve( model, @@ -1827,6 +1845,8 @@ def call_solver(model, solver, config, timing_obj, timer_name, err_msg): results.solver, TIC_TOC_SOLVE_TIME_ATTR, tt_timer.toc(msg=None, delta=True) ) finally: + pyomo_nl_writer.TOL = current_nl_writer_tol + timing_obj.stop_timer(timer_name) revert_solver_max_time_adjustment( solver, orig_setting, custom_setting_present, config diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index c0d427747c5..049775fd9dd 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1932,7 +1932,8 @@ class Set(IndexedComponent): within : initialiser(set), optional A set that defines the valid values that can be contained - in this set + in this set. If the latter is indexed, the former can be indexed or + non-indexed, in which case it applies to all indices. domain : initializer(set), optional A set that defines the valid values that can be contained in this set @@ -2218,7 +2219,7 @@ def _getitem_when_not_present(self, index): domain = self._init_domain(_block, index, self) if domain is not None: - domain.construct() + domain.parent_component().construct() if _d is UnknownSetDimen and domain is not None and domain.dimen is not None: _d = domain.dimen diff --git a/pyomo/core/expr/compare.py b/pyomo/core/expr/compare.py index 44d9c4205d7..4a777a9b977 100644 --- a/pyomo/core/expr/compare.py +++ b/pyomo/core/expr/compare.py @@ -230,10 +230,14 @@ def assertExpressionsEqual(test, a, b, include_named_exprs=True, places=None): test.assertEqual(len(prefix_a), len(prefix_b)) for _a, _b in zip(prefix_a, prefix_b): test.assertIs(_a.__class__, _b.__class__) - if places is None: - test.assertEqual(_a, _b) + # If _a is nan, check _b is nan + if _a != _a: + test.assertTrue(_b != _b) else: - test.assertAlmostEqual(_a, _b, places=places) + if places is None: + test.assertEqual(_a, _b) + else: + test.assertAlmostEqual(_a, _b, places=places) except (PyomoException, AssertionError): test.fail( f"Expressions not equal:\n\t" @@ -292,10 +296,13 @@ def assertExpressionsStructurallyEqual( for _a, _b in zip(prefix_a, prefix_b): if _a.__class__ not in native_types and _b.__class__ not in native_types: test.assertIs(_a.__class__, _b.__class__) - if places is None: - test.assertEqual(_a, _b) + if _a != _a: + test.assertTrue(_b != _b) else: - test.assertAlmostEqual(_a, _b, places=places) + if places is None: + test.assertEqual(_a, _b) + else: + test.assertAlmostEqual(_a, _b, places=places) except (PyomoException, AssertionError): test.fail( f"Expressions not structurally equal:\n\t" diff --git a/pyomo/core/kernel/conic.py b/pyomo/core/kernel/conic.py index 1bb5f1b6ce8..bd78ba310f4 100644 --- a/pyomo/core/kernel/conic.py +++ b/pyomo/core/kernel/conic.py @@ -632,7 +632,7 @@ def as_domain(cls, r, x): b = block() b.r = variable_tuple([variable(lb=0) for i in range(len(r))]) b.x = variable() - b.c = _build_linking_constraints(list(r) + [x], list(b.r) + [x]) + b.c = _build_linking_constraints(list(r) + [x], list(b.r) + [b.x]) b.q = cls(r=b.r, x=b.x) return b @@ -934,7 +934,7 @@ def as_domain(cls, r, x): b = block() b.r = variable_tuple([variable(lb=0) for i in range(len(r))]) b.x = variable() - b.c = _build_linking_constraints(list(r) + [x], list(b.r) + [x]) + b.c = _build_linking_constraints(list(r) + [x], list(b.r) + [b.x]) b.q = cls(r=b.r, x=b.x) return b diff --git a/pyomo/core/tests/unit/kernel/test_conic.py b/pyomo/core/tests/unit/kernel/test_conic.py index ccfbcca7e1f..bd97c13fc2e 100644 --- a/pyomo/core/tests/unit/kernel/test_conic.py +++ b/pyomo/core/tests/unit/kernel/test_conic.py @@ -35,6 +35,8 @@ primal_power, dual_exponential, dual_power, + primal_geomean, + dual_geomean, ) @@ -784,6 +786,40 @@ def test_as_domain(self): x[1].value = None +# These mosek 10 constraints can't be evaluated, pprinted, checked for convexity, +# pickled, etc., so I won't use the _conic_tester_base for them +class Test_primal_geomean(unittest.TestCase): + def test_as_domain(self): + b = primal_geomean.as_domain(r=[2, 3], x=6) + self.assertIs(type(b), block) + self.assertIs(type(b.q), primal_geomean) + self.assertIs(type(b.r), variable_tuple) + self.assertIs(type(b.x), variable) + self.assertIs(type(b.c), constraint_tuple) + self.assertExpressionsEqual(b.c[0].body, b.r[0]) + self.assertExpressionsEqual(b.c[0].rhs, 2) + self.assertExpressionsEqual(b.c[1].body, b.r[1]) + self.assertExpressionsEqual(b.c[1].rhs, 3) + self.assertExpressionsEqual(b.c[2].body, b.x) + self.assertExpressionsEqual(b.c[2].rhs, 6) + + +class Test_dual_geomean(unittest.TestCase): + def test_as_domain(self): + b = dual_geomean.as_domain(r=[2, 3], x=6) + self.assertIs(type(b), block) + self.assertIs(type(b.q), dual_geomean) + self.assertIs(type(b.r), variable_tuple) + self.assertIs(type(b.x), variable) + self.assertIs(type(b.c), constraint_tuple) + self.assertExpressionsEqual(b.c[0].body, b.r[0]) + self.assertExpressionsEqual(b.c[0].rhs, 2) + self.assertExpressionsEqual(b.c[1].body, b.r[1]) + self.assertExpressionsEqual(b.c[1].rhs, 3) + self.assertExpressionsEqual(b.c[2].body, b.x) + self.assertExpressionsEqual(b.c[2].rhs, 6) + + class TestMisc(unittest.TestCase): def test_build_linking_constraints(self): c = _build_linking_constraints([], []) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index f62589a6873..2b0da8b861d 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -4543,9 +4543,11 @@ def test_construction(self): m.I = Set(initialize=[1, 2, 3]) m.J = Set(initialize=[4, 5, 6]) m.K = Set(initialize=[(1, 4), (2, 6), (3, 5)], within=m.I * m.J) + m.L = Set(initialize=[1, 3], within=m.I) m.II = Set([1, 2, 3], initialize={1: [0], 2: [1, 2], 3: range(3)}) m.JJ = Set([1, 2, 3], initialize={1: [0], 2: [1, 2], 3: range(3)}) m.KK = Set([1, 2], initialize=[], dimen=lambda m, i: i) + m.LL = Set([2, 3], within=m.II, initialize={2: [1, 2], 3: [1]}) output = StringIO() m.I.pprint(ostream=output) @@ -4569,6 +4571,8 @@ def test_construction(self): 'I': [-1, 0], 'II': {1: [10, 11], 3: [30]}, 'K': [-1, 4, -1, 6, 0, 5], + 'L': [-1], + 'LL': {3: [30]}, } } ) @@ -4576,6 +4580,7 @@ def test_construction(self): self.assertEqual(list(i.I), [-1, 0]) self.assertEqual(list(i.J), [4, 5, 6]) self.assertEqual(list(i.K), [(-1, 4), (-1, 6), (0, 5)]) + self.assertEqual(list(i.L), [-1]) self.assertEqual(list(i.II[1]), [10, 11]) self.assertEqual(list(i.II[3]), [30]) self.assertEqual(list(i.JJ[1]), [0]) @@ -4583,9 +4588,11 @@ def test_construction(self): self.assertEqual(list(i.JJ[3]), [0, 1, 2]) self.assertEqual(list(i.KK[1]), []) self.assertEqual(list(i.KK[2]), []) + self.assertEqual(list(i.LL[3]), [30]) # Implicitly-constructed set should fall back on initialize! self.assertEqual(list(i.II[2]), [1, 2]) + self.assertEqual(list(i.LL[2]), [1, 2]) # Additional tests for tuplize: i = m.create_instance(data={None: {'K': [(1, 4), (2, 6)], 'KK': [1, 4, 2, 6]}}) @@ -6388,3 +6395,209 @@ def test_issue_1112(self): self.assertEqual(len(vals), 1) self.assertIsInstance(vals[0], SetProduct_OrderedSet) self.assertIsNot(vals[0], cross) + + def test_issue_3284(self): + # test creating (indexed and non-indexed) sets using the within argument + # using concrete model and initialization + problem = ConcreteModel() + # non-indexed sets not using the within argument + problem.A = Set(initialize=[1, 2, 3]) + problem.B = Set(dimen=2, initialize=[(1, 2), (3, 4), (5, 6)]) + # non-indexed sets using within argument + problem.subset_A = Set(within=problem.A, initialize=[2, 3]) + problem.subset_B = Set(within=problem.B, dimen=2, initialize=[(1, 2), (5, 6)]) + # indexed sets not using the within argument + problem.C = Set(problem.A, initialize={1: [-1, 3], 2: [4, 7], 3: [3, 8]}) + problem.D = Set( + problem.B, initialize={(1, 2): [1, 5], (3, 4): [3], (5, 6): [6, 8, 9]} + ) + # indexed sets using an indexed set for the within argument + problem.subset_C = Set( + problem.A, within=problem.C, initialize={1: [-1], 2: [4], 3: [3, 8]} + ) + problem.subset_D = Set( + problem.B, + within=problem.D, + initialize={(1, 2): [1, 5], (3, 4): [], (5, 6): [6]}, + ) + # indexed sets using a non-indexed set for the within argument + problem.E = Set([0, 1], within=problem.A, initialize={0: [1, 2], 1: [3]}) + problem.F = Set( + [(1, 2, 3), (4, 5, 6)], + within=problem.B, + initialize={(1, 2, 3): [(1, 2)], (4, 5, 6): [(3, 4)]}, + ) + # check them + self.assertEqual(list(problem.A), [1, 2, 3]) + self.assertEqual(list(problem.B), [(1, 2), (3, 4), (5, 6)]) + self.assertEqual(list(problem.subset_A), [2, 3]) + self.assertEqual(list(problem.subset_B), [(1, 2), (5, 6)]) + self.assertEqual(list(problem.C[1]), [-1, 3]) + self.assertEqual(list(problem.C[2]), [4, 7]) + self.assertEqual(list(problem.C[3]), [3, 8]) + self.assertEqual(list(problem.D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.D[(3, 4)]), [3]) + self.assertEqual(list(problem.D[(5, 6)]), [6, 8, 9]) + self.assertEqual(list(problem.subset_C[1]), [-1]) + self.assertEqual(list(problem.subset_C[2]), [4]) + self.assertEqual(list(problem.subset_C[3]), [3, 8]) + self.assertEqual(list(problem.subset_D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.subset_D[(3, 4)]), []) + self.assertEqual(list(problem.subset_D[(5, 6)]), [6]) + self.assertEqual(list(problem.E[0]), [1, 2]) + self.assertEqual(list(problem.E[1]), [3]) + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(3, 4)]) + + # try adding elements to test the domains (1 compatible, 1 incompatible) + # set subset_A + problem.subset_A.add(1) + error_message = ( + "Cannot add value 4 to Set subset_A.\n\tThe value is not in the domain A" + ) + with self.assertRaisesRegex(ValueError, error_message): + problem.subset_A.add(4) + # set subset_B + problem.subset_B.add((3, 4)) + with self.assertRaisesRegex(ValueError, r".*Cannot add value \(7, 8\)"): + problem.subset_B.add((7, 8)) + # set subset_C + problem.subset_C[2].add(7) + with self.assertRaisesRegex(ValueError, ".*Cannot add value 8 to Set"): + problem.subset_C[2].add(8) + # set subset_D + problem.subset_D[(5, 6)].add(9) + with self.assertRaisesRegex(ValueError, ".*Cannot add value 2 to Set"): + problem.subset_D[(3, 4)].add(2) + # set E + problem.E[1].add(2) + with self.assertRaisesRegex(ValueError, ".*Cannot add value 4 to Set"): + problem.E[1].add(4) + # set F + problem.F[(1, 2, 3)].add((3, 4)) + with self.assertRaisesRegex(ValueError, r".*Cannot add value \(4, 3\)"): + problem.F[(4, 5, 6)].add((4, 3)) + # check them + self.assertEqual(list(problem.A), [1, 2, 3]) + self.assertEqual(list(problem.B), [(1, 2), (3, 4), (5, 6)]) + self.assertEqual(list(problem.subset_A), [2, 3, 1]) + self.assertEqual(list(problem.subset_B), [(1, 2), (5, 6), (3, 4)]) + self.assertEqual(list(problem.C[1]), [-1, 3]) + self.assertEqual(list(problem.C[2]), [4, 7]) + self.assertEqual(list(problem.C[3]), [3, 8]) + self.assertEqual(list(problem.D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.D[(3, 4)]), [3]) + self.assertEqual(list(problem.D[(5, 6)]), [6, 8, 9]) + self.assertEqual(list(problem.subset_C[1]), [-1]) + self.assertEqual(list(problem.subset_C[2]), [4, 7]) + self.assertEqual(list(problem.subset_C[3]), [3, 8]) + self.assertEqual(list(problem.subset_D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.subset_D[(3, 4)]), []) + self.assertEqual(list(problem.subset_D[(5, 6)]), [6, 9]) + self.assertEqual(list(problem.E[0]), [1, 2]) + self.assertEqual(list(problem.E[1]), [3, 2]) + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2), (3, 4)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(3, 4)]) + + # using abstract model and no initialization + model = AbstractModel() + # non-indexed sets not using the within argument + model.A = Set() + model.B = Set(dimen=2) + # non-indexed sets using within argument + model.subset_A = Set(within=model.A) + model.subset_B = Set(within=model.B, dimen=2) + # indexed sets not using the within argument + model.C = Set(model.A) + model.D = Set(model.B) + # indexed sets using an indexed set for the within argument + model.subset_C = Set(model.A, within=model.C) + model.subset_D = Set(model.B, within=model.D) + # indexed sets using a non-indexed set for the within argument + model.E_index = Set() + model.F_index = Set() + model.E = Set(model.E_index, within=model.A) + model.F = Set(model.F_index, within=model.B) + problem = model.create_instance( + data={ + None: { + 'A': [3, 4, 5], + 'B': [(1, 2), (7, 8)], + 'subset_A': [3, 4], + 'subset_B': [(1, 2)], + 'C': {3: [3], 4: [4, 8], 5: [5, 6]}, + 'D': {(1, 2): [2], (7, 8): [0, 1]}, + 'subset_C': {3: [3], 4: [8], 5: []}, + 'subset_D': {(1, 2): [], (7, 8): [0, 1]}, + 'E_index': [0, 1], + 'F_index': [(1, 2, 3), (4, 5, 6)], + 'E': {0: [3, 4], 1: [5]}, + 'F': {(1, 2, 3): [(1, 2)], (4, 5, 6): [(7, 8)]}, + } + } + ) + + # check them + self.assertEqual(list(problem.A), [3, 4, 5]) + self.assertEqual(list(problem.B), [(1, 2), (7, 8)]) + self.assertEqual(list(problem.subset_A), [3, 4]) + self.assertEqual(list(problem.subset_B), [(1, 2)]) + self.assertEqual(list(problem.C[3]), [3]) + self.assertEqual(list(problem.C[4]), [4, 8]) + self.assertEqual(list(problem.C[5]), [5, 6]) + self.assertEqual(list(problem.D[(1, 2)]), [2]) + self.assertEqual(list(problem.D[(7, 8)]), [0, 1]) + self.assertEqual(list(problem.subset_C[3]), [3]) + self.assertEqual(list(problem.subset_C[4]), [8]) + self.assertEqual(list(problem.subset_C[5]), []) + self.assertEqual(list(problem.subset_D[(1, 2)]), []) + self.assertEqual(list(problem.subset_D[(7, 8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3, 4]) + self.assertEqual(list(problem.E[1]), [5]) + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(7, 8)]) + + # try adding elements to test the domains (1 compatible, 1 incompatible) + # set subset_A + problem.subset_A.add(5) + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + problem.subset_A.add(6) + # set subset_B + problem.subset_B.add((7, 8)) + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + problem.subset_B.add((3, 4)) + # set subset_C + problem.subset_C[4].add(4) + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + problem.subset_C[4].add(9) + # set subset_D + problem.subset_D[(1, 2)].add(2) + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + problem.subset_D[(1, 2)].add(3) + # set E + problem.E[1].add(4) + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + problem.E[1].add(1) + # set F + problem.F[(1, 2, 3)].add((7, 8)) + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + problem.F[(4, 5, 6)].add((4, 3)) + # check them + self.assertEqual(list(problem.A), [3, 4, 5]) + self.assertEqual(list(problem.B), [(1, 2), (7, 8)]) + self.assertEqual(list(problem.subset_A), [3, 4, 5]) + self.assertEqual(list(problem.subset_B), [(1, 2), (7, 8)]) + self.assertEqual(list(problem.C[3]), [3]) + self.assertEqual(list(problem.C[4]), [4, 8]) + self.assertEqual(list(problem.C[5]), [5, 6]) + self.assertEqual(list(problem.D[(1, 2)]), [2]) + self.assertEqual(list(problem.D[(7, 8)]), [0, 1]) + self.assertEqual(list(problem.subset_C[3]), [3]) + self.assertEqual(list(problem.subset_C[4]), [8, 4]) + self.assertEqual(list(problem.subset_C[5]), []) + self.assertEqual(list(problem.subset_D[(1, 2)]), [2]) + self.assertEqual(list(problem.subset_D[(7, 8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3, 4]) + self.assertEqual(list(problem.E[1]), [5, 4]) + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2), (7, 8)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(7, 8)]) diff --git a/pyomo/gdp/plugins/multiple_bigm.py b/pyomo/gdp/plugins/multiple_bigm.py index 4dffd4e9f9a..3362276246b 100644 --- a/pyomo/gdp/plugins/multiple_bigm.py +++ b/pyomo/gdp/plugins/multiple_bigm.py @@ -12,7 +12,7 @@ import itertools import logging -from pyomo.common.collections import ComponentMap +from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.config import ConfigDict, ConfigValue from pyomo.common.gc_manager import PauseGC from pyomo.common.modeling import unique_component_name @@ -310,9 +310,12 @@ def _transform_disjunctionData(self, obj, index, parent_disjunct, root_disjunct) arg_Ms = self._config.bigM if self._config.bigM is not None else {} + # ESJ: I am relying on the fact that the ComponentSet is going to be + # ordered here, but using a set because I will remove infeasible + # Disjuncts from it if I encounter them calculating M's. + active_disjuncts = ComponentSet(disj for disj in obj.disjuncts if disj.active) # First handle the bound constraints if we are dealing with them # separately - active_disjuncts = [disj for disj in obj.disjuncts if disj.active] transformed_constraints = set() if self._config.reduce_bound_constraints: transformed_constraints = self._transform_bound_constraints( @@ -585,7 +588,7 @@ def _calculate_missing_M_values( ): if disjunct is other_disjunct: continue - if id(other_disjunct) in scratch_blocks: + elif id(other_disjunct) in scratch_blocks: scratch = scratch_blocks[id(other_disjunct)] else: scratch = scratch_blocks[id(other_disjunct)] = Block() @@ -631,7 +634,10 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.lower scratch.obj.sense = minimize lower_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg + other_disjunct, + scratch, + unsuccessful_solve_msg, + active_disjuncts, ) if constraint.upper is not None and upper_M is None: # last resort: calculate @@ -639,7 +645,10 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.upper scratch.obj.sense = maximize upper_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg + other_disjunct, + scratch, + unsuccessful_solve_msg, + active_disjuncts, ) arg_Ms[constraint, other_disjunct] = (lower_M, upper_M) transBlock._mbm_values[constraint, other_disjunct] = (lower_M, upper_M) @@ -651,9 +660,18 @@ def _calculate_missing_M_values( return arg_Ms def _solve_disjunct_for_M( - self, other_disjunct, scratch_block, unsuccessful_solve_msg + self, other_disjunct, scratch_block, unsuccessful_solve_msg, active_disjuncts ): + if not other_disjunct.active: + # If a Disjunct is infeasible, we will discover that and deactivate + # it when we are calculating the M values. We remove that disjunct + # from active_disjuncts inside of the loop in + # _calculate_missing_M_values. So that means that we might have + # deactivated Disjuncts here that we should skip over. + return 0 + solver = self._config.solver + results = solver.solve(other_disjunct, load_solutions=False) if results.solver.termination_condition is TerminationCondition.infeasible: # [2/18/24]: TODO: After the solver rewrite is complete, we will not @@ -669,6 +687,7 @@ def _solve_disjunct_for_M( "Disjunct '%s' is infeasible, deactivating." % other_disjunct.name ) other_disjunct.deactivate() + active_disjuncts.remove(other_disjunct) M = 0 else: # This is a solver that might report diff --git a/pyomo/gdp/tests/test_mbigm.py b/pyomo/gdp/tests/test_mbigm.py index 9e82b1010f9..14a23160574 100644 --- a/pyomo/gdp/tests/test_mbigm.py +++ b/pyomo/gdp/tests/test_mbigm.py @@ -1019,39 +1019,34 @@ def test_calculate_Ms_infeasible_Disjunct(self): out.getvalue().strip(), ) - # We just fixed the infeasible by to False + # We just fixed the infeasible disjunct to False self.assertFalse(m.disjunction.disjuncts[0].active) self.assertTrue(m.disjunction.disjuncts[0].indicator_var.fixed) self.assertFalse(value(m.disjunction.disjuncts[0].indicator_var)) + # We didn't actually transform the infeasible disjunct + self.assertIsNone(m.disjunction.disjuncts[0].transformation_block) + # the remaining constraints are transformed correctly. cons = mbm.get_transformed_constraints(m.disjunction.disjuncts[1].constraint[1]) self.assertEqual(len(cons), 1) assertExpressionsEqual( self, cons[0].expr, - 21 + m.x - m.y - <= 0 * m.disjunction.disjuncts[0].binary_indicator_var - + 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, + 21 + m.x - m.y <= 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, ) cons = mbm.get_transformed_constraints(m.disjunction.disjuncts[2].constraint[1]) self.assertEqual(len(cons), 2) - print(cons[0].expr) - print(cons[1].expr) assertExpressionsEqual( self, cons[0].expr, - 0.0 * m.disjunction_disjuncts[0].binary_indicator_var - - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var - <= m.x - (m.y - 9), + -12.0 * m.disjunction_disjuncts[1].binary_indicator_var <= m.x - (m.y - 9), ) assertExpressionsEqual( self, cons[1].expr, - m.x - (m.y - 9) - <= 0.0 * m.disjunction_disjuncts[0].binary_indicator_var - - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var, + m.x - (m.y - 9) <= -12.0 * m.disjunction_disjuncts[1].binary_indicator_var, ) @unittest.skipUnless( diff --git a/pyomo/neos/__init__.py b/pyomo/neos/__init__.py index 7d18535e753..9f910f4a302 100644 --- a/pyomo/neos/__init__.py +++ b/pyomo/neos/__init__.py @@ -30,7 +30,7 @@ 'minos': 'SLC NLP solver', 'minto': 'MILP solver', 'mosek': 'Interior point NLP solver', - 'octeract': 'Deterministic global MINLP solver', + #'octeract': 'Deterministic global MINLP solver', 'ooqp': 'Convex QP solver', 'path': 'Nonlinear MCP solver', 'snopt': 'SQP NLP solver', diff --git a/pyomo/neos/tests/test_neos.py b/pyomo/neos/tests/test_neos.py index a4c4e9e6367..681856781be 100644 --- a/pyomo/neos/tests/test_neos.py +++ b/pyomo/neos/tests/test_neos.py @@ -79,6 +79,9 @@ def test_doc(self): doc = pyomo.neos.doc dockeys = set(doc.keys()) + # Octeract interface is disabled, see #3321 + amplsolvers.remove('octeract') + self.assertEqual(amplsolvers, dockeys) # gamssolvers = set(v[0].lower() for v in tmp if v[1]=='GAMS') @@ -149,8 +152,11 @@ def test_minto(self): def test_mosek(self): self._run('mosek') - def test_octeract(self): - self._run('octeract') + # [16 Jul 24] Octeract is erroring. We will disable the interface + # (and testing) until we have time to resolve #3321 + # + # def test_octeract(self): + # self._run('octeract') def test_ooqp(self): if self.sense == pyo.maximize: diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 6d084067511..029fe892b62 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -183,8 +183,6 @@ def to_expression(visitor, arg): return arg[1].to_expression(visitor) -_exit_node_handlers = {} - # # NEGATION handlers # @@ -199,11 +197,6 @@ def _handle_negation_ANY(visitor, node, arg): return arg -_exit_node_handlers[NegationExpression] = { - None: _handle_negation_ANY, - (_CONSTANT,): _handle_negation_constant, -} - # # PRODUCT handlers # @@ -272,16 +265,6 @@ def _handle_product_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[ProductExpression] = { - None: _handle_product_nonlinear, - (_CONSTANT, _CONSTANT): _handle_product_constant_constant, - (_CONSTANT, _LINEAR): _handle_product_constant_ANY, - (_CONSTANT, _GENERAL): _handle_product_constant_ANY, - (_LINEAR, _CONSTANT): _handle_product_ANY_constant, - (_GENERAL, _CONSTANT): _handle_product_ANY_constant, -} -_exit_node_handlers[MonomialTermExpression] = _exit_node_handlers[ProductExpression] - # # DIVISION handlers # @@ -302,13 +285,6 @@ def _handle_division_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[DivisionExpression] = { - None: _handle_division_nonlinear, - (_CONSTANT, _CONSTANT): _handle_division_constant_constant, - (_LINEAR, _CONSTANT): _handle_division_ANY_constant, - (_GENERAL, _CONSTANT): _handle_division_ANY_constant, -} - # # EXPONENTIATION handlers # @@ -345,13 +321,6 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[PowExpression] = { - None: _handle_pow_nonlinear, - (_CONSTANT, _CONSTANT): _handle_pow_constant_constant, - (_LINEAR, _CONSTANT): _handle_pow_ANY_constant, - (_GENERAL, _CONSTANT): _handle_pow_ANY_constant, -} - # # ABS and UNARY handlers # @@ -371,12 +340,6 @@ def _handle_unary_nonlinear(visitor, node, arg): return _GENERAL, ans -_exit_node_handlers[UnaryFunctionExpression] = { - None: _handle_unary_nonlinear, - (_CONSTANT,): _handle_unary_constant, -} -_exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] - # # NAMED EXPRESSION handlers # @@ -395,11 +358,6 @@ def _handle_named_ANY(visitor, node, arg1): return _type, arg1.duplicate() -_exit_node_handlers[Expression] = { - None: _handle_named_ANY, - (_CONSTANT,): _handle_named_constant, -} - # # EXPR_IF handlers # @@ -430,11 +388,6 @@ def _handle_expr_if_nonlinear(visitor, node, arg1, arg2, arg3): return _GENERAL, ans -_exit_node_handlers[Expr_ifExpression] = {None: _handle_expr_if_nonlinear} -for j in (_CONSTANT, _LINEAR, _GENERAL): - for k in (_CONSTANT, _LINEAR, _GENERAL): - _exit_node_handlers[Expr_ifExpression][_CONSTANT, j, k] = _handle_expr_if_const - # # Relational expression handlers # @@ -462,12 +415,6 @@ def _handle_equality_general(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[EqualityExpression] = { - None: _handle_equality_general, - (_CONSTANT, _CONSTANT): _handle_equality_const, -} - - def _handle_inequality_const(visitor, node, arg1, arg2): # It is exceptionally likely that if we get here, one of the # arguments is an InvalidNumber @@ -490,12 +437,6 @@ def _handle_inequality_general(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[InequalityExpression] = { - None: _handle_inequality_general, - (_CONSTANT, _CONSTANT): _handle_inequality_const, -} - - def _handle_ranged_const(visitor, node, arg1, arg2, arg3): # It is exceptionally likely that if we get here, one of the # arguments is an InvalidNumber @@ -523,10 +464,62 @@ def _handle_ranged_general(visitor, node, arg1, arg2, arg3): return _GENERAL, ans -_exit_node_handlers[RangedExpression] = { - None: _handle_ranged_general, - (_CONSTANT, _CONSTANT, _CONSTANT): _handle_ranged_const, -} +def define_exit_node_handlers(_exit_node_handlers=None): + if _exit_node_handlers is None: + _exit_node_handlers = {} + _exit_node_handlers[NegationExpression] = { + None: _handle_negation_ANY, + (_CONSTANT,): _handle_negation_constant, + } + _exit_node_handlers[ProductExpression] = { + None: _handle_product_nonlinear, + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, + (_CONSTANT, _LINEAR): _handle_product_constant_ANY, + (_CONSTANT, _GENERAL): _handle_product_constant_ANY, + (_LINEAR, _CONSTANT): _handle_product_ANY_constant, + (_GENERAL, _CONSTANT): _handle_product_ANY_constant, + } + _exit_node_handlers[MonomialTermExpression] = _exit_node_handlers[ProductExpression] + _exit_node_handlers[DivisionExpression] = { + None: _handle_division_nonlinear, + (_CONSTANT, _CONSTANT): _handle_division_constant_constant, + (_LINEAR, _CONSTANT): _handle_division_ANY_constant, + (_GENERAL, _CONSTANT): _handle_division_ANY_constant, + } + _exit_node_handlers[PowExpression] = { + None: _handle_pow_nonlinear, + (_CONSTANT, _CONSTANT): _handle_pow_constant_constant, + (_LINEAR, _CONSTANT): _handle_pow_ANY_constant, + (_GENERAL, _CONSTANT): _handle_pow_ANY_constant, + } + _exit_node_handlers[UnaryFunctionExpression] = { + None: _handle_unary_nonlinear, + (_CONSTANT,): _handle_unary_constant, + } + _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] + _exit_node_handlers[Expression] = { + None: _handle_named_ANY, + (_CONSTANT,): _handle_named_constant, + } + _exit_node_handlers[Expr_ifExpression] = {None: _handle_expr_if_nonlinear} + for j in (_CONSTANT, _LINEAR, _GENERAL): + for k in (_CONSTANT, _LINEAR, _GENERAL): + _exit_node_handlers[Expr_ifExpression][ + _CONSTANT, j, k + ] = _handle_expr_if_const + _exit_node_handlers[EqualityExpression] = { + None: _handle_equality_general, + (_CONSTANT, _CONSTANT): _handle_equality_const, + } + _exit_node_handlers[InequalityExpression] = { + None: _handle_inequality_general, + (_CONSTANT, _CONSTANT): _handle_inequality_const, + } + _exit_node_handlers[RangedExpression] = { + None: _handle_ranged_general, + (_CONSTANT, _CONSTANT, _CONSTANT): _handle_ranged_const, + } + return _exit_node_handlers class LinearBeforeChildDispatcher(BeforeChildDispatcher): @@ -728,9 +721,8 @@ def _initialize_exit_node_dispatcher(exit_handlers): class LinearRepnVisitor(StreamBasedExpressionVisitor): Result = LinearRepn - exit_node_handlers = _exit_node_handlers exit_node_dispatcher = ExitNodeDispatcher( - _initialize_exit_node_dispatcher(_exit_node_handlers) + _initialize_exit_node_dispatcher(define_exit_node_handlers()) ) expand_nonlinear_products = False max_exponential_expansion = 1 diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py new file mode 100644 index 00000000000..d1295b73e14 --- /dev/null +++ b/pyomo/repn/parameterized_linear.py @@ -0,0 +1,395 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import copy + +from pyomo.common.collections import ComponentSet +from pyomo.common.numeric_types import native_numeric_types +from pyomo.core import Var +from pyomo.core.expr.logical_expr import _flattened +from pyomo.core.expr.numeric_expr import ( + AbsExpression, + DivisionExpression, + LinearExpression, + MonomialTermExpression, + NegationExpression, + mutable_expression, + PowExpression, + ProductExpression, + SumExpression, + UnaryFunctionExpression, +) +from pyomo.repn.linear import ( + ExitNodeDispatcher, + _initialize_exit_node_dispatcher, + LinearBeforeChildDispatcher, + LinearRepn, + LinearRepnVisitor, +) +from pyomo.repn.util import ExprType +import pyomo.repn.linear as linear + + +_FIXED = ExprType.FIXED +_CONSTANT = ExprType.CONSTANT +_LINEAR = ExprType.LINEAR +_GENERAL = ExprType.GENERAL + + +def _merge_dict(dest_dict, mult, src_dict): + if mult.__class__ not in native_numeric_types or mult != 1: + for vid, coef in src_dict.items(): + if vid in dest_dict: + dest_dict[vid] += mult * coef + else: + dest_dict[vid] = mult * coef + else: + for vid, coef in src_dict.items(): + if vid in dest_dict: + dest_dict[vid] += coef + else: + dest_dict[vid] = coef + + +def to_expression(visitor, arg): + if arg[0] in (_CONSTANT, _FIXED): + return arg[1] + else: + return arg[1].to_expression(visitor) + + +class ParameterizedLinearRepn(LinearRepn): + def __str__(self): + return ( + f"ParameterizedLinearRepn(mult={self.multiplier}, const={self.constant}, " + f"linear={self.linear}, nonlinear={self.nonlinear})" + ) + + def walker_exitNode(self): + if self.nonlinear is not None: + return _GENERAL, self + elif self.linear: + return _LINEAR, self + elif self.constant.__class__ in native_numeric_types: + return _CONSTANT, self.multiplier * self.constant + else: + return _FIXED, self.multiplier * self.constant + + def to_expression(self, visitor): + if self.nonlinear is not None: + # We want to start with the nonlinear term (and use + # assignment) in case the term is a non-numeric node (like a + # relational expression) + ans = self.nonlinear + else: + ans = 0 + if self.linear: + var_map = visitor.var_map + with mutable_expression() as e: + for vid, coef in self.linear.items(): + if coef.__class__ not in native_numeric_types or coef: + e += coef * var_map[vid] + if e.nargs() > 1: + ans += e + elif e.nargs() == 1: + ans += e.arg(0) + if self.constant.__class__ not in native_numeric_types or self.constant: + ans += self.constant + if ( + self.multiplier.__class__ not in native_numeric_types + or self.multiplier != 1 + ): + ans *= self.multiplier + return ans + + def append(self, other): + """Append a child result from acceptChildResult + + Notes + ----- + This method assumes that the operator was "+". It is implemented + so that we can directly use a ParameterizedLinearRepn() as a `data` object in + the expression walker (thereby allowing us to use the default + implementation of acceptChildResult [which calls + `data.append()`] and avoid the function call for a custom + callback). + + """ + _type, other = other + if _type is _CONSTANT or _type is _FIXED: + self.constant += other + return + + mult = other.multiplier + try: + _mult = bool(mult) + if not _mult: + return + if mult == 1: + _mult = False + except: + _mult = True + + const = other.constant + try: + _const = bool(const) + except: + _const = True + + if _mult: + if _const: + self.constant += mult * const + if other.linear: + _merge_dict(self.linear, mult, other.linear) + if other.nonlinear is not None: + nl = mult * other.nonlinear + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl + else: + if _const: + self.constant += const + if other.linear: + _merge_dict(self.linear, 1, other.linear) + if other.nonlinear is not None: + nl = other.nonlinear + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl + + +class ParameterizedLinearBeforeChildDispatcher(LinearBeforeChildDispatcher): + def __init__(self): + super().__init__() + self[Var] = self._before_var + self[MonomialTermExpression] = self._before_monomial + self[LinearExpression] = self._before_linear + self[SumExpression] = self._before_general_expression + + @staticmethod + def _before_linear(visitor, child): + return True, None + + @staticmethod + def _before_monomial(visitor, child): + return True, None + + @staticmethod + def _before_general_expression(visitor, child): + return True, None + + @staticmethod + def _before_var(visitor, child): + _id = id(child) + if _id not in visitor.var_map: + if child.fixed: + return False, (_CONSTANT, visitor.check_constant(child.value, child)) + if child in visitor.wrt: + # pseudo-constant + # We aren't treating this Var as a Var for the purposes of this walker + return False, (_FIXED, child) + # This is a normal situation + ParameterizedLinearBeforeChildDispatcher._record_var(visitor, child) + ans = visitor.Result() + ans.linear[_id] = 1 + return False, (ExprType.LINEAR, ans) + + +_before_child_dispatcher = ParameterizedLinearBeforeChildDispatcher() + +# +# NEGATION handlers +# + + +def _handle_negation_pseudo_constant(visitor, node, arg): + return (_FIXED, -1 * arg[1]) + + +# +# PRODUCT handlers +# + + +def _handle_product_constant_constant(visitor, node, arg1, arg2): + # [ESJ 5/22/24]: Overriding this handler to exclude the deprecation path for + # 0 * nan. It doesn't need overridden when that deprecation path goes away. + return _CONSTANT, arg1[1] * arg2[1] + + +def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): + return _FIXED, arg1[1] * arg2[1] + + +# +# DIVISION handlers +# + + +def _handle_division_pseudo_constant_constant(visitor, node, arg1, arg2): + return _FIXED, arg1[1] / arg2[1] + + +def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): + arg1[1].multiplier = arg1[1].multiplier / arg2[1] + return arg1 + + +# +# EXPONENTIATION handlers +# + + +def _handle_pow_pseudo_constant_constant(visitor, node, arg1, arg2): + return _FIXED, to_expression(visitor, arg1) ** to_expression(visitor, arg2) + + +def _handle_pow_nonlinear(visitor, node, arg1, arg2): + # ESJ: We override this because we need our own to_expression implementation + # if pseudo constants are involved. + ans = visitor.Result() + ans.nonlinear = to_expression(visitor, arg1) ** to_expression(visitor, arg2) + return _GENERAL, ans + + +# +# ABS and UNARY handlers +# + + +def _handle_unary_pseudo_constant(visitor, node, arg): + # We override this because we can't blindly use apply_node_operation in this case + return _FIXED, node.create_node_with_local_data((to_expression(visitor, arg),)) + + +def define_exit_node_handlers(exit_node_handlers=None): + if exit_node_handlers is None: + exit_node_handlers = {} + linear.define_exit_node_handlers(exit_node_handlers) + + exit_node_handlers[NegationExpression].update( + {(_FIXED,): _handle_negation_pseudo_constant} + ) + + exit_node_handlers[ProductExpression].update( + { + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, + (_FIXED, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_product_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _LINEAR): linear._handle_product_constant_ANY, + (_LINEAR, _FIXED): linear._handle_product_ANY_constant, + (_FIXED, _GENERAL): linear._handle_product_constant_ANY, + (_GENERAL, _FIXED): linear._handle_product_ANY_constant, + } + ) + + exit_node_handlers[MonomialTermExpression].update( + exit_node_handlers[ProductExpression] + ) + + exit_node_handlers[DivisionExpression].update( + { + (_FIXED, _FIXED): _handle_division_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_division_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_division_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_division_ANY_pseudo_constant, + (_GENERAL, _FIXED): _handle_division_ANY_pseudo_constant, + } + ) + + exit_node_handlers[PowExpression].update( + { + (_FIXED, _FIXED): _handle_pow_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_pow_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_pow_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_pow_nonlinear, + (_FIXED, _LINEAR): _handle_pow_nonlinear, + (_GENERAL, _FIXED): _handle_pow_nonlinear, + (_FIXED, _GENERAL): _handle_pow_nonlinear, + } + ) + exit_node_handlers[UnaryFunctionExpression].update( + {(_FIXED,): _handle_unary_pseudo_constant} + ) + exit_node_handlers[AbsExpression] = exit_node_handlers[UnaryFunctionExpression] + + return exit_node_handlers + + +class ParameterizedLinearRepnVisitor(LinearRepnVisitor): + Result = ParameterizedLinearRepn + exit_node_dispatcher = ExitNodeDispatcher( + _initialize_exit_node_dispatcher(define_exit_node_handlers()) + ) + + def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): + super().__init__(subexpression_cache, var_map, var_order, sorter) + self.wrt = ComponentSet(_flattened(wrt)) + + def beforeChild(self, node, child, child_idx): + return _before_child_dispatcher[child.__class__](self, child) + + def _factor_multiplier_into_linear_terms(self, ans, mult): + linear = ans.linear + zeros = [] + for vid, coef in linear.items(): + if coef.__class__ not in native_numeric_types or coef: + linear[vid] = mult * coef + else: + zeros.append(vid) + for vid in zeros: + del linear[vid] + if ans.nonlinear is not None: + ans.nonlinear *= mult + if ans.constant.__class__ not in native_numeric_types or ans.constant: + ans.constant *= mult + ans.multiplier = 1 + + def finalizeResult(self, result): + ans = result[1] + if ans.__class__ is self.Result: + mult = ans.multiplier + if mult.__class__ not in native_numeric_types: + # mult is an expression--we should push it back into the other terms + self._factor_multiplier_into_linear_terms(ans, mult) + return ans + if mult == 1: + zeros = [ + (vid, coef) + for vid, coef in ans.linear.items() + if coef.__class__ in native_numeric_types and not coef + ] + for vid, coef in zeros: + del ans.linear[vid] + elif not mult: + # the multiplier has cleared out the entire expression. Check + # if this is suppressing a NaN because we can't clear everything + # out if it is + if ans.constant != ans.constant or any( + c != c for c in ans.linear.values() + ): + # There's a nan in here, so we distribute the 0 + self._factor_multiplier_into_linear_terms(ans, mult) + return ans + return self.Result() + else: + # mult not in {0, 1}: factor it into the constant, + # linear coefficients, and nonlinear term + self._factor_multiplier_into_linear_terms(ans, mult) + return ans + + ans = self.Result() + assert result[0] in (_CONSTANT, _FIXED) + ans.constant = result[1] + return ans diff --git a/pyomo/repn/plugins/nl_writer.py b/pyomo/repn/plugins/nl_writer.py index a8966e44f71..8fc82d21d30 100644 --- a/pyomo/repn/plugins/nl_writer.py +++ b/pyomo/repn/plugins/nl_writer.py @@ -1988,6 +1988,18 @@ def _record_named_expression_usage(self, named_exprs, src, comp_type): elif info[comp_type] != src: info[comp_type] = 0 + def _resolve_subexpression_args(self, nl, args): + final_args = [] + for arg in args: + if arg in self.var_id_to_nl_map: + final_args.append(self.var_id_to_nl_map[arg]) + else: + _nl, _ids, _ = self.subexpression_cache[arg][1].compile_repn( + self.visitor + ) + final_args.append(self._resolve_subexpression_args(_nl, _ids)) + return nl % tuple(final_args) + def _write_nl_expression(self, repn, include_const): # Note that repn.mult should always be 1 (the AMPLRepn was # compiled before this point). Omitting the assertion for @@ -2007,18 +2019,7 @@ def _write_nl_expression(self, repn, include_const): nl % tuple(map(self.var_id_to_nl_map.__getitem__, args)) ) except KeyError: - final_args = [] - for arg in args: - if arg in self.var_id_to_nl_map: - final_args.append(self.var_id_to_nl_map[arg]) - else: - _nl, _ids, _ = self.subexpression_cache[arg][1].compile_repn( - self.visitor - ) - final_args.append( - _nl % tuple(map(self.var_id_to_nl_map.__getitem__, _ids)) - ) - self.ostream.write(nl % tuple(final_args)) + self.ostream.write(self._resolve_subexpression_args(nl, args)) elif include_const: self.ostream.write(self.template.const % repn.const) diff --git a/pyomo/repn/quadratic.py b/pyomo/repn/quadratic.py index f6e0a43623d..a9c8b7bf2b5 100644 --- a/pyomo/repn/quadratic.py +++ b/pyomo/repn/quadratic.py @@ -157,17 +157,6 @@ def append(self, other): self.nonlinear += nl -_exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) - -# -# NEGATION -# -_exit_node_handlers[NegationExpression][(_QUADRATIC,)] = linear._handle_negation_ANY - - -# -# PRODUCT -# def _mul_linear_linear(varOrder, linear1, linear2): quadratic = {} for vid1, coef1 in linear1.items(): @@ -275,69 +264,73 @@ def _handle_product_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[ProductExpression].update( - { - None: _handle_product_nonlinear, - (_CONSTANT, _QUADRATIC): linear._handle_product_constant_ANY, - (_QUADRATIC, _CONSTANT): linear._handle_product_ANY_constant, - # Replace handler from the linear walker - (_LINEAR, _LINEAR): _handle_product_linear_linear, - } -) - -# -# DIVISION -# -_exit_node_handlers[DivisionExpression].update( - {(_QUADRATIC, _CONSTANT): linear._handle_division_ANY_constant} -) - - -# -# EXPONENTIATION -# -_exit_node_handlers[PowExpression].update( - {(_QUADRATIC, _CONSTANT): linear._handle_pow_ANY_constant} -) - -# -# ABS and UNARY handlers -# -# (no changes needed) - -# -# NAMED EXPRESSION handlers -# -# (no changes needed) - -# -# EXPR_IF handlers -# -# Note: it is easier to just recreate the entire data structure, rather -# than update it -_exit_node_handlers[Expr_ifExpression].update( - { - (_CONSTANT, i, _QUADRATIC): linear._handle_expr_if_const - for i in (_CONSTANT, _LINEAR, _QUADRATIC, _GENERAL) - } -) -_exit_node_handlers[Expr_ifExpression].update( - { - (_CONSTANT, _QUADRATIC, i): linear._handle_expr_if_const - for i in (_CONSTANT, _LINEAR, _GENERAL) - } -) - -# -# RELATIONAL handlers -# -# (no changes needed) +def define_exit_node_handlers(_exit_node_handlers=None): + if _exit_node_handlers is None: + _exit_node_handlers = {} + linear.define_exit_node_handlers(_exit_node_handlers) + # + # NEGATION + # + _exit_node_handlers[NegationExpression][(_QUADRATIC,)] = linear._handle_negation_ANY + # + # PRODUCT + # + _exit_node_handlers[ProductExpression].update( + { + None: _handle_product_nonlinear, + (_CONSTANT, _QUADRATIC): linear._handle_product_constant_ANY, + (_QUADRATIC, _CONSTANT): linear._handle_product_ANY_constant, + # Replace handler from the linear walker + (_LINEAR, _LINEAR): _handle_product_linear_linear, + } + ) + # + # DIVISION + # + _exit_node_handlers[DivisionExpression].update( + {(_QUADRATIC, _CONSTANT): linear._handle_division_ANY_constant} + ) + # + # EXPONENTIATION + # + _exit_node_handlers[PowExpression].update( + {(_QUADRATIC, _CONSTANT): linear._handle_pow_ANY_constant} + ) + # + # ABS and UNARY handlers + # + # (no changes needed) + # + # NAMED EXPRESSION handlers + # + # (no changes needed) + # + # EXPR_IF handlers + # + # Note: it is easier to just recreate the entire data structure, rather + # than update it + _exit_node_handlers[Expr_ifExpression].update( + { + (_CONSTANT, i, _QUADRATIC): linear._handle_expr_if_const + for i in (_CONSTANT, _LINEAR, _QUADRATIC, _GENERAL) + } + ) + _exit_node_handlers[Expr_ifExpression].update( + { + (_CONSTANT, _QUADRATIC, i): linear._handle_expr_if_const + for i in (_CONSTANT, _LINEAR, _GENERAL) + } + ) + # + # RELATIONAL handlers + # + # (no changes needed) + return _exit_node_handlers class QuadraticRepnVisitor(linear.LinearRepnVisitor): Result = QuadraticRepn - exit_node_handlers = _exit_node_handlers exit_node_dispatcher = linear.ExitNodeDispatcher( - linear._initialize_exit_node_dispatcher(_exit_node_handlers) + linear._initialize_exit_node_dispatcher(define_exit_node_handlers()) ) max_exponential_expansion = 2 diff --git a/pyomo/repn/tests/ampl/test_nlv2.py b/pyomo/repn/tests/ampl/test_nlv2.py index b6bb5f6c074..4d7b5d9ab6c 100644 --- a/pyomo/repn/tests/ampl/test_nlv2.py +++ b/pyomo/repn/tests/ampl/test_nlv2.py @@ -2703,3 +2703,72 @@ def test_presolve_check_invalid_monomial_constraints(self): r"\(fixed body value 5.0 outside bounds \[10, None\]\)\.", ): nl_writer.NLWriter().write(m, OUT, linear_presolve=True) + + def test_nested_external_expressions(self): + # This tests nested external functions in a single expression + DLL = find_GSL() + if not DLL: + self.skipTest("Could not find the amplgsl.dll library") + + m = ConcreteModel() + m.hypot = ExternalFunction(library=DLL, function="gsl_hypot") + m.p = Param(initialize=1, mutable=True) + m.x = Var(bounds=(None, 3)) + m.y = Var(bounds=(3, None)) + m.z = Var(initialize=1) + m.o = Objective(expr=m.z**2 * m.hypot(m.z, m.hypot(m.x, m.y)) ** 2) + m.c = Constraint(expr=m.x == m.y) + + OUT = io.StringIO() + nl_writer.NLWriter().write( + m, OUT, symbolic_solver_labels=True, linear_presolve=False + ) + self.assertEqual( + *nl_diff( + """g3 1 1 0 #problem unknown + 3 1 1 0 1 #vars, constraints, objectives, ranges, eqns + 0 1 0 0 0 0 #nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb + 0 0 #network constraints: nonlinear, linear + 0 3 0 #nonlinear vars in constraints, objectives, both + 0 1 0 1 #linear network variables; functions; arith, flags + 0 0 0 0 0 #discrete variables: binary, integer, nonlinear (b,c,o) + 2 3 #nonzeros in Jacobian, obj. gradient + 1 1 #max name lengths: constraints, variables + 0 0 0 0 0 #common exprs: b,c,o,c1,o1 +F0 1 -1 gsl_hypot +C0 #c +n0 +O0 0 #o +o2 #* +o5 #^ +v0 #z +n2 +o5 #^ +f0 2 #hypot +v0 #z +f0 2 #hypot +v1 #x +v2 #y +n2 +x1 #initial guess +0 1 #z +r #1 ranges (rhs's) +4 0 #c +b #3 bounds (on variables) +3 #z +1 3 #x +2 3 #y +k2 #intermediate Jacobian column lengths +0 +1 +J0 2 #c +1 1 +2 -1 +G0 3 #o +0 0 +1 0 +2 0 +""", + OUT.getvalue(), + ) + ) diff --git a/pyomo/repn/tests/test_linear.py b/pyomo/repn/tests/test_linear.py index 861fecc7888..d88ddf96baa 100644 --- a/pyomo/repn/tests/test_linear.py +++ b/pyomo/repn/tests/test_linear.py @@ -1659,7 +1659,7 @@ def test_zero_elimination(self): self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 0) self.assertEqual(repn.linear, {}) - self.assertEqual(repn.nonlinear, None) + self.assertIsNone(repn.nonlinear) m.p = Param(mutable=True, within=Any, initialize=None) e = m.p * m.x[0] + m.p * m.x[1] * m.x[2] + m.p * log(m.x[3]) diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py new file mode 100644 index 00000000000..624f8390d16 --- /dev/null +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -0,0 +1,552 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.common.log import LoggingIntercept +import pyomo.common.unittest as unittest +from pyomo.core.expr.compare import assertExpressionsEqual +from pyomo.environ import Any, Binary, ConcreteModel, log, Param, Var +from pyomo.repn.parameterized_linear import ParameterizedLinearRepnVisitor +from pyomo.repn.tests.test_linear import VisitorConfig +from pyomo.repn.util import InvalidNumber + + +class TestParameterizedLinearRepnVisitor(unittest.TestCase): + def make_model(self): + m = ConcreteModel() + m.x = Var(bounds=(0, 45)) + m.y = Var(domain=Binary) + m.z = Var() + + return m + + def test_walk_sum(self): + m = self.make_model() + e = m.x + m.y + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 1) + self.assertIs(repn.constant, m.y) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.y) + + def test_walk_triple_sum(self): + m = self.make_model() + e = m.x + m.z * m.y + m.z + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.x), repn.linear) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 1) + self.assertIs(repn.linear[id(m.y)], m.z) + self.assertIs(repn.constant, m.z) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.z * m.y + m.z) + + def test_sum_two_of_the_same(self): + # This hits the mult == 1 and vid in dest_dict case in _merge_dict + m = self.make_model() + e = m.x + m.x + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 2) + self.assertEqual(repn.constant, 0) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), 2 * m.x) + + def test_sum_with_mult_0(self): + m = self.make_model() + e = 0 * m.x + m.x - m.y + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 1) + assertExpressionsEqual(self, repn.constant, -m.y) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x - m.y) + + def test_sum_nonlinear_to_linear(self): + m = self.make_model() + e = m.y * m.x**2 + m.y * m.x - 3 + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + assertExpressionsEqual(self, repn.nonlinear, m.y * m.x**2) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertIs(repn.linear[id(m.x)], m.y) + self.assertEqual(repn.constant, -3) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, repn.to_expression(visitor), m.y * m.x**2 + m.y * m.x - 3 + ) + + def test_sum_nonlinear_to_nonlinear(self): + m = self.make_model() + e = m.x**3 + 3 + m.x**2 + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + assertExpressionsEqual(self, repn.nonlinear, m.x**3 + m.x**2) + self.assertEqual(repn.constant, 3) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x**3 + m.x**2 + 3) + + def test_sum_to_linear_expr(self): + m = self.make_model() + e = m.x + m.y * (m.x + 5) + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + m.y) + assertExpressionsEqual(self, repn.constant, m.y * 5) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, repn.to_expression(visitor), (1 + m.y) * m.x + m.y * 5 + ) + + def test_bilinear_term(self): + m = self.make_model() + e = m.x * m.y + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertIs(repn.linear[id(m.x)], m.y) + self.assertEqual(repn.constant, 0) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x) + + def test_distributed_bilinear_term(self): + m = self.make_model() + e = m.y * (m.x + 7) + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertIs(repn.linear[id(m.x)], m.y) + assertExpressionsEqual(self, repn.constant, m.y * 7) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x + m.y * 7) + + def test_monomial(self): + m = self.make_model() + e = 45 * m.y + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x, m.z]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 45) + self.assertEqual(repn.constant, 0) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), 45 * m.y) + + def test_constant(self): + m = self.make_model() + e = 45 * m.y + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 0) + assertExpressionsEqual(self, repn.constant, 45 * m.y) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), 45 * m.y) + + def test_fixed_var(self): + m = self.make_model() + m.x.fix(42) + e = (m.y**2) * (m.x + m.x**2) + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 0) + assertExpressionsEqual(self, repn.constant, (m.y**2) * 1806) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), (m.y**2) * 1806) + + def test_nonlinear(self): + m = self.make_model() + e = (m.y * log(m.x)) * (m.y + 2) / m.x + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) + + repn = visitor.walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.nonlinear, log(m.x) * (m.y * (m.y + 2)) / m.x) + assertExpressionsEqual( + self, repn.to_expression(visitor), log(m.x) * (m.y * (m.y + 2)) / m.x + ) + + def test_finalize(self): + m = self.make_model() + m.w = Var() + + e = m.x + 2 * m.w**2 * m.y - m.x - m.w * m.z + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.w]).walk_expression(e) + self.assertEqual(cfg.subexpr, {}) + self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) + self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.y), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.y)], 2 * m.w**2) + self.assertIn(id(m.z), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.z)], -m.w) + self.assertEqual(repn.nonlinear, None) + + e *= 5 + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.w]).walk_expression(e) + self.assertEqual(cfg.subexpr, {}) + self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) + self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.y), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * (2 * m.w**2)) + self.assertIn(id(m.z), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.z)], -5 * m.w) + self.assertEqual(repn.nonlinear, None) + + e = 5 * (m.w * m.y + m.z**2 + 3 * m.w * m.y**3) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.w]).walk_expression(e) + self.assertEqual(cfg.subexpr, {}) + self.assertEqual(cfg.var_map, {id(m.y): m.y, id(m.z): m.z}) + self.assertEqual(cfg.var_order, {id(m.y): 0, id(m.z): 1}) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * m.w) + assertExpressionsEqual(self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5) + + def test_ANY_over_constant_division(self): + m = ConcreteModel() + m.p = Param(mutable=True, initialize=2, domain=Any) + m.x = Var() + m.z = Var() + m.y = Var() + # We will use the fixed value regardless of the fact that we aren't + # treating this as a Var. + m.y.fix(1) + + expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression( + expr + ) + + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, 1 + m.z) + self.assertEqual(len(repn.linear), 1) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z) + self.assertEqual(repn.nonlinear, None) + + def test_errors_propagate_nan(self): + m = ConcreteModel() + m.p = Param(mutable=True, initialize=0, domain=Any) + m.x = Var() + m.z = Var() + m.y = Var() + m.y.fix(1) + + expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y + cfg = VisitorConfig() + with LoggingIntercept() as LOG: + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression( + expr + ) + self.assertEqual( + LOG.getvalue(), + "Exception encountered evaluating expression 'div(3*z, 0)'\n" + "\tmessage: division by zero\n" + "\texpression: 3*z*x/p\n", + ) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, 1 + m.z) + self.assertEqual(len(repn.linear), 1) + self.assertIsInstance(repn.linear[id(m.x)], InvalidNumber) + assertExpressionsEqual(self, repn.linear[id(m.x)].value, 1 + float('nan')) + self.assertEqual(repn.nonlinear, None) + + m.y.fix(None) + expr = m.z * log(m.y) + 3 + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression( + expr + ) + self.assertEqual(repn.multiplier, 1) + self.assertIsInstance(repn.constant, InvalidNumber) + assertExpressionsEqual(self, repn.constant.value, float('nan') * m.z + 3) + self.assertEqual(repn.linear, {}) + self.assertEqual(repn.nonlinear, None) + + def test_negation_constant(self): + m = self.make_model() + e = -(m.y * m.z + 17) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, -1 * (m.y * m.z + 17)) + self.assertIsNone(repn.nonlinear) + + def test_product_nonlinear(self): + m = self.make_model() + e = (m.x**2) * (log(m.y) * m.z**4) * m.y + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual( + self, repn.nonlinear, (m.x**2) * (m.z**4 * log(m.y)) * m.y + ) + + def test_division_pseudo_constant_constant(self): + m = self.make_model() + e = m.x / 4 + m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 1) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, m.x / 4) + self.assertIsNone(repn.nonlinear) + + e = 4 / m.x + m.y + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 1) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, 4 / m.x) + self.assertIsNone(repn.nonlinear) + + e = m.z / m.x + m.y + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x, m.z]).walk_expression(e) + + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 1) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, m.z / m.x) + self.assertIsNone(repn.nonlinear) + + def test_division_ANY_pseudo_constant(self): + m = self.make_model() + e = (m.x + 3 * m.z) / m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.x), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 / m.y) + self.assertIn(id(m.z), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.z)], (1 / m.y) * 3) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertIsNone(repn.nonlinear) + + def test_duplicate(self): + m = self.make_model() + e = (1 + m.x) ** 2 + m.y + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]) + visitor.max_exponential_expansion = 2 + repn = visitor.walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIs(repn.constant, m.y) + assertExpressionsEqual(self, repn.nonlinear, (m.x + 1) * (m.x + 1)) + + def test_pow_ANY_pseudo_constant(self): + m = self.make_model() + e = (m.x**2 + 3 * m.z) ** m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, (m.x**2 + 3 * m.z) ** m.y) + + def test_pow_pseudo_constant_ANY(self): + m = self.make_model() + e = m.y ** (m.x**2 + 3 * m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, m.y ** (m.x**2 + 3 * m.z)) + + def test_pow_linear_pseudo_constant(self): + m = self.make_model() + e = (m.x + 3 * m.z) ** m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, (m.x + 3 * m.z) ** m.y) + + def test_pow_pseudo_constant_linear(self): + m = self.make_model() + e = m.y ** (m.x + 3 * m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, m.y ** (m.x + 3 * m.z)) + + def test_0_mult(self): + m = self.make_model() + m.p = Var() + m.p.fix(0) + e = m.p * (m.y**2 + m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertEqual(repn.constant, 0) + + def test_0_mult_nan(self): + m = self.make_model() + m.p = Param(initialize=0, mutable=True) + m.y.domain = Any + m.y.fix(float('nan')) + e = m.p * (m.y**2 + m.x) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertIsInstance(repn.constant, InvalidNumber) + assertExpressionsEqual(self, repn.constant.value, 0 * (float('nan') + m.x)) + + def test_0_mult_nan_param(self): + m = self.make_model() + m.p = Param(initialize=0, mutable=True) + m.y.fix(float('nan')) + e = m.p * (m.y**2) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertIsInstance(repn.constant, InvalidNumber) + assertExpressionsEqual(self, repn.constant.value, 0 * float('nan')) + + def test_0_mult_linear_with_nan(self): + m = self.make_model() + m.p = Param(initialize=0, mutable=True) + m.x.domain = Any + m.x.fix(float('nan')) + e = m.p * (3 * m.x * m.y + m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.y), repn.linear) + self.assertIsInstance(repn.linear[id(m.y)], InvalidNumber) + assertExpressionsEqual(self, repn.linear[id(m.y)].value, 0 * 3 * float('nan')) + self.assertIn(id(m.z), repn.linear) + self.assertEqual(repn.linear[id(m.z)], 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertEqual(repn.constant, 0) diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index 8d902d0f99a..32ec99dac0f 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -67,6 +67,7 @@ class ExprType(enum.IntEnum): CONSTANT = 0 + FIXED = 5 MONOMIAL = 10 LINEAR = 20 QUADRATIC = 30 @@ -378,18 +379,16 @@ class ExitNodeDispatcher(collections.defaultdict): `exitNode` callback This dispatcher implements a specialization of :py:`defaultdict` - that supports automatic type registration. Any missing types will - return the :py:meth:`register_dispatcher` method, which (when called - as a callback) will interrogate the type, identify the appropriate - callback, add the callback to the dict, and return the result of - calling the callback. As the callback is added to the dict, no type - will incur the overhead of `register_dispatcher` more than once. + that supports automatic type registration. As the identified + callback is added to the dict, no type will incur the overhead of + `register_dispatcher` more than once. Note that in this case, the client is expected to register all non-NPV expression types. The auto-registration is designed to only handle two cases: - Auto-detection of user-defined Named Expression types - Automatic mappimg of NPV expressions to their equivalent non-NPV handlers + - Automatic registration of derived expression types """ diff --git a/pyomo/solvers/plugins/solvers/ASL.py b/pyomo/solvers/plugins/solvers/ASL.py index bb8174a013e..7acd59936b1 100644 --- a/pyomo/solvers/plugins/solvers/ASL.py +++ b/pyomo/solvers/plugins/solvers/ASL.py @@ -101,7 +101,8 @@ def _get_version(self): timeout=5, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True, + text=True, + errors='ignore', ) ver = _extract_version(results.stdout) if ver is None: diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index f2cfe279fdc..d7b09e29fde 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -629,6 +629,7 @@ def _uploadMpsFile(self, s, unique): data=mpscsv_table_name, casOut={"name": mpsdata_table_name, "replace": True}, format="FREE", + maxLength=256, ) # Delete the table we don't need anymore @@ -641,6 +642,7 @@ def _uploadMpsFile(self, s, unique): mpsFileString=mps_file.read(), casout={"name": mpsdata_table_name, "replace": True}, format="FREE", + maxLength=256, ) return mpsdata_table_name @@ -716,7 +718,7 @@ def _apply_solver(self): action = "solveMilp" if self._has_integer_variables() else "solveLp" # Get a unique identifier, always use the same with different prefixes - unique = uuid4().hex[:16] + unique = uuid.uuid4().hex[:16] # Creat the output stream, we want to print to a log string as well as to the console self._log = StringIO()