Merge pull request Pyomo#3088 from michaelbynum/ginac

Interface to GiNaC for Simplification

.github/workflows/test_branches.yml

@@ -181,7 +181,7 @@ jobs:
         # Notes:
         #  - install glpk
         #  - pyodbc needs: gcc pkg-config unixodbc freetds
-        for pkg in bash pkg-config unixodbc freetds glpk; do
+        for pkg in bash pkg-config unixodbc freetds glpk ginac; do
             brew list $pkg || brew install $pkg
         done
 
@@ -193,7 +193,8 @@ jobs:
         #  - install glpk
         #  - ipopt needs: libopenblas-dev gfortran liblapack-dev
         sudo apt-get -o Dir::Cache=${GITHUB_WORKSPACE}/cache/os \
-            install libopenblas-dev gfortran liblapack-dev glpk-utils
+            install libopenblas-dev gfortran liblapack-dev glpk-utils \
+            libginac-dev
         sudo chmod -R 777 ${GITHUB_WORKSPACE}/cache/os
 
     - name: Update Windows
@@ -264,7 +265,7 @@ jobs:
         if test -z "${{matrix.slim}}"; then
             python -m pip install --cache-dir cache/pip cplex docplex \
                 || echo "WARNING: CPLEX Community Edition is not available"
-            python -m pip install --cache-dir cache/pip gurobipy==10.0.3\
+            python -m pip install --cache-dir cache/pip gurobipy==10.0.3 \
                 || echo "WARNING: Gurobi is not available"
             python -m pip install --cache-dir cache/pip xpress \
                 || echo "WARNING: Xpress Community Edition is not available"
@@ -339,7 +340,7 @@ jobs:
         echo "*** Install Pyomo dependencies ***"
         # Note: this will fail the build if any installation fails (or
         # possibly if it outputs messages to stderr)
-        conda install --update-deps -y $CONDA_DEPENDENCIES
+        conda install --update-deps -q -y $CONDA_DEPENDENCIES
         if test -z "${{matrix.slim}}"; then
             PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g')
             echo "Installing for $PYVER"

.github/workflows/test_pr_and_main.yml

@@ -218,7 +218,7 @@ jobs:
         # Notes:
         #  - install glpk
         #  - pyodbc needs: gcc pkg-config unixodbc freetds
-        for pkg in bash pkg-config unixodbc freetds glpk; do
+        for pkg in bash pkg-config unixodbc freetds glpk ginac; do
             brew list $pkg || brew install $pkg
         done
 
@@ -230,7 +230,8 @@ jobs:
         #  - install glpk
         #  - ipopt needs: libopenblas-dev gfortran liblapack-dev
         sudo apt-get -o Dir::Cache=${GITHUB_WORKSPACE}/cache/os \
-            install libopenblas-dev gfortran liblapack-dev glpk-utils
+            install libopenblas-dev gfortran liblapack-dev glpk-utils \
+            libginac-dev
         sudo chmod -R 777 ${GITHUB_WORKSPACE}/cache/os
 
     - name: Update Windows
@@ -372,6 +373,7 @@ jobs:
                 CONDA_DEPENDENCIES="$CONDA_DEPENDENCIES $PKG"
             fi
         done
+        echo ""
         echo "*** Install Pyomo dependencies ***"
         # Note: this will fail the build if any installation fails (or
         # possibly if it outputs messages to stderr)

.jenkins.sh

@@ -43,9 +43,6 @@ fi
 if test -z "$SLIM"; then
     export VENV_SYSTEM_PACKAGES='--system-site-packages'
 fi
-if test ! -z "$CATEGORY"; then
-    export PY_CAT="-m $CATEGORY"
-fi
 
 if test "$WORKSPACE" != "`pwd`"; then
     echo "ERROR: pwd is not WORKSPACE"
@@ -122,10 +119,23 @@ if test -z "$MODE" -o "$MODE" == setup; then
     echo "PYOMO_CONFIG_DIR=$PYOMO_CONFIG_DIR"
     echo ""
 
+    # Call Pyomo build scripts to build TPLs that would normally be
+    # skipped by the pyomo download-extensions / build-extensions
+    # actions below
+    if [[ " $CATEGORY " == *" builders "* ]]; then
+        echo ""
+        echo "Running local build scripts..."
+        echo ""
+        set -x
+        python pyomo/contrib/simplification/build.py --build-deps || exit 1
+        set +x
+    fi
+
     # Use Pyomo to download & compile binary extensions
     i=0
     while /bin/true; do
         i=$[$i+1]
+        echo ""
         echo "Downloading pyomo extensions (attempt $i)"
         pyomo download-extensions $PYOMO_DOWNLOAD_ARGS
         if test $? == 0; then
@@ -178,7 +188,7 @@ if test -z "$MODE" -o "$MODE" == test; then
     python -m pytest -v \
         -W ignore::Warning \
         --junitxml="TEST-pyomo.xml" \
-        $PY_CAT $TEST_SUITES $PYTEST_EXTRA_ARGS
+        -m "$CATEGORY" $TEST_SUITES $PYTEST_EXTRA_ARGS
 
     # Combine the coverage results and upload
     if test -z "$DISABLE_COVERAGE"; then

conftest.py

@@ -11,6 +11,22 @@
 
 import pytest
 
+_implicit_markers = {'default'}
+_extended_implicit_markers = _implicit_markers.union({'solver'})
+
+
+def pytest_collection_modifyitems(items):
+    """
+    This method will mark any unmarked tests with the implicit marker ('default')
+
+    """
+    for item in items:
+        try:
+            next(item.iter_markers())
+        except StopIteration:
+            for marker in _implicit_markers:
+                item.add_marker(getattr(pytest.mark, marker))
+
 
 def pytest_runtest_setup(item):
     """
@@ -32,23 +48,20 @@ def pytest_runtest_setup(item):
     the default mode; but if solver tests are also marked with an explicit
     category (e.g., "expensive"), we will skip them.
     """
-    marker = item.iter_markers()
     solvernames = [mark.args[0] for mark in item.iter_markers(name="solver")]
     solveroption = item.config.getoption("--solver")
     markeroption = item.config.getoption("-m")
-    implicit_markers = ['default']
-    extended_implicit_markers = implicit_markers + ['solver']
-    item_markers = set(mark.name for mark in marker)
+    item_markers = set(mark.name for mark in item.iter_markers())
     if solveroption:
         if solveroption not in solvernames:
             pytest.skip("SKIPPED: Test not marked {!r}".format(solveroption))
             return
     elif markeroption:
         return
     elif item_markers:
-        if not set(implicit_markers).issubset(
-            item_markers
-        ) and not item_markers.issubset(set(extended_implicit_markers)):
+        if not _implicit_markers.issubset(item_markers) and not item_markers.issubset(
+            _extended_implicit_markers
+        ):
             pytest.skip('SKIPPED: Only running default, solver, and unmarked tests.')
 
 

pyomo/common/download.py

@@ -29,6 +29,7 @@
 urllib_error = attempt_import('urllib.error')[0]
 ssl = attempt_import('ssl')[0]
 zipfile = attempt_import('zipfile')[0]
+tarfile = attempt_import('tarfile')[0]
 gzip = attempt_import('gzip')[0]
 distro, distro_available = attempt_import('distro')
 
@@ -371,7 +372,7 @@ def get_zip_archive(self, url, dirOffset=0):
         # Simple sanity checks
         for info in zip_file.infolist():
             f = info.filename
-            if f[0] in '\\/' or '..' in f:
+            if f[0] in '\\/' or '..' in f or os.path.isabs(f):
                 logger.error(
                     "malformed (potentially insecure) filename (%s) "
                     "found in zip archive.  Skipping file." % (f,)
@@ -387,6 +388,61 @@ def get_zip_archive(self, url, dirOffset=0):
             info.filename = target[-1] + '/' if f[-1] == '/' else target[-1]
             zip_file.extract(f, os.path.join(self._fname, *tuple(target[dirOffset:-1])))
 
+    def get_tar_archive(self, url, dirOffset=0):
+        if self._fname is None:
+            raise DeveloperError(
+                "target file name has not been initialized "
+                "with set_destination_filename"
+            )
+        if os.path.exists(self._fname) and not os.path.isdir(self._fname):
+            raise RuntimeError(
+                "Target directory (%s) exists, but is not a directory" % (self._fname,)
+            )
+
+        def filter_fcn(info):
+            # this mocks up the `tarfile` filter introduced in Python
+            # 3.12 and backported to later releases of Python (e.g.,
+            # 3.8.17, 3.9.17, 3.10.12, and 3.11.4)
+            f = info.name
+            if os.path.isabs(f) or '..' in f or f.startswith(('/', os.sep)):
+                logger.error(
+                    "malformed or potentially insecure filename (%s).  "
+                    "Skipping file." % (f,)
+                )
+                return False
+            target = self._splitpath(f)
+            if len(target) <= dirOffset:
+                if not info.isdir():
+                    logger.warning(
+                        "Skipping file (%s) in tar archive due to dirOffset." % (f,)
+                    )
+                return False
+            info.name = f = '/'.join(target[dirOffset:])
+            target = os.path.realpath(os.path.join(dest, f))
+            try:
+                if os.path.commonpath([target, dest]) != dest:
+                    logger.error(
+                        "potentially insecure filename (%s) resolves outside target "
+                        "directory.  Skipping file." % (f,)
+                    )
+                    return False
+            except ValueError:
+                # commonpath() will raise ValueError for paths that
+                # don't have anything in common (notably, when files are
+                # on different drives on Windows)
+                logger.error(
+                    "potentially insecure filename (%s) resolves outside target "
+                    "directory.  Skipping file." % (f,)
+                )
+                return False
+            # Strip high bits & group/other write bits
+            info.mode &= 0o755
+            return True
+
+        with tarfile.open(fileobj=io.BytesIO(self.retrieve_url(url))) as TAR:
+            dest = os.path.realpath(self._fname)
+            TAR.extractall(dest, filter(filter_fcn, TAR.getmembers()))
+
     def get_gzipped_binary_file(self, url):
         if self._fname is None:
             raise DeveloperError(

pyomo/common/enums.py

@@ -17,6 +17,7 @@
 .. autosummary::
 
    ExtendedEnumType
+   NamedIntEnum
 
 Standard Enums:
 
@@ -130,7 +131,21 @@ def __new__(metacls, cls, bases, classdict, **kwds):
         return super().__new__(metacls, cls, bases, classdict, **kwds)
 
 
-class ObjectiveSense(enum.IntEnum):
+class NamedIntEnum(enum.IntEnum):
+    """An extended version of :py:class:`enum.IntEnum` that supports
+    creating members by name as well as value.
+
+    """
+
+    @classmethod
+    def _missing_(cls, value):
+        for member in cls:
+            if member.name == value:
+                return member
+        return None
+
+
+class ObjectiveSense(NamedIntEnum):
     """Flag indicating if an objective is minimizing (1) or maximizing (-1).
 
     While the numeric values are arbitrary, there are parts of Pyomo
@@ -150,13 +165,6 @@ class ObjectiveSense(enum.IntEnum):
     def __str__(self):
         return self.name
 
-    @classmethod
-    def _missing_(cls, value):
-        for member in cls:
-            if member.name == value:
-                return member
-        return None
-
 
 minimize = ObjectiveSense.minimize
 maximize = ObjectiveSense.maximize

pyomo/common/fileutils.py

@@ -38,6 +38,7 @@
 import os
 import platform
 import importlib.util
+import subprocess
 import sys
 
 from . import envvar
@@ -375,9 +376,27 @@ def find_library(libname, cwd=True, include_PATH=True, pathlist=None):
     if libname_base.startswith('lib') and _system() != 'windows':
         libname_base = libname_base[3:]
     if ext.lower().startswith(('.so', '.dll', '.dylib')):
-        return ctypes.util.find_library(libname_base)
+        lib = ctypes.util.find_library(libname_base)
     else:
-        return ctypes.util.find_library(libname)
+        lib = ctypes.util.find_library(libname)
+    if lib and os.path.sep not in lib:
+        # work around https://github.com/python/cpython/issues/65241,
+        # where python does not return the absolute path on *nix
+        try:
+            libname = lib + ' '
+            with subprocess.Popen(
+                ['/sbin/ldconfig', '-p'],
+                stdin=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                env={'LC_ALL': 'C', 'LANG': 'C'},
+            ) as p:
+                for line in os.fsdecode(p.stdout.read()).splitlines():
+                    if line.lstrip().startswith(libname):
+                        return os.path.realpath(line.split()[-1])
+        except:
+            pass
+    return lib
 
 
 def find_executable(exename, cwd=True, include_PATH=True, pathlist=None):