From 5eeffde0b36302a6ab073bd5e3e6a5de2b6f6655 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 23 Jun 2023 09:42:08 -0400 Subject: [PATCH 001/150] Initial version of the SAS solver interfaces and unit tests. --- pyomo/solvers/plugins/solvers/SAS.py | 700 ++++++++++++++++++++++ pyomo/solvers/plugins/solvers/__init__.py | 1 + pyomo/solvers/tests/checks/test_SAS.py | 462 ++++++++++++++ 3 files changed, 1163 insertions(+) create mode 100644 pyomo/solvers/plugins/solvers/SAS.py create mode 100644 pyomo/solvers/tests/checks/test_SAS.py diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py new file mode 100644 index 00000000000..7f50b7a2970 --- /dev/null +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -0,0 +1,700 @@ +__all__ = ['SAS'] + +import logging +import sys +import os + +from io import StringIO +from abc import ABC, abstractmethod +from contextlib import redirect_stdout + +from pyomo.opt.base import ProblemFormat, ResultsFormat, OptSolver +from pyomo.opt.base.solvers import SolverFactory +from pyomo.common.collections import Bunch +from pyomo.opt.results import ( + SolverResults, + SolverStatus, + TerminationCondition, + SolutionStatus, + ProblemSense, +) +from pyomo.common.tempfiles import TempfileManager +from pyomo.core.base import Var +from pyomo.core.base.block import _BlockData +from pyomo.core.kernel.block import IBlock + + +logger = logging.getLogger('pyomo.solvers') + + +STATUS_TO_SOLVERSTATUS = { + "OK": SolverStatus.ok, + "SYNTAX_ERROR": SolverStatus.error, + "DATA_ERROR": SolverStatus.error, + "OUT_OF_MEMORY": SolverStatus.aborted, + "IO_ERROR": SolverStatus.error, + "ERROR": SolverStatus.error, +} + +# This combines all status codes from OPTLP/solvelp and OPTMILP/solvemilp +SOLSTATUS_TO_TERMINATIONCOND = { + "OPTIMAL": TerminationCondition.optimal, + "OPTIMAL_AGAP": TerminationCondition.optimal, + "OPTIMAL_RGAP": TerminationCondition.optimal, + "OPTIMAL_COND": TerminationCondition.optimal, + "TARGET": TerminationCondition.optimal, + "CONDITIONAL_OPTIMAL": TerminationCondition.optimal, + "FEASIBLE": TerminationCondition.feasible, + "INFEASIBLE": TerminationCondition.infeasible, + "UNBOUNDED": TerminationCondition.unbounded, + "INFEASIBLE_OR_UNBOUNDED": TerminationCondition.infeasibleOrUnbounded, + "SOLUTION_LIM": TerminationCondition.maxEvaluations, + "NODE_LIM_SOL": TerminationCondition.maxEvaluations, + "NODE_LIM_NOSOL": TerminationCondition.maxEvaluations, + "ITERATION_LIMIT_REACHED": TerminationCondition.maxIterations, + "TIME_LIM_SOL": TerminationCondition.maxTimeLimit, + "TIME_LIM_NOSOL": TerminationCondition.maxTimeLimit, + "TIME_LIMIT_REACHED": TerminationCondition.maxTimeLimit, + "ABORTED": TerminationCondition.userInterrupt, + "ABORT_SOL": TerminationCondition.userInterrupt, + "ABORT_NOSOL": TerminationCondition.userInterrupt, + "OUTMEM_SOL": TerminationCondition.solverFailure, + "OUTMEM_NOSOL": TerminationCondition.solverFailure, + "FAILED": TerminationCondition.solverFailure, + "FAIL_SOL": TerminationCondition.solverFailure, + "FAIL_NOSOL": TerminationCondition.solverFailure, +} + + +SOLSTATUS_TO_MESSAGE = { + "OPTIMAL": "The solution is optimal.", + "OPTIMAL_AGAP": "The solution is optimal within the absolute gap specified by the ABSOBJGAP= option.", + "OPTIMAL_RGAP": "The solution is optimal within the relative gap specified by the RELOBJGAP= option.", + "OPTIMAL_COND": "The solution is optimal, but some infeasibilities (primal, bound, or integer) exceed tolerances due to scaling or choice of a small INTTOL= value.", + "TARGET": "The solution is not worse than the target specified by the TARGET= option.", + "CONDITIONAL_OPTIMAL": "The solution is optimal, but some infeasibilities (primal, dual or bound) exceed tolerances due to scaling or preprocessing.", + "FEASIBLE": "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", + "INFEASIBLE": "The problem is infeasible.", + "UNBOUNDED": "The problem is unbounded.", + "INFEASIBLE_OR_UNBOUNDED": "The problem is infeasible or unbounded.", + "SOLUTION_LIM": "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", + "NODE_LIM_SOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", + "NODE_LIM_NOSOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and did not find a solution.", + "ITERATION_LIMIT_REACHED": "The maximum allowable number of iterations was reached.", + "TIME_LIM_SOL": "The solver reached the execution time limit specified by the MAXTIME= option and found a solution.", + "TIME_LIM_NOSOL": "The solver reached the execution time limit specified by the MAXTIME= option and did not find a solution.", + "TIME_LIMIT_REACHED": "The solver reached its execution time limit.", + "ABORTED": "The solver was interrupted externally.", + "ABORT_SOL": "The solver was stopped by the user but still found a solution.", + "ABORT_NOSOL": "The solver was stopped by the user and did not find a solution.", + "OUTMEM_SOL": "The solver ran out of memory but still found a solution.", + "OUTMEM_NOSOL": "The solver ran out of memory and either did not find a solution or failed to output the solution due to insufficient memory.", + "FAILED": "The solver failed to converge, possibly due to numerical issues.", + "FAIL_SOL": "The solver stopped due to errors but still found a solution.", + "FAIL_NOSOL": "The solver stopped due to errors and did not find a solution.", +} + + +CAS_OPTION_NAMES = [ + "hostname", + "port", + "username", + "password", + "session", + "locale", + "name", + "nworkers", + "authinfo", + "protocol", + "path", + "ssl_ca_list", + "authcode", +] + + +@SolverFactory.register('sas', doc='The SAS LP/MIP solver') +class SAS(OptSolver): + """The SAS optimization solver""" + + def __new__(cls, *args, **kwds): + mode = kwds.pop('solver_io', None) + if mode != None: + return SolverFactory(mode) + else: + # Choose solver factory automatically + # bassed on what can be loaded. + s = SolverFactory('_sas94', **kwds) + if not s.available(): + s = SolverFactory('_sascas', **kwds) + return s + + +class SASAbc(ABC, OptSolver): + """Abstract base class for the SAS solver interfaces. Simply to avoid code duplication.""" + + def __init__(self, **kwds): + """Initialize the SAS solver interfaces.""" + kwds['type'] = 'sas' + super(SASAbc, self).__init__(**kwds) + + # + # Set up valid problem formats and valid results for each + # problem format + # + self._valid_problem_formats = [ProblemFormat.mps] + self._valid_result_formats = {ProblemFormat.mps: [ResultsFormat.soln]} + + self._keepfiles = False + self._capabilities.linear = True + self._capabilities.integer = True + + super(SASAbc, self).set_problem_format(ProblemFormat.mps) + + def _presolve(self, *args, **kwds): + """ "Set things up for the actual solve.""" + # create a context in the temporary file manager for + # this plugin - is "pop"ed in the _postsolve method. + TempfileManager.push() + + # Get the warmstart flag + self.warmstart_flag = kwds.pop('warmstart', False) + + # Call parent presolve function + super(SASAbc, self)._presolve(*args, **kwds) + + # Store the model, too bad this is not done in the base class + for arg in args: + if isinstance(arg, (_BlockData, IBlock)): + # Store the instance + self._instance = arg + self._vars = [] + for block in self._instance.block_data_objects(active=True): + for vardata in block.component_data_objects( + Var, active=True, descend_into=False + ): + self._vars.append(vardata) + # Store the symbal map, we need this for example when writing the warmstart file + if isinstance(self._instance, IBlock): + self._smap = getattr(self._instance, "._symbol_maps")[self._smap_id] + else: + self._smap = self._instance.solutions.symbol_map[self._smap_id] + + # Create the primalin data + if self.warmstart_flag: + filename = self._warm_start_file_name = TempfileManager.create_tempfile( + ".sol", text=True + ) + smap = self._smap + numWritten = 0 + with open(filename, 'w') as file: + file.write('_VAR_,_VALUE_\n') + for var in self._vars: + if (var.value is not None) and (id(var) in smap.byObject): + name = smap.byObject[id(var)] + file.write( + "{name},{value}\n".format(name=name, value=var.value) + ) + numWritten += 1 + if numWritten == 0: + # No solution available, disable warmstart + self.warmstart_flag = False + + def available(self, exception_flag=False): + """True if the solver is available""" + return self._python_api_exists + + def _has_integer_variables(self): + """True if the problem has integer variables.""" + for vardata in self._vars: + if vardata.is_binary() or vardata.is_integer(): + return True + return False + + def _create_results_from_status(self, status, solution_status): + """Create a results object and set the status code and messages.""" + results = SolverResults() + results.solver.name = "SAS" + results.solver.status = STATUS_TO_SOLVERSTATUS[status] + if results.solver.status == SolverStatus.ok: + results.solver.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + solution_status + ] + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE[solution_status] + results.solver.status = TerminationCondition.to_solver_status( + results.solver.termination_condition + ) + elif results.solver.status == SolverStatus.aborted: + results.solver.termination_condition = TerminationCondition.userInterrupt + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE["ABORTED"] + else: + results.solver.termination_condition = TerminationCondition.error + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE["FAILED"] + return results + + @abstractmethod + def _apply_solver(self): + """The routine that performs the solve""" + raise NotImplemented("This is an abstract function and thus not implemented!") + + def _postsolve(self): + """Clean up at the end, especially the temp files.""" + # Let the base class deal with returning results. + results = super(SASAbc, self)._postsolve() + + # Finally, clean any temporary files registered with the temp file + # manager, created populated *directly* by this plugin. does not + # include, for example, the execution script. but does include + # the warm-start file. + TempfileManager.pop(remove=not self._keepfiles) + + return results + + def warm_start_capable(self): + """True if the solver interface supports MILP warmstarting.""" + return True + + +@SolverFactory.register('_sas94', doc='SAS 9.4 interface') +class SAS94(SASAbc): + """ + Solver interface for SAS 9.4 using saspy. See the saspy documentation about + how to create a connection. + """ + + def __init__(self, **kwds): + """Initialize the solver interface and see if the saspy package is available.""" + super(SAS94, self).__init__(**kwds) + + try: + import saspy + + self._sas = saspy + except ImportError: + self._python_api_exists = False + except Exception as e: + self._python_api_exists = False + # For other exceptions, raise it so that it does not get lost + raise e + else: + self._python_api_exists = True + self._sas.logger.setLevel(logger.level) + + def _create_statement_str(self, statement): + """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" + stmt = self.options.pop(statement, None) + if stmt: + return ( + statement.strip() + + " " + + " ".join(option + "=" + str(value) for option, value in stmt.items()) + + ";" + ) + else: + return "" + + def _apply_solver(self): + """ "Prepare the options and run the solver. Then store the data to be returned.""" + logger.debug("Running SAS") + + # Set return code to issue an error if we get interrupted + self._rc = -1 + + # Figure out if the problem has integer variables + with_opt = self.options.pop("with", None) + if with_opt == "lp": + proc = "OPTLP" + elif with_opt == "milp": + proc = "OPTMILP" + else: + # Check if there are integer variables, this might be slow + proc = "OPTMILP" if self._has_integer_variables() else "OPTLP" + + # Remove CAS options in case they were specified + for opt in CAS_OPTION_NAMES: + self.options.pop(opt, None) + + # Get the rootnode options + decomp_str = self._create_statement_str("decomp") + decompmaster_str = self._create_statement_str("decompmaster") + decompmasterip_str = self._create_statement_str("decompmasterip") + decompsubprob_str = self._create_statement_str("decompsubprob") + rootnode_str = self._create_statement_str("rootnode") + + # Handle warmstart + warmstart_str = "" + if self.warmstart_flag: + # Set the warmstart basis option + if proc != "OPTLP": + warmstart_str = """ + proc import datafile='{primalin}' + out=primalin + dbms=csv + replace; + getnames=yes; + run; + """.format( + primalin=self._warm_start_file_name + ) + self.options["primalin"] = "primalin" + + # Convert options to string + opt_str = " ".join( + option + "=" + str(value) for option, value in self.options.items() + ) + + # Start a SAS session, submit the code and return the results`` + with self._sas.SASsession() as sas: + # Find the version of 9.4 we are using + if sas.sasver.startswith("9.04.01M5"): + # In 9.4M5 we have to create an MPS data set from an MPS file first + # Earlier versions will not work because the MPS format in incompatible + res = sas.submit( + """ + {warmstart} + %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA=mpsdata, MAXLEN=256, FORMAT=FREE); + proc {proc} data=mpsdata {options} primalout=primalout dualout=dualout; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + else: + # Since 9.4M6+ optlp/optmilp can read mps files directly + res = sas.submit( + """ + {warmstart} + proc {proc} mpsfile=\"{mpsfile}\" {options} primalout=primalout dualout=dualout; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + + # Store log and ODS output + self._log = res["LOG"] + self._lst = res["LST"] + # Print log if requested by the user + if self._tee: + print(self._log) + if "ERROR 22-322: Syntax error" in self._log: + raise ValueError( + "An option passed to the SAS solver caused a syntax error: {log}".format( + log=self._log + ) + ) + self._macro = dict( + (key.strip(), value.strip()) + for key, value in ( + pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() + ) + ) + primal_out = sas.sd2df("primalout") + dual_out = sas.sd2df("dualout") + + # Prepare the solver results + results = self.results = self._create_results_from_status( + self._macro.get("STATUS", "ERROR"), self._macro.get("SOLUTION_STATUS","ERROR") + ) + + if "Objective Sense Maximization" in self._lst: + results.problem.sense = ProblemSense.maximize + else: + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if results.solver.termination_condition == TerminationCondition.optimal: + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = TerminationCondition.optimal + + # Store objective value in solution + sol.objective['__default_objective__'] = {'Value': self._macro["OBJECTIVE"]} + + if proc == "OPTLP": + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_']] + primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out.rename( + {'_VALUE_': 'Value', '_STATUS_': 'Status', '_R_COST_': 'rc'}, + axis='columns', + ) + sol.variable = primal_out.to_dict('index') + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = dual_out[['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_']] + dual_out = dual_out.set_index('_ROW_', drop=True) + dual_out = dual_out.rename( + {'_VALUE_': 'dual', '_STATUS_': 'Status', '_ACTIVITY_': 'slack'}, + axis='columns', + ) + sol.constraint = dual_out.to_dict('index') + else: + # Convert primal out data set to variable dictionary + # Use pandas functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_']] + primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out.rename({'_VALUE_': 'Value'}, axis='columns') + sol.variable = primal_out.to_dict('index') + + self._rc = 0 + return Bunch(rc=self._rc, log=self._log) + + +class SASLogWriter: + """Helper class to take the log from stdout and put it also in a StringIO.""" + + def __init__(self, tee): + """Set up the two outputs.""" + self.tee = tee + self._log = StringIO() + self.stdout = sys.stdout + + def write(self, message): + """If the tee options is specified, write to both outputs.""" + if self.tee: + self.stdout.write(message) + self._log.write(message) + + def flush(self): + """Nothing to do, just here for compatibility reasons.""" + # Do nothing since we flush right away + pass + + def log(self): + """ "Get the log as a string.""" + return self._log.getvalue() + + +@SolverFactory.register('_sascas', doc='SAS Viya CAS Server interface') +class SASCAS(SASAbc): + """ + Solver interface connection to a SAS Viya CAS server using swat. + See the documentation for the swat package about how to create a connection. + The swat connection options can be passed as options to the solve function. + """ + + def __init__(self, **kwds): + """Initialize and try to load the swat package.""" + super(SASCAS, self).__init__(**kwds) + + try: + import swat + + self._sas = swat + except ImportError: + self._python_api_exists = False + except Exception as e: + self._python_api_exists = False + # For other exceptions, raise it so that it does not get lost + raise e + else: + self._python_api_exists = True + + def _apply_solver(self): + """ "Prepare the options and run the solver. Then store the data to be returned.""" + logger.debug("Running SAS Viya") + + # Set return code to issue an error if we get interrupted + self._rc = -1 + + # Extract CAS connection options + cas_opts = {} + for opt in CAS_OPTION_NAMES: + val = self.options.pop(opt, None) + if val != None: + cas_opts[opt] = val + + # Figure out if the problem has integer variables + with_opt = self.options.pop("with", None) + if with_opt == "lp": + action = "solveLp" + elif with_opt == "milp": + action = "solveMilp" + else: + # Check if there are integer variables, this might be slow + action = "solveMilp" if self._has_integer_variables() else "solveLp" + + # Connect to CAS server + with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: + s = self._sas.CAS(**cas_opts) + try: + # Load the optimization action set + s.loadactionset('optimization') + + # Upload mps file to CAS + if os.stat(self._problem_files[0]).st_size >= 2 * 1024**3: + # For large files, use convertMPS, first create file for upload + mpsWithIdFileName = TempfileManager.create_tempfile( + ".mps.csv", text=True + ) + with open(mpsWithIdFileName, 'w') as mpsWithId: + mpsWithId.write('_ID_\tText\n') + with open(self._problem_files[0], 'r') as f: + id = 0 + for line in f: + id += 1 + mpsWithId.write(str(id) + '\t' + line.rstrip() + '\n') + + # Upload .mps.csv file + s.upload_file( + mpsWithIdFileName, + casout={"name": "mpscsv", "replace": True}, + importoptions={"filetype": "CSV", "delimiter": "\t"}, + ) + + # Convert .mps.csv file to .mps + s.optimization.convertMps( + data="mpscsv", + casOut={"name": "mpsdata", "replace": True}, + format="FREE", + ) + else: + # For small files, use loadMPS + with open(self._problem_files[0], 'r') as mps_file: + s.optimization.loadMps( + mpsFileString=mps_file.read(), + casout={"name": "mpsdata", "replace": True}, + format="FREE", + ) + + if self.warmstart_flag: + # Upload warmstart file to CAS + s.upload_file( + self._warm_start_file_name, + casout={"name": "primalin", "replace": True}, + importoptions={"filetype": "CSV"}, + ) + self.options["primalin"] = "primalin" + + # Solve the problem in CAS + if action == "solveMilp": + r = s.optimization.solveMilp( + data={"name": "mpsdata"}, + primalOut={"name": "primalout", "replace": True}, + **self.options + ) + else: + r = s.optimization.solveLp( + data={"name": "mpsdata"}, + primalOut={"name": "primalout", "replace": True}, + dualOut={"name": "dualout", "replace": True}, + **self.options + ) + + # Prepare the solver results + if r: + # Get back the primal and dual solution data sets + results = self.results = self._create_results_from_status( + r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") + ) + + if r.ProblemSummary["cValue1"][1] == "Maximization": + results.problem.sense = ProblemSense.maximize + else: + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if ( + results.solver.termination_condition + == TerminationCondition.optimal + ): + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = TerminationCondition.optimal + + # Store objective value in solution + sol.objective['__default_objective__'] = { + 'Value': r["objective"] + } + + if action == "solveMilp": + primal_out = s.CASTable(name="primalout") + # Use pandas functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_']] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {'Value': row[1]} + else: + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = s.CASTable(name="primalout") + primal_out = primal_out[ + ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] + ] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = { + 'Value': row[1], + 'Status': row[2], + 'rc': row[3], + } + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = s.CASTable(name="dualout") + dual_out = dual_out[ + ['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_'] + ] + sol.constraint = {} + for row in dual_out.itertuples(index=False): + sol.constraint[row[0]] = { + 'dual': row[1], + 'Status': row[2], + 'slack': row[3], + } + else: + results = self.results = SolverResults() + results.solver.name = "SAS" + results.solver.status = SolverStatus.error + raise ValueError( + "An option passed to the SAS solver caused a syntax error." + ) + + finally: + s.close() + + self._log = self._log_writer.log() + if self._tee: + print(self._log) + self._rc = 0 + return Bunch(rc=self._rc, log=self._log) diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index c5fbfa97e42..23b7fe06526 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -30,3 +30,4 @@ import pyomo.solvers.plugins.solvers.mosek_persistent import pyomo.solvers.plugins.solvers.xpress_direct import pyomo.solvers.plugins.solvers.xpress_persistent +import pyomo.solvers.plugins.solvers.SAS diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py new file mode 100644 index 00000000000..4592343b17f --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -0,0 +1,462 @@ +import os +import pyomo.common.unittest as unittest +from pyomo.environ import ( + ConcreteModel, + Var, + Objective, + Constraint, + NonNegativeIntegers, + NonNegativeReals, + Reals, + Integers, + maximize, + minimize, + Suffix, +) +from pyomo.opt.results import ( + SolverStatus, + TerminationCondition, + ProblemSense, +) +from pyomo.opt import ( + SolverFactory, + check_available_solvers, +) + + +CAS_OPTIONS = { + "hostname": os.environ.get('CAS_SERVER', None), + "port": os.environ.get('CAS_PORT', None), + "authinfo": os.environ.get('CAS_AUTHINFO', None), +} + + +sas_available = check_available_solvers('sas') + + +class SASTestAbc: + solver_io = '_sas94' + base_options = {} + + def setObj(self): + X = self.instance.X + self.instance.Obj = Objective( + expr=2 * X[1] - 3 * X[2] - 4 * X[3], sense=minimize + ) + + def setX(self): + self.instance.X = Var([1, 2, 3], within=NonNegativeReals) + + def setUp(self): + instance = self.instance = ConcreteModel() + self.setX() + X = instance.X + instance.R1 = Constraint(expr=-2 * X[2] - 3 * X[3] >= -5) + instance.R2 = Constraint(expr=X[1] + X[2] + 2 * X[3] <= 4) + instance.R3 = Constraint(expr=X[1] + 2 * X[2] + 3 * X[3] <= 7) + self.setObj() + + # Declare suffixes for solution information + instance.status = Suffix(direction=Suffix.IMPORT) + instance.slack = Suffix(direction=Suffix.IMPORT) + instance.rc = Suffix(direction=Suffix.IMPORT) + instance.dual = Suffix(direction=Suffix.IMPORT) + + self.opt_sas = SolverFactory('sas', solver_io=self.solver_io) + + def tearDown(self): + del self.opt_sas + del self.instance + + def run_solver(self, **kwargs): + opt_sas = self.opt_sas + instance = self.instance + + # Add base options for connection data etc. + options = kwargs.get("options", {}) + if self.base_options: + kwargs["options"] = {**options, **self.base_options} + + # Call the solver + self.results = opt_sas.solve(instance, **kwargs) + + +class SASTestLP(SASTestAbc, unittest.TestCase): + def checkSolution(self): + instance = self.instance + results = self.results + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7.5) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 2.5) + self.assertAlmostEqual(instance.X[3].value, 0.0) + + # Check reduced cost + self.assertAlmostEqual(instance.rc[instance.X[1]], sense * 2.0) + self.assertAlmostEqual(instance.rc[instance.X[2]], sense * 0.0) + self.assertAlmostEqual(instance.rc[instance.X[3]], sense * 0.5) + + # Check slack + self.assertAlmostEqual(instance.slack[instance.R1], -5.0) + self.assertAlmostEqual(instance.slack[instance.R2], 2.5) + self.assertAlmostEqual(instance.slack[instance.R3], 5.0) + + # Check dual solution + self.assertAlmostEqual(instance.dual[instance.R1], sense * 1.5) + self.assertAlmostEqual(instance.dual[instance.R2], sense * 0.0) + self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) + + # Check basis status + self.assertEqual(instance.status[instance.X[1]], 'L') + self.assertEqual(instance.status[instance.X[2]], 'B') + self.assertEqual(instance.status[instance.X[3]], 'L') + self.assertEqual(instance.status[instance.R1], 'U') + self.assertEqual(instance.status[instance.R2], 'B') + self.assertEqual(instance.status[instance.R3], 'B') + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_default(self): + self.run_solver() + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_primal(self): + self.run_solver(options={"algorithm": "ps"}) + self.assertIn("NOTE: The Primal Simplex algorithm is used.", self.opt_sas._log) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_ipm(self): + self.run_solver(options={"algorithm": "ip"}) + self.assertIn("NOTE: The Interior Point algorithm is used.", self.opt_sas._log) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_intoption(self): + self.run_solver(options={"maxiter": 20}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_invalidoption(self): + with self.assertRaisesRegex(ValueError, "syntax error"): + self.run_solver(options={"foo": "bar"}) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_max(self): + X = self.instance.X + self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) + self.instance.Obj.sense = maximize + self.run_solver() + self.checkSolution() + self.assertEqual(self.results.problem.sense, ProblemSense.maximize) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible(self): + instance = self.instance + X = instance.X + instance.R4 = Constraint(expr=-2 * X[2] - 3 * X[3] <= -6) + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + self.assertEqual(results.solver.message, "The problem is infeasible.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible_or_unbounded(self): + self.instance.X.domain = Reals + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, + TerminationCondition.infeasibleOrUnbounded, + ) + self.assertEqual( + results.solver.message, "The problem is infeasible or unbounded." + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_unbounded(self): + self.instance.X.domain = Reals + self.run_solver(options={"presolver": "none", "algorithm": "primal"}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.unbounded + ) + self.assertEqual(results.solver.message, "The problem is unbounded.") + + def checkSolutionDecomp(self): + instance = self.instance + results = self.results + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7.5) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 2.5) + self.assertAlmostEqual(instance.X[3].value, 0.0) + + # Check reduced cost + self.assertAlmostEqual(instance.rc[instance.X[1]], sense * 2.0) + self.assertAlmostEqual(instance.rc[instance.X[2]], sense * 0.0) + self.assertAlmostEqual(instance.rc[instance.X[3]], sense * 0.5) + + # Check slack + self.assertAlmostEqual(instance.slack[instance.R1], -5.0) + self.assertAlmostEqual(instance.slack[instance.R2], 2.5) + self.assertAlmostEqual(instance.slack[instance.R3], 5.0) + + # Check dual solution + self.assertAlmostEqual(instance.dual[instance.R1], sense * 1.5) + self.assertAlmostEqual(instance.dual[instance.R2], sense * 0.0) + self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) + + # Don't check basis status for decomp + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_decomp(self): + self.run_solver( + options={ + "decomp": {"absobjgap": 0.0}, + "decompmaster": {"algorithm": "dual"}, + "decompsubprob": {"presolver": "none"}, + } + ) + self.assertIn( + "NOTE: The DECOMP method value DEFAULT is applied.", self.opt_sas._log + ) + self.checkSolutionDecomp() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_iis(self): + self.run_solver(options={"iis": "true"}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.feasible + ) + self.assertIn("NOTE: The IIS= option is enabled.", self.opt_sas._log) + self.assertEqual( + results.solver.message, + "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxiter(self): + self.run_solver(options={"maxiter": 1}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxIterations + ) + self.assertEqual( + results.solver.message, + "The maximum allowable number of iterations was reached.", + ) + + +class SASTestLPCAS(SASTestLP): + solver_io = '_sascas' + base_options = CAS_OPTIONS + + +class SASTestMILP(SASTestAbc, unittest.TestCase): + def setX(self): + self.instance.X = Var([1, 2, 3], within=NonNegativeIntegers) + + def checkSolution(self): + instance = self.instance + results = self.results + + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 1.0) + self.assertAlmostEqual(instance.X[3].value, 1.0) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_default(self): + self.run_solver(options={}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_presolve(self): + self.run_solver(options={"presolver": "none"}) + self.assertIn( + "NOTE: The MILP presolver value NONE is applied.", self.opt_sas._log + ) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_intoption(self): + self.run_solver(options={"maxnodes": 20}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_invalidoption(self): + with self.assertRaisesRegex(ValueError, "syntax error"): + self.run_solver(options={"foo": "bar"}) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_max(self): + X = self.instance.X + self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) + self.instance.Obj.sense = maximize + self.run_solver() + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible(self): + instance = self.instance + X = instance.X + instance.R4 = Constraint(expr=-2 * X[2] - 3 * X[3] <= -6) + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + self.assertEqual(results.solver.message, "The problem is infeasible.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + @unittest.skip("Returns wrong status for some versions.") + def test_solver_infeasible_or_unbounded(self): + self.instance.X.domain = Integers + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, + TerminationCondition.infeasibleOrUnbounded, + ) + self.assertEqual( + results.solver.message, "The problem is infeasible or unbounded." + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_unbounded(self): + self.instance.X.domain = Integers + self.run_solver( + options={"presolver": "none", "rootnode": {"algorithm": "primal"}} + ) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.unbounded + ) + self.assertEqual(results.solver.message, "The problem is unbounded.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_decomp(self): + self.run_solver( + options={ + "decomp": {"hybrid": "off"}, + "decompmaster": {"algorithm": "dual"}, + "decompmasterip": {"presolver": "none"}, + "decompsubprob": {"presolver": "none"}, + } + ) + self.assertIn( + "NOTE: The DECOMP method value DEFAULT is applied.", self.opt_sas._log + ) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_rootnode(self): + self.run_solver(options={"rootnode": {"presolver": "automatic"}}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxnodes(self): + self.run_solver(options={"maxnodes": 0}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxEvaluations + ) + self.assertEqual( + results.solver.message, + "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxsols(self): + self.run_solver(options={"maxsols": 1}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxEvaluations + ) + self.assertEqual( + results.solver.message, + "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_target(self): + self.run_solver(options={"target": -6.0}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + self.assertEqual( + results.solver.message, + "The solution is not worse than the target specified by the TARGET= option.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_primalin(self): + X = self.instance.X + X[1] = None + X[2] = 3 + X[3] = 7 + self.run_solver(warmstart=True) + self.checkSolution() + self.assertIn( + "NOTE: The input solution is infeasible or incomplete. Repair heuristics are applied.", + self.opt_sas._log, + ) + + +class SASTestMILPCAS(SASTestMILP): + solver_io = '_sascas' + base_options = CAS_OPTIONS + + +if __name__ == '__main__': + unittest.main() From be423b99adfa97a6cc3d3cb20985398891acecbe Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 23 Jun 2023 09:56:54 -0400 Subject: [PATCH 002/150] Just some black adjustments --- pyomo/solvers/plugins/solvers/SAS.py | 3 ++- pyomo/solvers/tests/checks/test_SAS.py | 11 ++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 7f50b7a2970..ed0e63d44d6 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -427,7 +427,8 @@ def _apply_solver(self): # Prepare the solver results results = self.results = self._create_results_from_status( - self._macro.get("STATUS", "ERROR"), self._macro.get("SOLUTION_STATUS","ERROR") + self._macro.get("STATUS", "ERROR"), + self._macro.get("SOLUTION_STATUS", "ERROR"), ) if "Objective Sense Maximization" in self._lst: diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 4592343b17f..654820f5060 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -13,15 +13,8 @@ minimize, Suffix, ) -from pyomo.opt.results import ( - SolverStatus, - TerminationCondition, - ProblemSense, -) -from pyomo.opt import ( - SolverFactory, - check_available_solvers, -) +from pyomo.opt.results import SolverStatus, TerminationCondition, ProblemSense +from pyomo.opt import SolverFactory, check_available_solvers CAS_OPTIONS = { From 284ab98e470cf67c5273837a28c11afbb2b153a5 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 23 Jun 2023 09:42:08 -0400 Subject: [PATCH 003/150] Initial version of the SAS solver interfaces and unit tests. --- pyomo/solvers/plugins/solvers/SAS.py | 700 ++++++++++++++++++++++ pyomo/solvers/plugins/solvers/__init__.py | 1 + pyomo/solvers/tests/checks/test_SAS.py | 462 ++++++++++++++ 3 files changed, 1163 insertions(+) create mode 100644 pyomo/solvers/plugins/solvers/SAS.py create mode 100644 pyomo/solvers/tests/checks/test_SAS.py diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py new file mode 100644 index 00000000000..7f50b7a2970 --- /dev/null +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -0,0 +1,700 @@ +__all__ = ['SAS'] + +import logging +import sys +import os + +from io import StringIO +from abc import ABC, abstractmethod +from contextlib import redirect_stdout + +from pyomo.opt.base import ProblemFormat, ResultsFormat, OptSolver +from pyomo.opt.base.solvers import SolverFactory +from pyomo.common.collections import Bunch +from pyomo.opt.results import ( + SolverResults, + SolverStatus, + TerminationCondition, + SolutionStatus, + ProblemSense, +) +from pyomo.common.tempfiles import TempfileManager +from pyomo.core.base import Var +from pyomo.core.base.block import _BlockData +from pyomo.core.kernel.block import IBlock + + +logger = logging.getLogger('pyomo.solvers') + + +STATUS_TO_SOLVERSTATUS = { + "OK": SolverStatus.ok, + "SYNTAX_ERROR": SolverStatus.error, + "DATA_ERROR": SolverStatus.error, + "OUT_OF_MEMORY": SolverStatus.aborted, + "IO_ERROR": SolverStatus.error, + "ERROR": SolverStatus.error, +} + +# This combines all status codes from OPTLP/solvelp and OPTMILP/solvemilp +SOLSTATUS_TO_TERMINATIONCOND = { + "OPTIMAL": TerminationCondition.optimal, + "OPTIMAL_AGAP": TerminationCondition.optimal, + "OPTIMAL_RGAP": TerminationCondition.optimal, + "OPTIMAL_COND": TerminationCondition.optimal, + "TARGET": TerminationCondition.optimal, + "CONDITIONAL_OPTIMAL": TerminationCondition.optimal, + "FEASIBLE": TerminationCondition.feasible, + "INFEASIBLE": TerminationCondition.infeasible, + "UNBOUNDED": TerminationCondition.unbounded, + "INFEASIBLE_OR_UNBOUNDED": TerminationCondition.infeasibleOrUnbounded, + "SOLUTION_LIM": TerminationCondition.maxEvaluations, + "NODE_LIM_SOL": TerminationCondition.maxEvaluations, + "NODE_LIM_NOSOL": TerminationCondition.maxEvaluations, + "ITERATION_LIMIT_REACHED": TerminationCondition.maxIterations, + "TIME_LIM_SOL": TerminationCondition.maxTimeLimit, + "TIME_LIM_NOSOL": TerminationCondition.maxTimeLimit, + "TIME_LIMIT_REACHED": TerminationCondition.maxTimeLimit, + "ABORTED": TerminationCondition.userInterrupt, + "ABORT_SOL": TerminationCondition.userInterrupt, + "ABORT_NOSOL": TerminationCondition.userInterrupt, + "OUTMEM_SOL": TerminationCondition.solverFailure, + "OUTMEM_NOSOL": TerminationCondition.solverFailure, + "FAILED": TerminationCondition.solverFailure, + "FAIL_SOL": TerminationCondition.solverFailure, + "FAIL_NOSOL": TerminationCondition.solverFailure, +} + + +SOLSTATUS_TO_MESSAGE = { + "OPTIMAL": "The solution is optimal.", + "OPTIMAL_AGAP": "The solution is optimal within the absolute gap specified by the ABSOBJGAP= option.", + "OPTIMAL_RGAP": "The solution is optimal within the relative gap specified by the RELOBJGAP= option.", + "OPTIMAL_COND": "The solution is optimal, but some infeasibilities (primal, bound, or integer) exceed tolerances due to scaling or choice of a small INTTOL= value.", + "TARGET": "The solution is not worse than the target specified by the TARGET= option.", + "CONDITIONAL_OPTIMAL": "The solution is optimal, but some infeasibilities (primal, dual or bound) exceed tolerances due to scaling or preprocessing.", + "FEASIBLE": "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", + "INFEASIBLE": "The problem is infeasible.", + "UNBOUNDED": "The problem is unbounded.", + "INFEASIBLE_OR_UNBOUNDED": "The problem is infeasible or unbounded.", + "SOLUTION_LIM": "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", + "NODE_LIM_SOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", + "NODE_LIM_NOSOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and did not find a solution.", + "ITERATION_LIMIT_REACHED": "The maximum allowable number of iterations was reached.", + "TIME_LIM_SOL": "The solver reached the execution time limit specified by the MAXTIME= option and found a solution.", + "TIME_LIM_NOSOL": "The solver reached the execution time limit specified by the MAXTIME= option and did not find a solution.", + "TIME_LIMIT_REACHED": "The solver reached its execution time limit.", + "ABORTED": "The solver was interrupted externally.", + "ABORT_SOL": "The solver was stopped by the user but still found a solution.", + "ABORT_NOSOL": "The solver was stopped by the user and did not find a solution.", + "OUTMEM_SOL": "The solver ran out of memory but still found a solution.", + "OUTMEM_NOSOL": "The solver ran out of memory and either did not find a solution or failed to output the solution due to insufficient memory.", + "FAILED": "The solver failed to converge, possibly due to numerical issues.", + "FAIL_SOL": "The solver stopped due to errors but still found a solution.", + "FAIL_NOSOL": "The solver stopped due to errors and did not find a solution.", +} + + +CAS_OPTION_NAMES = [ + "hostname", + "port", + "username", + "password", + "session", + "locale", + "name", + "nworkers", + "authinfo", + "protocol", + "path", + "ssl_ca_list", + "authcode", +] + + +@SolverFactory.register('sas', doc='The SAS LP/MIP solver') +class SAS(OptSolver): + """The SAS optimization solver""" + + def __new__(cls, *args, **kwds): + mode = kwds.pop('solver_io', None) + if mode != None: + return SolverFactory(mode) + else: + # Choose solver factory automatically + # bassed on what can be loaded. + s = SolverFactory('_sas94', **kwds) + if not s.available(): + s = SolverFactory('_sascas', **kwds) + return s + + +class SASAbc(ABC, OptSolver): + """Abstract base class for the SAS solver interfaces. Simply to avoid code duplication.""" + + def __init__(self, **kwds): + """Initialize the SAS solver interfaces.""" + kwds['type'] = 'sas' + super(SASAbc, self).__init__(**kwds) + + # + # Set up valid problem formats and valid results for each + # problem format + # + self._valid_problem_formats = [ProblemFormat.mps] + self._valid_result_formats = {ProblemFormat.mps: [ResultsFormat.soln]} + + self._keepfiles = False + self._capabilities.linear = True + self._capabilities.integer = True + + super(SASAbc, self).set_problem_format(ProblemFormat.mps) + + def _presolve(self, *args, **kwds): + """ "Set things up for the actual solve.""" + # create a context in the temporary file manager for + # this plugin - is "pop"ed in the _postsolve method. + TempfileManager.push() + + # Get the warmstart flag + self.warmstart_flag = kwds.pop('warmstart', False) + + # Call parent presolve function + super(SASAbc, self)._presolve(*args, **kwds) + + # Store the model, too bad this is not done in the base class + for arg in args: + if isinstance(arg, (_BlockData, IBlock)): + # Store the instance + self._instance = arg + self._vars = [] + for block in self._instance.block_data_objects(active=True): + for vardata in block.component_data_objects( + Var, active=True, descend_into=False + ): + self._vars.append(vardata) + # Store the symbal map, we need this for example when writing the warmstart file + if isinstance(self._instance, IBlock): + self._smap = getattr(self._instance, "._symbol_maps")[self._smap_id] + else: + self._smap = self._instance.solutions.symbol_map[self._smap_id] + + # Create the primalin data + if self.warmstart_flag: + filename = self._warm_start_file_name = TempfileManager.create_tempfile( + ".sol", text=True + ) + smap = self._smap + numWritten = 0 + with open(filename, 'w') as file: + file.write('_VAR_,_VALUE_\n') + for var in self._vars: + if (var.value is not None) and (id(var) in smap.byObject): + name = smap.byObject[id(var)] + file.write( + "{name},{value}\n".format(name=name, value=var.value) + ) + numWritten += 1 + if numWritten == 0: + # No solution available, disable warmstart + self.warmstart_flag = False + + def available(self, exception_flag=False): + """True if the solver is available""" + return self._python_api_exists + + def _has_integer_variables(self): + """True if the problem has integer variables.""" + for vardata in self._vars: + if vardata.is_binary() or vardata.is_integer(): + return True + return False + + def _create_results_from_status(self, status, solution_status): + """Create a results object and set the status code and messages.""" + results = SolverResults() + results.solver.name = "SAS" + results.solver.status = STATUS_TO_SOLVERSTATUS[status] + if results.solver.status == SolverStatus.ok: + results.solver.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + solution_status + ] + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE[solution_status] + results.solver.status = TerminationCondition.to_solver_status( + results.solver.termination_condition + ) + elif results.solver.status == SolverStatus.aborted: + results.solver.termination_condition = TerminationCondition.userInterrupt + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE["ABORTED"] + else: + results.solver.termination_condition = TerminationCondition.error + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE["FAILED"] + return results + + @abstractmethod + def _apply_solver(self): + """The routine that performs the solve""" + raise NotImplemented("This is an abstract function and thus not implemented!") + + def _postsolve(self): + """Clean up at the end, especially the temp files.""" + # Let the base class deal with returning results. + results = super(SASAbc, self)._postsolve() + + # Finally, clean any temporary files registered with the temp file + # manager, created populated *directly* by this plugin. does not + # include, for example, the execution script. but does include + # the warm-start file. + TempfileManager.pop(remove=not self._keepfiles) + + return results + + def warm_start_capable(self): + """True if the solver interface supports MILP warmstarting.""" + return True + + +@SolverFactory.register('_sas94', doc='SAS 9.4 interface') +class SAS94(SASAbc): + """ + Solver interface for SAS 9.4 using saspy. See the saspy documentation about + how to create a connection. + """ + + def __init__(self, **kwds): + """Initialize the solver interface and see if the saspy package is available.""" + super(SAS94, self).__init__(**kwds) + + try: + import saspy + + self._sas = saspy + except ImportError: + self._python_api_exists = False + except Exception as e: + self._python_api_exists = False + # For other exceptions, raise it so that it does not get lost + raise e + else: + self._python_api_exists = True + self._sas.logger.setLevel(logger.level) + + def _create_statement_str(self, statement): + """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" + stmt = self.options.pop(statement, None) + if stmt: + return ( + statement.strip() + + " " + + " ".join(option + "=" + str(value) for option, value in stmt.items()) + + ";" + ) + else: + return "" + + def _apply_solver(self): + """ "Prepare the options and run the solver. Then store the data to be returned.""" + logger.debug("Running SAS") + + # Set return code to issue an error if we get interrupted + self._rc = -1 + + # Figure out if the problem has integer variables + with_opt = self.options.pop("with", None) + if with_opt == "lp": + proc = "OPTLP" + elif with_opt == "milp": + proc = "OPTMILP" + else: + # Check if there are integer variables, this might be slow + proc = "OPTMILP" if self._has_integer_variables() else "OPTLP" + + # Remove CAS options in case they were specified + for opt in CAS_OPTION_NAMES: + self.options.pop(opt, None) + + # Get the rootnode options + decomp_str = self._create_statement_str("decomp") + decompmaster_str = self._create_statement_str("decompmaster") + decompmasterip_str = self._create_statement_str("decompmasterip") + decompsubprob_str = self._create_statement_str("decompsubprob") + rootnode_str = self._create_statement_str("rootnode") + + # Handle warmstart + warmstart_str = "" + if self.warmstart_flag: + # Set the warmstart basis option + if proc != "OPTLP": + warmstart_str = """ + proc import datafile='{primalin}' + out=primalin + dbms=csv + replace; + getnames=yes; + run; + """.format( + primalin=self._warm_start_file_name + ) + self.options["primalin"] = "primalin" + + # Convert options to string + opt_str = " ".join( + option + "=" + str(value) for option, value in self.options.items() + ) + + # Start a SAS session, submit the code and return the results`` + with self._sas.SASsession() as sas: + # Find the version of 9.4 we are using + if sas.sasver.startswith("9.04.01M5"): + # In 9.4M5 we have to create an MPS data set from an MPS file first + # Earlier versions will not work because the MPS format in incompatible + res = sas.submit( + """ + {warmstart} + %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA=mpsdata, MAXLEN=256, FORMAT=FREE); + proc {proc} data=mpsdata {options} primalout=primalout dualout=dualout; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + else: + # Since 9.4M6+ optlp/optmilp can read mps files directly + res = sas.submit( + """ + {warmstart} + proc {proc} mpsfile=\"{mpsfile}\" {options} primalout=primalout dualout=dualout; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + + # Store log and ODS output + self._log = res["LOG"] + self._lst = res["LST"] + # Print log if requested by the user + if self._tee: + print(self._log) + if "ERROR 22-322: Syntax error" in self._log: + raise ValueError( + "An option passed to the SAS solver caused a syntax error: {log}".format( + log=self._log + ) + ) + self._macro = dict( + (key.strip(), value.strip()) + for key, value in ( + pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() + ) + ) + primal_out = sas.sd2df("primalout") + dual_out = sas.sd2df("dualout") + + # Prepare the solver results + results = self.results = self._create_results_from_status( + self._macro.get("STATUS", "ERROR"), self._macro.get("SOLUTION_STATUS","ERROR") + ) + + if "Objective Sense Maximization" in self._lst: + results.problem.sense = ProblemSense.maximize + else: + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if results.solver.termination_condition == TerminationCondition.optimal: + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = TerminationCondition.optimal + + # Store objective value in solution + sol.objective['__default_objective__'] = {'Value': self._macro["OBJECTIVE"]} + + if proc == "OPTLP": + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_']] + primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out.rename( + {'_VALUE_': 'Value', '_STATUS_': 'Status', '_R_COST_': 'rc'}, + axis='columns', + ) + sol.variable = primal_out.to_dict('index') + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = dual_out[['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_']] + dual_out = dual_out.set_index('_ROW_', drop=True) + dual_out = dual_out.rename( + {'_VALUE_': 'dual', '_STATUS_': 'Status', '_ACTIVITY_': 'slack'}, + axis='columns', + ) + sol.constraint = dual_out.to_dict('index') + else: + # Convert primal out data set to variable dictionary + # Use pandas functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_']] + primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out.rename({'_VALUE_': 'Value'}, axis='columns') + sol.variable = primal_out.to_dict('index') + + self._rc = 0 + return Bunch(rc=self._rc, log=self._log) + + +class SASLogWriter: + """Helper class to take the log from stdout and put it also in a StringIO.""" + + def __init__(self, tee): + """Set up the two outputs.""" + self.tee = tee + self._log = StringIO() + self.stdout = sys.stdout + + def write(self, message): + """If the tee options is specified, write to both outputs.""" + if self.tee: + self.stdout.write(message) + self._log.write(message) + + def flush(self): + """Nothing to do, just here for compatibility reasons.""" + # Do nothing since we flush right away + pass + + def log(self): + """ "Get the log as a string.""" + return self._log.getvalue() + + +@SolverFactory.register('_sascas', doc='SAS Viya CAS Server interface') +class SASCAS(SASAbc): + """ + Solver interface connection to a SAS Viya CAS server using swat. + See the documentation for the swat package about how to create a connection. + The swat connection options can be passed as options to the solve function. + """ + + def __init__(self, **kwds): + """Initialize and try to load the swat package.""" + super(SASCAS, self).__init__(**kwds) + + try: + import swat + + self._sas = swat + except ImportError: + self._python_api_exists = False + except Exception as e: + self._python_api_exists = False + # For other exceptions, raise it so that it does not get lost + raise e + else: + self._python_api_exists = True + + def _apply_solver(self): + """ "Prepare the options and run the solver. Then store the data to be returned.""" + logger.debug("Running SAS Viya") + + # Set return code to issue an error if we get interrupted + self._rc = -1 + + # Extract CAS connection options + cas_opts = {} + for opt in CAS_OPTION_NAMES: + val = self.options.pop(opt, None) + if val != None: + cas_opts[opt] = val + + # Figure out if the problem has integer variables + with_opt = self.options.pop("with", None) + if with_opt == "lp": + action = "solveLp" + elif with_opt == "milp": + action = "solveMilp" + else: + # Check if there are integer variables, this might be slow + action = "solveMilp" if self._has_integer_variables() else "solveLp" + + # Connect to CAS server + with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: + s = self._sas.CAS(**cas_opts) + try: + # Load the optimization action set + s.loadactionset('optimization') + + # Upload mps file to CAS + if os.stat(self._problem_files[0]).st_size >= 2 * 1024**3: + # For large files, use convertMPS, first create file for upload + mpsWithIdFileName = TempfileManager.create_tempfile( + ".mps.csv", text=True + ) + with open(mpsWithIdFileName, 'w') as mpsWithId: + mpsWithId.write('_ID_\tText\n') + with open(self._problem_files[0], 'r') as f: + id = 0 + for line in f: + id += 1 + mpsWithId.write(str(id) + '\t' + line.rstrip() + '\n') + + # Upload .mps.csv file + s.upload_file( + mpsWithIdFileName, + casout={"name": "mpscsv", "replace": True}, + importoptions={"filetype": "CSV", "delimiter": "\t"}, + ) + + # Convert .mps.csv file to .mps + s.optimization.convertMps( + data="mpscsv", + casOut={"name": "mpsdata", "replace": True}, + format="FREE", + ) + else: + # For small files, use loadMPS + with open(self._problem_files[0], 'r') as mps_file: + s.optimization.loadMps( + mpsFileString=mps_file.read(), + casout={"name": "mpsdata", "replace": True}, + format="FREE", + ) + + if self.warmstart_flag: + # Upload warmstart file to CAS + s.upload_file( + self._warm_start_file_name, + casout={"name": "primalin", "replace": True}, + importoptions={"filetype": "CSV"}, + ) + self.options["primalin"] = "primalin" + + # Solve the problem in CAS + if action == "solveMilp": + r = s.optimization.solveMilp( + data={"name": "mpsdata"}, + primalOut={"name": "primalout", "replace": True}, + **self.options + ) + else: + r = s.optimization.solveLp( + data={"name": "mpsdata"}, + primalOut={"name": "primalout", "replace": True}, + dualOut={"name": "dualout", "replace": True}, + **self.options + ) + + # Prepare the solver results + if r: + # Get back the primal and dual solution data sets + results = self.results = self._create_results_from_status( + r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") + ) + + if r.ProblemSummary["cValue1"][1] == "Maximization": + results.problem.sense = ProblemSense.maximize + else: + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if ( + results.solver.termination_condition + == TerminationCondition.optimal + ): + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = TerminationCondition.optimal + + # Store objective value in solution + sol.objective['__default_objective__'] = { + 'Value': r["objective"] + } + + if action == "solveMilp": + primal_out = s.CASTable(name="primalout") + # Use pandas functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_']] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {'Value': row[1]} + else: + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = s.CASTable(name="primalout") + primal_out = primal_out[ + ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] + ] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = { + 'Value': row[1], + 'Status': row[2], + 'rc': row[3], + } + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = s.CASTable(name="dualout") + dual_out = dual_out[ + ['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_'] + ] + sol.constraint = {} + for row in dual_out.itertuples(index=False): + sol.constraint[row[0]] = { + 'dual': row[1], + 'Status': row[2], + 'slack': row[3], + } + else: + results = self.results = SolverResults() + results.solver.name = "SAS" + results.solver.status = SolverStatus.error + raise ValueError( + "An option passed to the SAS solver caused a syntax error." + ) + + finally: + s.close() + + self._log = self._log_writer.log() + if self._tee: + print(self._log) + self._rc = 0 + return Bunch(rc=self._rc, log=self._log) diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index c5fbfa97e42..23b7fe06526 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -30,3 +30,4 @@ import pyomo.solvers.plugins.solvers.mosek_persistent import pyomo.solvers.plugins.solvers.xpress_direct import pyomo.solvers.plugins.solvers.xpress_persistent +import pyomo.solvers.plugins.solvers.SAS diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py new file mode 100644 index 00000000000..4592343b17f --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -0,0 +1,462 @@ +import os +import pyomo.common.unittest as unittest +from pyomo.environ import ( + ConcreteModel, + Var, + Objective, + Constraint, + NonNegativeIntegers, + NonNegativeReals, + Reals, + Integers, + maximize, + minimize, + Suffix, +) +from pyomo.opt.results import ( + SolverStatus, + TerminationCondition, + ProblemSense, +) +from pyomo.opt import ( + SolverFactory, + check_available_solvers, +) + + +CAS_OPTIONS = { + "hostname": os.environ.get('CAS_SERVER', None), + "port": os.environ.get('CAS_PORT', None), + "authinfo": os.environ.get('CAS_AUTHINFO', None), +} + + +sas_available = check_available_solvers('sas') + + +class SASTestAbc: + solver_io = '_sas94' + base_options = {} + + def setObj(self): + X = self.instance.X + self.instance.Obj = Objective( + expr=2 * X[1] - 3 * X[2] - 4 * X[3], sense=minimize + ) + + def setX(self): + self.instance.X = Var([1, 2, 3], within=NonNegativeReals) + + def setUp(self): + instance = self.instance = ConcreteModel() + self.setX() + X = instance.X + instance.R1 = Constraint(expr=-2 * X[2] - 3 * X[3] >= -5) + instance.R2 = Constraint(expr=X[1] + X[2] + 2 * X[3] <= 4) + instance.R3 = Constraint(expr=X[1] + 2 * X[2] + 3 * X[3] <= 7) + self.setObj() + + # Declare suffixes for solution information + instance.status = Suffix(direction=Suffix.IMPORT) + instance.slack = Suffix(direction=Suffix.IMPORT) + instance.rc = Suffix(direction=Suffix.IMPORT) + instance.dual = Suffix(direction=Suffix.IMPORT) + + self.opt_sas = SolverFactory('sas', solver_io=self.solver_io) + + def tearDown(self): + del self.opt_sas + del self.instance + + def run_solver(self, **kwargs): + opt_sas = self.opt_sas + instance = self.instance + + # Add base options for connection data etc. + options = kwargs.get("options", {}) + if self.base_options: + kwargs["options"] = {**options, **self.base_options} + + # Call the solver + self.results = opt_sas.solve(instance, **kwargs) + + +class SASTestLP(SASTestAbc, unittest.TestCase): + def checkSolution(self): + instance = self.instance + results = self.results + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7.5) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 2.5) + self.assertAlmostEqual(instance.X[3].value, 0.0) + + # Check reduced cost + self.assertAlmostEqual(instance.rc[instance.X[1]], sense * 2.0) + self.assertAlmostEqual(instance.rc[instance.X[2]], sense * 0.0) + self.assertAlmostEqual(instance.rc[instance.X[3]], sense * 0.5) + + # Check slack + self.assertAlmostEqual(instance.slack[instance.R1], -5.0) + self.assertAlmostEqual(instance.slack[instance.R2], 2.5) + self.assertAlmostEqual(instance.slack[instance.R3], 5.0) + + # Check dual solution + self.assertAlmostEqual(instance.dual[instance.R1], sense * 1.5) + self.assertAlmostEqual(instance.dual[instance.R2], sense * 0.0) + self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) + + # Check basis status + self.assertEqual(instance.status[instance.X[1]], 'L') + self.assertEqual(instance.status[instance.X[2]], 'B') + self.assertEqual(instance.status[instance.X[3]], 'L') + self.assertEqual(instance.status[instance.R1], 'U') + self.assertEqual(instance.status[instance.R2], 'B') + self.assertEqual(instance.status[instance.R3], 'B') + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_default(self): + self.run_solver() + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_primal(self): + self.run_solver(options={"algorithm": "ps"}) + self.assertIn("NOTE: The Primal Simplex algorithm is used.", self.opt_sas._log) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_ipm(self): + self.run_solver(options={"algorithm": "ip"}) + self.assertIn("NOTE: The Interior Point algorithm is used.", self.opt_sas._log) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_intoption(self): + self.run_solver(options={"maxiter": 20}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_invalidoption(self): + with self.assertRaisesRegex(ValueError, "syntax error"): + self.run_solver(options={"foo": "bar"}) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_max(self): + X = self.instance.X + self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) + self.instance.Obj.sense = maximize + self.run_solver() + self.checkSolution() + self.assertEqual(self.results.problem.sense, ProblemSense.maximize) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible(self): + instance = self.instance + X = instance.X + instance.R4 = Constraint(expr=-2 * X[2] - 3 * X[3] <= -6) + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + self.assertEqual(results.solver.message, "The problem is infeasible.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible_or_unbounded(self): + self.instance.X.domain = Reals + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, + TerminationCondition.infeasibleOrUnbounded, + ) + self.assertEqual( + results.solver.message, "The problem is infeasible or unbounded." + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_unbounded(self): + self.instance.X.domain = Reals + self.run_solver(options={"presolver": "none", "algorithm": "primal"}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.unbounded + ) + self.assertEqual(results.solver.message, "The problem is unbounded.") + + def checkSolutionDecomp(self): + instance = self.instance + results = self.results + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7.5) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 2.5) + self.assertAlmostEqual(instance.X[3].value, 0.0) + + # Check reduced cost + self.assertAlmostEqual(instance.rc[instance.X[1]], sense * 2.0) + self.assertAlmostEqual(instance.rc[instance.X[2]], sense * 0.0) + self.assertAlmostEqual(instance.rc[instance.X[3]], sense * 0.5) + + # Check slack + self.assertAlmostEqual(instance.slack[instance.R1], -5.0) + self.assertAlmostEqual(instance.slack[instance.R2], 2.5) + self.assertAlmostEqual(instance.slack[instance.R3], 5.0) + + # Check dual solution + self.assertAlmostEqual(instance.dual[instance.R1], sense * 1.5) + self.assertAlmostEqual(instance.dual[instance.R2], sense * 0.0) + self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) + + # Don't check basis status for decomp + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_decomp(self): + self.run_solver( + options={ + "decomp": {"absobjgap": 0.0}, + "decompmaster": {"algorithm": "dual"}, + "decompsubprob": {"presolver": "none"}, + } + ) + self.assertIn( + "NOTE: The DECOMP method value DEFAULT is applied.", self.opt_sas._log + ) + self.checkSolutionDecomp() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_iis(self): + self.run_solver(options={"iis": "true"}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.feasible + ) + self.assertIn("NOTE: The IIS= option is enabled.", self.opt_sas._log) + self.assertEqual( + results.solver.message, + "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxiter(self): + self.run_solver(options={"maxiter": 1}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxIterations + ) + self.assertEqual( + results.solver.message, + "The maximum allowable number of iterations was reached.", + ) + + +class SASTestLPCAS(SASTestLP): + solver_io = '_sascas' + base_options = CAS_OPTIONS + + +class SASTestMILP(SASTestAbc, unittest.TestCase): + def setX(self): + self.instance.X = Var([1, 2, 3], within=NonNegativeIntegers) + + def checkSolution(self): + instance = self.instance + results = self.results + + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 1.0) + self.assertAlmostEqual(instance.X[3].value, 1.0) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_default(self): + self.run_solver(options={}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_presolve(self): + self.run_solver(options={"presolver": "none"}) + self.assertIn( + "NOTE: The MILP presolver value NONE is applied.", self.opt_sas._log + ) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_intoption(self): + self.run_solver(options={"maxnodes": 20}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_invalidoption(self): + with self.assertRaisesRegex(ValueError, "syntax error"): + self.run_solver(options={"foo": "bar"}) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_max(self): + X = self.instance.X + self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) + self.instance.Obj.sense = maximize + self.run_solver() + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible(self): + instance = self.instance + X = instance.X + instance.R4 = Constraint(expr=-2 * X[2] - 3 * X[3] <= -6) + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + self.assertEqual(results.solver.message, "The problem is infeasible.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + @unittest.skip("Returns wrong status for some versions.") + def test_solver_infeasible_or_unbounded(self): + self.instance.X.domain = Integers + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, + TerminationCondition.infeasibleOrUnbounded, + ) + self.assertEqual( + results.solver.message, "The problem is infeasible or unbounded." + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_unbounded(self): + self.instance.X.domain = Integers + self.run_solver( + options={"presolver": "none", "rootnode": {"algorithm": "primal"}} + ) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.unbounded + ) + self.assertEqual(results.solver.message, "The problem is unbounded.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_decomp(self): + self.run_solver( + options={ + "decomp": {"hybrid": "off"}, + "decompmaster": {"algorithm": "dual"}, + "decompmasterip": {"presolver": "none"}, + "decompsubprob": {"presolver": "none"}, + } + ) + self.assertIn( + "NOTE: The DECOMP method value DEFAULT is applied.", self.opt_sas._log + ) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_rootnode(self): + self.run_solver(options={"rootnode": {"presolver": "automatic"}}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxnodes(self): + self.run_solver(options={"maxnodes": 0}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxEvaluations + ) + self.assertEqual( + results.solver.message, + "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxsols(self): + self.run_solver(options={"maxsols": 1}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxEvaluations + ) + self.assertEqual( + results.solver.message, + "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_target(self): + self.run_solver(options={"target": -6.0}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + self.assertEqual( + results.solver.message, + "The solution is not worse than the target specified by the TARGET= option.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_primalin(self): + X = self.instance.X + X[1] = None + X[2] = 3 + X[3] = 7 + self.run_solver(warmstart=True) + self.checkSolution() + self.assertIn( + "NOTE: The input solution is infeasible or incomplete. Repair heuristics are applied.", + self.opt_sas._log, + ) + + +class SASTestMILPCAS(SASTestMILP): + solver_io = '_sascas' + base_options = CAS_OPTIONS + + +if __name__ == '__main__': + unittest.main() From bc0e1feb794f553d0e7a859120aeeabd297708e9 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 23 Jun 2023 09:56:54 -0400 Subject: [PATCH 004/150] Just some black adjustments --- pyomo/solvers/plugins/solvers/SAS.py | 3 ++- pyomo/solvers/tests/checks/test_SAS.py | 11 ++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 7f50b7a2970..ed0e63d44d6 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -427,7 +427,8 @@ def _apply_solver(self): # Prepare the solver results results = self.results = self._create_results_from_status( - self._macro.get("STATUS", "ERROR"), self._macro.get("SOLUTION_STATUS","ERROR") + self._macro.get("STATUS", "ERROR"), + self._macro.get("SOLUTION_STATUS", "ERROR"), ) if "Objective Sense Maximization" in self._lst: diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 4592343b17f..654820f5060 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -13,15 +13,8 @@ minimize, Suffix, ) -from pyomo.opt.results import ( - SolverStatus, - TerminationCondition, - ProblemSense, -) -from pyomo.opt import ( - SolverFactory, - check_available_solvers, -) +from pyomo.opt.results import SolverStatus, TerminationCondition, ProblemSense +from pyomo.opt import SolverFactory, check_available_solvers CAS_OPTIONS = { From cda764d85ff7b37d72f8206c599f1592a0b477de Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 7 Aug 2023 09:44:00 -0400 Subject: [PATCH 005/150] Updated version for remote testing --- .github/workflows/typos.toml | 2 + .gitignore | 5 +- pyomo/solvers/plugins/solvers/SAS.py | 257 +++++++++++++++++-------- pyomo/solvers/tests/checks/test_SAS.py | 183 ++++++++++++------ 4 files changed, 307 insertions(+), 140 deletions(-) diff --git a/.github/workflows/typos.toml b/.github/workflows/typos.toml index c9fe9e804a2..71d9ad0355f 100644 --- a/.github/workflows/typos.toml +++ b/.github/workflows/typos.toml @@ -38,4 +38,6 @@ caf = "caf" WRONLY = "WRONLY" # Ignore the name Hax Hax = "Hax" +# Ignore dout (short for dual output in SAS solvers) +dout = "dout" # AS NEEDED: Add More Words Below diff --git a/.gitignore b/.gitignore index 09069552990..7309ff1e8a8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ .spyder* .ropeproject .vscode +.env +venv +sascfg_personal.py # Python generates numerous files when byte compiling / installing packages *.pyx *.pyc @@ -24,4 +27,4 @@ gurobi.log # Jupyterhub/Jupyterlab checkpoints .ipynb_checkpoints -cplex.log \ No newline at end of file +cplex.log diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index ed0e63d44d6..87a6a18c1f4 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -1,8 +1,9 @@ -__all__ = ['SAS'] +__all__ = ["SAS"] import logging import sys -import os +from os import stat +import uuid from io import StringIO from abc import ABC, abstractmethod @@ -24,7 +25,7 @@ from pyomo.core.kernel.block import IBlock -logger = logging.getLogger('pyomo.solvers') +logger = logging.getLogger("pyomo.solvers") STATUS_TO_SOLVERSTATUS = { @@ -112,20 +113,20 @@ ] -@SolverFactory.register('sas', doc='The SAS LP/MIP solver') +@SolverFactory.register("sas", doc="The SAS LP/MIP solver") class SAS(OptSolver): """The SAS optimization solver""" def __new__(cls, *args, **kwds): - mode = kwds.pop('solver_io', None) + mode = kwds.pop("solver_io", None) if mode != None: return SolverFactory(mode) else: # Choose solver factory automatically - # bassed on what can be loaded. - s = SolverFactory('_sas94', **kwds) + # based on what can be loaded. + s = SolverFactory("_sas94", **kwds) if not s.available(): - s = SolverFactory('_sascas', **kwds) + s = SolverFactory("_sascas", **kwds) return s @@ -134,7 +135,7 @@ class SASAbc(ABC, OptSolver): def __init__(self, **kwds): """Initialize the SAS solver interfaces.""" - kwds['type'] = 'sas' + kwds["type"] = "sas" super(SASAbc, self).__init__(**kwds) # @@ -157,7 +158,7 @@ def _presolve(self, *args, **kwds): TempfileManager.push() # Get the warmstart flag - self.warmstart_flag = kwds.pop('warmstart', False) + self.warmstart_flag = kwds.pop("warmstart", False) # Call parent presolve function super(SASAbc, self)._presolve(*args, **kwds) @@ -173,7 +174,7 @@ def _presolve(self, *args, **kwds): Var, active=True, descend_into=False ): self._vars.append(vardata) - # Store the symbal map, we need this for example when writing the warmstart file + # Store the symbol map, we need this for example when writing the warmstart file if isinstance(self._instance, IBlock): self._smap = getattr(self._instance, "._symbol_maps")[self._smap_id] else: @@ -186,8 +187,8 @@ def _presolve(self, *args, **kwds): ) smap = self._smap numWritten = 0 - with open(filename, 'w') as file: - file.write('_VAR_,_VALUE_\n') + with open(filename, "w") as file: + file.write("_VAR_,_VALUE_\n") for var in self._vars: if (var.value is not None) and (id(var) in smap.byObject): name = smap.byObject[id(var)] @@ -239,8 +240,7 @@ def _create_results_from_status(self, status, solution_status): @abstractmethod def _apply_solver(self): - """The routine that performs the solve""" - raise NotImplemented("This is an abstract function and thus not implemented!") + pass def _postsolve(self): """Clean up at the end, especially the temp files.""" @@ -260,7 +260,7 @@ def warm_start_capable(self): return True -@SolverFactory.register('_sas94', doc='SAS 9.4 interface') +@SolverFactory.register("_sas94", doc="SAS 9.4 interface") class SAS94(SASAbc): """ Solver interface for SAS 9.4 using saspy. See the saspy documentation about @@ -298,6 +298,9 @@ def _create_statement_str(self, statement): else: return "" + def sas_version(self): + return self._sasver + def _apply_solver(self): """ "Prepare the options and run the solver. Then store the data to be returned.""" logger.debug("Running SAS") @@ -326,39 +329,81 @@ def _apply_solver(self): decompsubprob_str = self._create_statement_str("decompsubprob") rootnode_str = self._create_statement_str("rootnode") + # Get a unique identifier, always use the same with different prefixes + unique = uuid.uuid4().hex[:16] + + # Create unique filename for output datasets + primalout_dataset_name = "pout" + unique + dualout_dataset_name = "dout" + unique + primalin_dataset_name = None + # Handle warmstart warmstart_str = "" if self.warmstart_flag: # Set the warmstart basis option + primalin_dataset_name = "pin" + unique if proc != "OPTLP": warmstart_str = """ proc import datafile='{primalin}' - out=primalin + out={primalin_dataset_name} dbms=csv replace; getnames=yes; run; """.format( - primalin=self._warm_start_file_name + primalin=self._warm_start_file_name, + primalin_dataset_name=primalin_dataset_name, ) - self.options["primalin"] = "primalin" + self.options["primalin"] = primalin_dataset_name # Convert options to string opt_str = " ".join( option + "=" + str(value) for option, value in self.options.items() ) - # Start a SAS session, submit the code and return the results`` + # Set some SAS options to make the log more clean + sas_options = "option notes nonumber nodate nosource pagesize=max;" + + # Start a SAS session, submit the code and return the results with self._sas.SASsession() as sas: # Find the version of 9.4 we are using - if sas.sasver.startswith("9.04.01M5"): + self._sasver = sas.sasver + + # Upload files, only if not accessible locally + upload_mps = False + if not sas.file_info(self._problem_files[0], quiet=True): + sas.upload( + self._problem_files[0], self._problem_files[0], overwrite=True + ) + upload_mps = True + + upload_pin = False + if self.warmstart_flag and not sas.file_info( + self._warm_start_file_name, quiet=True + ): + sas.upload( + self._warm_start_file_name, + self._warm_start_file_name, + overwrite=True, + ) + upload_pin = True + + # Using a function call to make it easier to moch the version check + version = self.sas_version().split("M", 1)[1][0] + if int(version) < 5: + raise NotImplementedError( + "Support for SAS 9.4 M4 and earlier is no implemented." + ) + elif int(version) == 5: # In 9.4M5 we have to create an MPS data set from an MPS file first # Earlier versions will not work because the MPS format in incompatible + mps_dataset_name = "mps" + unique res = sas.submit( """ + {sas_options} {warmstart} - %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA=mpsdata, MAXLEN=256, FORMAT=FREE); - proc {proc} data=mpsdata {options} primalout=primalout dualout=dualout; + %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA={mps_dataset_name}, MAXLEN=256, FORMAT=FREE); + proc {proc} data={mps_dataset_name} {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; {decomp} {decompmaster} {decompmasterip} @@ -366,10 +411,14 @@ def _apply_solver(self): {rootnode} run; """.format( + sas_options=sas_options, warmstart=warmstart_str, proc=proc, mpsfile=self._problem_files[0], + mps_dataset_name=mps_dataset_name, options=opt_str, + primalout_dataset_name=primalout_dataset_name, + dualout_dataset_name=dualout_dataset_name, decomp=decomp_str, decompmaster=decompmaster_str, decompmasterip=decompmasterip_str, @@ -378,12 +427,14 @@ def _apply_solver(self): ), results="TEXT", ) + sas.sasdata(mps_dataset_name).delete(quiet=True) else: # Since 9.4M6+ optlp/optmilp can read mps files directly res = sas.submit( """ + {sas_options} {warmstart} - proc {proc} mpsfile=\"{mpsfile}\" {options} primalout=primalout dualout=dualout; + proc {proc} mpsfile=\"{mpsfile}\" {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; {decomp} {decompmaster} {decompmasterip} @@ -391,10 +442,13 @@ def _apply_solver(self): {rootnode} run; """.format( + sas_options=sas_options, warmstart=warmstart_str, proc=proc, mpsfile=self._problem_files[0], options=opt_str, + primalout_dataset_name=primalout_dataset_name, + dualout_dataset_name=dualout_dataset_name, decomp=decomp_str, decompmaster=decompmaster_str, decompmasterip=decompmasterip_str, @@ -404,26 +458,40 @@ def _apply_solver(self): results="TEXT", ) + # Delete uploaded file + if upload_mps: + sas.file_delete(self._problem_files[0], quiet=True) + if self.warmstart_flag and upload_pin: + sas.file_delete(self._warm_start_file_name, quiet=True) + # Store log and ODS output self._log = res["LOG"] self._lst = res["LST"] - # Print log if requested by the user - if self._tee: - print(self._log) if "ERROR 22-322: Syntax error" in self._log: raise ValueError( "An option passed to the SAS solver caused a syntax error: {log}".format( log=self._log ) ) + else: + # Print log if requested by the user, only if we did not already print it + if self._tee: + print(self._log) self._macro = dict( (key.strip(), value.strip()) for key, value in ( pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() ) ) - primal_out = sas.sd2df("primalout") - dual_out = sas.sd2df("dualout") + if self._macro.get("STATUS", "ERROR") == "OK": + primal_out = sas.sd2df(primalout_dataset_name) + dual_out = sas.sd2df(dualout_dataset_name) + + # Delete data sets, they will go away automatically, but does not hurt to delete them + if primalin_dataset_name: + sas.sasdata(primalin_dataset_name).delete(quiet=True) + sas.sasdata(primalout_dataset_name).delete(quiet=True) + sas.sasdata(dualout_dataset_name).delete(quiet=True) # Prepare the solver results results = self.results = self._create_results_from_status( @@ -445,35 +513,35 @@ def _apply_solver(self): sol.termination_condition = TerminationCondition.optimal # Store objective value in solution - sol.objective['__default_objective__'] = {'Value': self._macro["OBJECTIVE"]} + sol.objective["__default_objective__"] = {"Value": self._macro["OBJECTIVE"]} if proc == "OPTLP": # Convert primal out data set to variable dictionary # Use panda functions for efficiency - primal_out = primal_out[['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_']] - primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] + primal_out = primal_out.set_index("_VAR_", drop=True) primal_out = primal_out.rename( - {'_VALUE_': 'Value', '_STATUS_': 'Status', '_R_COST_': 'rc'}, - axis='columns', + {"_VALUE_": "Value", "_STATUS_": "Status", "_R_COST_": "rc"}, + axis="columns", ) - sol.variable = primal_out.to_dict('index') + sol.variable = primal_out.to_dict("index") # Convert dual out data set to constraint dictionary # Use pandas functions for efficiency - dual_out = dual_out[['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_']] - dual_out = dual_out.set_index('_ROW_', drop=True) + dual_out = dual_out[["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"]] + dual_out = dual_out.set_index("_ROW_", drop=True) dual_out = dual_out.rename( - {'_VALUE_': 'dual', '_STATUS_': 'Status', '_ACTIVITY_': 'slack'}, - axis='columns', + {"_VALUE_": "dual", "_STATUS_": "Status", "_ACTIVITY_": "slack"}, + axis="columns", ) - sol.constraint = dual_out.to_dict('index') + sol.constraint = dual_out.to_dict("index") else: # Convert primal out data set to variable dictionary # Use pandas functions for efficiency - primal_out = primal_out[['_VAR_', '_VALUE_']] - primal_out = primal_out.set_index('_VAR_', drop=True) - primal_out = primal_out.rename({'_VALUE_': 'Value'}, axis='columns') - sol.variable = primal_out.to_dict('index') + primal_out = primal_out[["_VAR_", "_VALUE_"]] + primal_out = primal_out.set_index("_VAR_", drop=True) + primal_out = primal_out.rename({"_VALUE_": "Value"}, axis="columns") + sol.variable = primal_out.to_dict("index") self._rc = 0 return Bunch(rc=self._rc, log=self._log) @@ -504,7 +572,7 @@ def log(self): return self._log.getvalue() -@SolverFactory.register('_sascas', doc='SAS Viya CAS Server interface') +@SolverFactory.register("_sascas", doc="SAS Viya CAS Server interface") class SASCAS(SASAbc): """ Solver interface connection to a SAS Viya CAS server using swat. @@ -553,70 +621,89 @@ def _apply_solver(self): # Check if there are integer variables, this might be slow action = "solveMilp" if self._has_integer_variables() else "solveLp" + # Get a unique identifier, always use the same with different prefixes + unique = uuid.uuid4().hex[:16] + # Connect to CAS server with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: s = self._sas.CAS(**cas_opts) try: # Load the optimization action set - s.loadactionset('optimization') + s.loadactionset("optimization") + + # Declare a unique table name for the mps table + mpsdata_table_name = "mps" + unique # Upload mps file to CAS - if os.stat(self._problem_files[0]).st_size >= 2 * 1024**3: - # For large files, use convertMPS, first create file for upload + if stat(self._problem_files[0]).st_size >= 2 * 1024**3: + # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). + # Use convertMPS, first create file for upload. mpsWithIdFileName = TempfileManager.create_tempfile( ".mps.csv", text=True ) - with open(mpsWithIdFileName, 'w') as mpsWithId: - mpsWithId.write('_ID_\tText\n') - with open(self._problem_files[0], 'r') as f: + with open(mpsWithIdFileName, "w") as mpsWithId: + mpsWithId.write("_ID_\tText\n") + with open(self._problem_files[0], "r") as f: id = 0 for line in f: id += 1 - mpsWithId.write(str(id) + '\t' + line.rstrip() + '\n') + mpsWithId.write(str(id) + "\t" + line.rstrip() + "\n") # Upload .mps.csv file + mpscsv_table_name = "csv" + unique s.upload_file( mpsWithIdFileName, - casout={"name": "mpscsv", "replace": True}, + casout={"name": mpscsv_table_name, "replace": True}, importoptions={"filetype": "CSV", "delimiter": "\t"}, ) # Convert .mps.csv file to .mps s.optimization.convertMps( - data="mpscsv", - casOut={"name": "mpsdata", "replace": True}, + data=mpscsv_table_name, + casOut={"name": mpsdata_table_name, "replace": True}, format="FREE", ) + + # Delete the table we don't need anymore + if mpscsv_table_name: + s.dropTable(name=mpscsv_table_name, quiet=True) else: - # For small files, use loadMPS - with open(self._problem_files[0], 'r') as mps_file: + # For small files (less than 2 GB), use loadMps + with open(self._problem_files[0], "r") as mps_file: s.optimization.loadMps( mpsFileString=mps_file.read(), - casout={"name": "mpsdata", "replace": True}, + casout={"name": mpsdata_table_name, "replace": True}, format="FREE", ) + primalin_table_name = None if self.warmstart_flag: + primalin_table_name = "pin" + unique # Upload warmstart file to CAS s.upload_file( self._warm_start_file_name, - casout={"name": "primalin", "replace": True}, + casout={"name": primalin_table_name, "replace": True}, importoptions={"filetype": "CSV"}, ) - self.options["primalin"] = "primalin" + self.options["primalin"] = primalin_table_name + + # Define output table names + primalout_table_name = "pout" + unique + dualout_table_name = None # Solve the problem in CAS if action == "solveMilp": r = s.optimization.solveMilp( - data={"name": "mpsdata"}, - primalOut={"name": "primalout", "replace": True}, + data={"name": mpsdata_table_name}, + primalOut={"name": primalout_table_name, "replace": True}, **self.options ) else: + dualout_table_name = "dout" + unique r = s.optimization.solveLp( - data={"name": "mpsdata"}, - primalOut={"name": "primalout", "replace": True}, - dualOut={"name": "dualout", "replace": True}, + data={"name": mpsdata_table_name}, + primalOut={"name": primalout_table_name, "replace": True}, + dualOut={"name": dualout_table_name, "replace": True}, **self.options ) @@ -644,44 +731,44 @@ def _apply_solver(self): sol.termination_condition = TerminationCondition.optimal # Store objective value in solution - sol.objective['__default_objective__'] = { - 'Value': r["objective"] + sol.objective["__default_objective__"] = { + "Value": r["objective"] } if action == "solveMilp": - primal_out = s.CASTable(name="primalout") + primal_out = s.CASTable(name=primalout_table_name) # Use pandas functions for efficiency - primal_out = primal_out[['_VAR_', '_VALUE_']] + primal_out = primal_out[["_VAR_", "_VALUE_"]] sol.variable = {} for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = {'Value': row[1]} + sol.variable[row[0]] = {"Value": row[1]} else: # Convert primal out data set to variable dictionary # Use panda functions for efficiency - primal_out = s.CASTable(name="primalout") + primal_out = s.CASTable(name=primalout_table_name) primal_out = primal_out[ - ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] + ["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"] ] sol.variable = {} for row in primal_out.itertuples(index=False): sol.variable[row[0]] = { - 'Value': row[1], - 'Status': row[2], - 'rc': row[3], + "Value": row[1], + "Status": row[2], + "rc": row[3], } # Convert dual out data set to constraint dictionary # Use pandas functions for efficiency - dual_out = s.CASTable(name="dualout") + dual_out = s.CASTable(name=dualout_table_name) dual_out = dual_out[ - ['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_'] + ["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"] ] sol.constraint = {} for row in dual_out.itertuples(index=False): sol.constraint[row[0]] = { - 'dual': row[1], - 'Status': row[2], - 'slack': row[3], + "dual": row[1], + "Status": row[2], + "slack": row[3], } else: results = self.results = SolverResults() @@ -692,10 +779,16 @@ def _apply_solver(self): ) finally: + if mpsdata_table_name: + s.dropTable(name=mpsdata_table_name, quiet=True) + if primalin_table_name: + s.dropTable(name=primalin_table_name, quiet=True) + if primalout_table_name: + s.dropTable(name=primalout_table_name, quiet=True) + if dualout_table_name: + s.dropTable(name=dualout_table_name, quiet=True) s.close() self._log = self._log_writer.log() - if self._tee: - print(self._log) self._rc = 0 return Bunch(rc=self._rc, log=self._log) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 654820f5060..3a63e258600 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -1,5 +1,6 @@ import os import pyomo.common.unittest as unittest +from unittest import mock from pyomo.environ import ( ConcreteModel, Var, @@ -15,20 +16,20 @@ ) from pyomo.opt.results import SolverStatus, TerminationCondition, ProblemSense from pyomo.opt import SolverFactory, check_available_solvers - +import warnings CAS_OPTIONS = { - "hostname": os.environ.get('CAS_SERVER', None), - "port": os.environ.get('CAS_PORT', None), - "authinfo": os.environ.get('CAS_AUTHINFO', None), + "hostname": os.environ.get("CASHOST", None), + "port": os.environ.get("CASPORT", None), + "authinfo": os.environ.get("CASAUTHINFO", None), } -sas_available = check_available_solvers('sas') +sas_available = check_available_solvers("sas") class SASTestAbc: - solver_io = '_sas94' + solver_io = "_sas94" base_options = {} def setObj(self): @@ -41,6 +42,8 @@ def setX(self): self.instance.X = Var([1, 2, 3], within=NonNegativeReals) def setUp(self): + # Disable resource warnings + warnings.filterwarnings("ignore", category=ResourceWarning) instance = self.instance = ConcreteModel() self.setX() X = instance.X @@ -55,7 +58,7 @@ def setUp(self): instance.rc = Suffix(direction=Suffix.IMPORT) instance.dual = Suffix(direction=Suffix.IMPORT) - self.opt_sas = SolverFactory('sas', solver_io=self.solver_io) + self.opt_sas = SolverFactory("sas", solver_io=self.solver_io) def tearDown(self): del self.opt_sas @@ -74,7 +77,7 @@ def run_solver(self, **kwargs): self.results = opt_sas.solve(instance, **kwargs) -class SASTestLP(SASTestAbc, unittest.TestCase): +class SASTestLP(SASTestAbc): def checkSolution(self): instance = self.instance results = self.results @@ -111,41 +114,39 @@ def checkSolution(self): self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) # Check basis status - self.assertEqual(instance.status[instance.X[1]], 'L') - self.assertEqual(instance.status[instance.X[2]], 'B') - self.assertEqual(instance.status[instance.X[3]], 'L') - self.assertEqual(instance.status[instance.R1], 'U') - self.assertEqual(instance.status[instance.R2], 'B') - self.assertEqual(instance.status[instance.R3], 'B') - - @unittest.skipIf(not sas_available, "The SAS solver is not available") + self.assertEqual(instance.status[instance.X[1]], "L") + self.assertEqual(instance.status[instance.X[2]], "B") + self.assertEqual(instance.status[instance.X[3]], "L") + self.assertEqual(instance.status[instance.R1], "U") + self.assertEqual(instance.status[instance.R2], "B") + self.assertEqual(instance.status[instance.R3], "B") + def test_solver_default(self): self.run_solver() self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_tee(self): + self.run_solver(tee=True) + self.checkSolution() + def test_solver_primal(self): self.run_solver(options={"algorithm": "ps"}) self.assertIn("NOTE: The Primal Simplex algorithm is used.", self.opt_sas._log) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_ipm(self): self.run_solver(options={"algorithm": "ip"}) self.assertIn("NOTE: The Interior Point algorithm is used.", self.opt_sas._log) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_intoption(self): self.run_solver(options={"maxiter": 20}) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_invalidoption(self): with self.assertRaisesRegex(ValueError, "syntax error"): self.run_solver(options={"foo": "bar"}) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_max(self): X = self.instance.X self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) @@ -154,7 +155,6 @@ def test_solver_max(self): self.checkSolution() self.assertEqual(self.results.problem.sense, ProblemSense.maximize) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_infeasible(self): instance = self.instance X = instance.X @@ -167,21 +167,23 @@ def test_solver_infeasible(self): ) self.assertEqual(results.solver.message, "The problem is infeasible.") - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_infeasible_or_unbounded(self): self.instance.X.domain = Reals self.run_solver() results = self.results self.assertEqual(results.solver.status, SolverStatus.warning) - self.assertEqual( + self.assertIn( results.solver.termination_condition, - TerminationCondition.infeasibleOrUnbounded, + [ + TerminationCondition.infeasibleOrUnbounded, + TerminationCondition.unbounded, + ], ) - self.assertEqual( - results.solver.message, "The problem is infeasible or unbounded." + self.assertIn( + results.solver.message, + ["The problem is infeasible or unbounded.", "The problem is unbounded."], ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_unbounded(self): self.instance.X.domain = Reals self.run_solver(options={"presolver": "none", "algorithm": "primal"}) @@ -229,7 +231,6 @@ def checkSolutionDecomp(self): # Don't check basis status for decomp - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_decomp(self): self.run_solver( options={ @@ -243,7 +244,6 @@ def test_solver_decomp(self): ) self.checkSolutionDecomp() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_iis(self): self.run_solver(options={"iis": "true"}) results = self.results @@ -257,7 +257,6 @@ def test_solver_iis(self): "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_maxiter(self): self.run_solver(options={"maxiter": 1}) results = self.results @@ -270,13 +269,59 @@ def test_solver_maxiter(self): "The maximum allowable number of iterations was reached.", ) + def test_solver_with_milp(self): + self.run_solver(options={"with": "milp"}) + self.assertIn( + "WARNING: The problem has no integer variables.", self.opt_sas._log + ) + -class SASTestLPCAS(SASTestLP): - solver_io = '_sascas' +@unittest.skipIf(not sas_available, "The SAS solver is not available") +class SASTestLP94(SASTestLP, unittest.TestCase): + @mock.patch( + "pyomo.solvers.plugins.solvers.SAS.SAS94.sas_version", + return_value="2sd45s39M4234232", + ) + def test_solver_versionM4(self, sas): + with self.assertRaises(NotImplementedError): + self.run_solver() + + @mock.patch( + "pyomo.solvers.plugins.solvers.SAS.SAS94.sas_version", + return_value="234897293M5324u98", + ) + def test_solver_versionM5(self, sas): + self.run_solver() + self.checkSolution() + + @mock.patch("saspy.SASsession.submit", return_value={"LOG": "", "LST": ""}) + @mock.patch("saspy.SASsession.symget", return_value="STATUS=OUT_OF_MEMORY") + def test_solver_out_of_memory(self, submit_mock, symget_mocks): + self.run_solver(load_solutions=False) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.aborted) + + @mock.patch("saspy.SASsession.submit", return_value={"LOG": "", "LST": ""}) + @mock.patch("saspy.SASsession.symget", return_value="STATUS=ERROR") + def test_solver_error(self, submit_mock, symget_mock): + self.run_solver(load_solutions=False) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.error) + + +@unittest.skipIf(not sas_available, "The SAS solver is not available") +class SASTestLPCAS(SASTestLP, unittest.TestCase): + solver_io = "_sascas" base_options = CAS_OPTIONS + @mock.patch("pyomo.solvers.plugins.solvers.SAS.stat") + def test_solver_large_file(self, os_stat): + os_stat.return_value.st_size = 3 * 1024**3 + self.run_solver() + self.checkSolution() + -class SASTestMILP(SASTestAbc, unittest.TestCase): +class SASTestMILP(SASTestAbc): def setX(self): self.instance.X = Var([1, 2, 3], within=NonNegativeIntegers) @@ -301,12 +346,14 @@ def checkSolution(self): self.assertAlmostEqual(instance.X[2].value, 1.0) self.assertAlmostEqual(instance.X[3].value, 1.0) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_default(self): - self.run_solver(options={}) + self.run_solver() + self.checkSolution() + + def test_solver_tee(self): + self.run_solver(tee=True) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_presolve(self): self.run_solver(options={"presolver": "none"}) self.assertIn( @@ -314,17 +361,14 @@ def test_solver_presolve(self): ) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_intoption(self): self.run_solver(options={"maxnodes": 20}) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_invalidoption(self): with self.assertRaisesRegex(ValueError, "syntax error"): self.run_solver(options={"foo": "bar"}) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_max(self): X = self.instance.X self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) @@ -332,7 +376,6 @@ def test_solver_max(self): self.run_solver() self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_infeasible(self): instance = self.instance X = instance.X @@ -345,22 +388,23 @@ def test_solver_infeasible(self): ) self.assertEqual(results.solver.message, "The problem is infeasible.") - @unittest.skipIf(not sas_available, "The SAS solver is not available") - @unittest.skip("Returns wrong status for some versions.") def test_solver_infeasible_or_unbounded(self): self.instance.X.domain = Integers self.run_solver() results = self.results self.assertEqual(results.solver.status, SolverStatus.warning) - self.assertEqual( + self.assertIn( results.solver.termination_condition, - TerminationCondition.infeasibleOrUnbounded, + [ + TerminationCondition.infeasibleOrUnbounded, + TerminationCondition.unbounded, + ], ) - self.assertEqual( - results.solver.message, "The problem is infeasible or unbounded." + self.assertIn( + results.solver.message, + ["The problem is infeasible or unbounded.", "The problem is unbounded."], ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_unbounded(self): self.instance.X.domain = Integers self.run_solver( @@ -373,7 +417,6 @@ def test_solver_unbounded(self): ) self.assertEqual(results.solver.message, "The problem is unbounded.") - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_decomp(self): self.run_solver( options={ @@ -388,12 +431,10 @@ def test_solver_decomp(self): ) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_rootnode(self): self.run_solver(options={"rootnode": {"presolver": "automatic"}}) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_maxnodes(self): self.run_solver(options={"maxnodes": 0}) results = self.results @@ -406,7 +447,6 @@ def test_solver_maxnodes(self): "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_maxsols(self): self.run_solver(options={"maxsols": 1}) results = self.results @@ -419,7 +459,6 @@ def test_solver_maxsols(self): "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_target(self): self.run_solver(options={"target": -6.0}) results = self.results @@ -432,7 +471,6 @@ def test_solver_target(self): "The solution is not worse than the target specified by the TARGET= option.", ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_primalin(self): X = self.instance.X X[1] = None @@ -445,11 +483,42 @@ def test_solver_primalin(self): self.opt_sas._log, ) + def test_solver_primalin_nosol(self): + X = self.instance.X + X[1] = None + X[2] = None + X[3] = None + self.run_solver(warmstart=True) + self.checkSolution() + + @mock.patch("pyomo.solvers.plugins.solvers.SAS.stat") + def test_solver_large_file(self, os_stat): + os_stat.return_value.st_size = 3 * 1024**3 + self.run_solver() + self.checkSolution() + + def test_solver_with_lp(self): + self.run_solver(options={"with": "lp"}) + self.assertIn( + "contains integer variables; the linear relaxation will be solved.", + self.opt_sas._log, + ) + + def test_solver_warmstart_capable(self): + self.run_solver() + self.assertTrue(self.opt_sas.warm_start_capable()) + + +@unittest.skipIf(not sas_available, "The SAS solver is not available") +class SASTestMILP94(SASTestMILP, unittest.TestCase): + pass + -class SASTestMILPCAS(SASTestMILP): - solver_io = '_sascas' +@unittest.skipIf(not sas_available, "The SAS solver is not available") +class SASTestMILPCAS(SASTestMILP, unittest.TestCase): + solver_io = "_sascas" base_options = CAS_OPTIONS -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 4b6298da177aa6c16a1bf581dc6e0508e9ee6084 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 7 Aug 2023 11:26:10 -0400 Subject: [PATCH 006/150] Change SAS solver interfaces to keep the SAS connection --- pyomo/solvers/plugins/solvers/SAS.py | 266 +++++++++++++------------ pyomo/solvers/tests/checks/test_SAS.py | 11 +- 2 files changed, 149 insertions(+), 128 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 87a6a18c1f4..a5f74849813 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -285,6 +285,14 @@ def __init__(self, **kwds): self._python_api_exists = True self._sas.logger.setLevel(logger.level) + # Create the session only as its needed + self._sas_session = None + + def __del__(self): + # Close the session, if we created one + if self._sas_session: + self._sas_session.endsas() + def _create_statement_str(self, statement): """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" stmt = self.options.pop(statement, None) @@ -364,134 +372,133 @@ def _apply_solver(self): # Set some SAS options to make the log more clean sas_options = "option notes nonumber nodate nosource pagesize=max;" - # Start a SAS session, submit the code and return the results - with self._sas.SASsession() as sas: - # Find the version of 9.4 we are using - self._sasver = sas.sasver - - # Upload files, only if not accessible locally - upload_mps = False - if not sas.file_info(self._problem_files[0], quiet=True): - sas.upload( - self._problem_files[0], self._problem_files[0], overwrite=True - ) - upload_mps = True - - upload_pin = False - if self.warmstart_flag and not sas.file_info( - self._warm_start_file_name, quiet=True - ): - sas.upload( - self._warm_start_file_name, - self._warm_start_file_name, - overwrite=True, - ) - upload_pin = True + # Get the current SAS session, submit the code and return the results + sas = self._sas_session + if sas == None: + sas = self._sas_session = self._sas.SASsession() + + # Find the version of 9.4 we are using + self._sasver = sas.sasver + + # Upload files, only if not accessible locally + upload_mps = False + if not sas.file_info(self._problem_files[0], quiet=True): + sas.upload(self._problem_files[0], self._problem_files[0], overwrite=True) + upload_mps = True + + upload_pin = False + if self.warmstart_flag and not sas.file_info( + self._warm_start_file_name, quiet=True + ): + sas.upload( + self._warm_start_file_name, self._warm_start_file_name, overwrite=True + ) + upload_pin = True - # Using a function call to make it easier to moch the version check - version = self.sas_version().split("M", 1)[1][0] - if int(version) < 5: - raise NotImplementedError( - "Support for SAS 9.4 M4 and earlier is no implemented." - ) - elif int(version) == 5: - # In 9.4M5 we have to create an MPS data set from an MPS file first - # Earlier versions will not work because the MPS format in incompatible - mps_dataset_name = "mps" + unique - res = sas.submit( - """ - {sas_options} - {warmstart} - %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA={mps_dataset_name}, MAXLEN=256, FORMAT=FREE); - proc {proc} data={mps_dataset_name} {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; - {decomp} - {decompmaster} - {decompmasterip} - {decompsubprob} - {rootnode} - run; - """.format( - sas_options=sas_options, - warmstart=warmstart_str, - proc=proc, - mpsfile=self._problem_files[0], - mps_dataset_name=mps_dataset_name, - options=opt_str, - primalout_dataset_name=primalout_dataset_name, - dualout_dataset_name=dualout_dataset_name, - decomp=decomp_str, - decompmaster=decompmaster_str, - decompmasterip=decompmasterip_str, - decompsubprob=decompsubprob_str, - rootnode=rootnode_str, - ), - results="TEXT", - ) - sas.sasdata(mps_dataset_name).delete(quiet=True) - else: - # Since 9.4M6+ optlp/optmilp can read mps files directly - res = sas.submit( - """ - {sas_options} - {warmstart} - proc {proc} mpsfile=\"{mpsfile}\" {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; - {decomp} - {decompmaster} - {decompmasterip} - {decompsubprob} - {rootnode} - run; - """.format( - sas_options=sas_options, - warmstart=warmstart_str, - proc=proc, - mpsfile=self._problem_files[0], - options=opt_str, - primalout_dataset_name=primalout_dataset_name, - dualout_dataset_name=dualout_dataset_name, - decomp=decomp_str, - decompmaster=decompmaster_str, - decompmasterip=decompmasterip_str, - decompsubprob=decompsubprob_str, - rootnode=rootnode_str, - ), - results="TEXT", - ) + # Using a function call to make it easier to moch the version check + version = self.sas_version().split("M", 1)[1][0] + if int(version) < 5: + raise NotImplementedError( + "Support for SAS 9.4 M4 and earlier is no implemented." + ) + elif int(version) == 5: + # In 9.4M5 we have to create an MPS data set from an MPS file first + # Earlier versions will not work because the MPS format in incompatible + mps_dataset_name = "mps" + unique + res = sas.submit( + """ + {sas_options} + {warmstart} + %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA={mps_dataset_name}, MAXLEN=256, FORMAT=FREE); + proc {proc} data={mps_dataset_name} {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + sas_options=sas_options, + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + mps_dataset_name=mps_dataset_name, + options=opt_str, + primalout_dataset_name=primalout_dataset_name, + dualout_dataset_name=dualout_dataset_name, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + sas.sasdata(mps_dataset_name).delete(quiet=True) + else: + # Since 9.4M6+ optlp/optmilp can read mps files directly + res = sas.submit( + """ + {sas_options} + {warmstart} + proc {proc} mpsfile=\"{mpsfile}\" {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + sas_options=sas_options, + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + primalout_dataset_name=primalout_dataset_name, + dualout_dataset_name=dualout_dataset_name, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) - # Delete uploaded file - if upload_mps: - sas.file_delete(self._problem_files[0], quiet=True) - if self.warmstart_flag and upload_pin: - sas.file_delete(self._warm_start_file_name, quiet=True) - - # Store log and ODS output - self._log = res["LOG"] - self._lst = res["LST"] - if "ERROR 22-322: Syntax error" in self._log: - raise ValueError( - "An option passed to the SAS solver caused a syntax error: {log}".format( - log=self._log - ) - ) - else: - # Print log if requested by the user, only if we did not already print it - if self._tee: - print(self._log) - self._macro = dict( - (key.strip(), value.strip()) - for key, value in ( - pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() + # Delete uploaded file + if upload_mps: + sas.file_delete(self._problem_files[0], quiet=True) + if self.warmstart_flag and upload_pin: + sas.file_delete(self._warm_start_file_name, quiet=True) + + # Store log and ODS output + self._log = res["LOG"] + self._lst = res["LST"] + if "ERROR 22-322: Syntax error" in self._log: + raise ValueError( + "An option passed to the SAS solver caused a syntax error: {log}".format( + log=self._log ) ) - if self._macro.get("STATUS", "ERROR") == "OK": - primal_out = sas.sd2df(primalout_dataset_name) - dual_out = sas.sd2df(dualout_dataset_name) + else: + # Print log if requested by the user, only if we did not already print it + if self._tee: + print(self._log) + self._macro = dict( + (key.strip(), value.strip()) + for key, value in ( + pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() + ) + ) + if self._macro.get("STATUS", "ERROR") == "OK": + primal_out = sas.sd2df(primalout_dataset_name) + dual_out = sas.sd2df(dualout_dataset_name) - # Delete data sets, they will go away automatically, but does not hurt to delete them - if primalin_dataset_name: - sas.sasdata(primalin_dataset_name).delete(quiet=True) - sas.sasdata(primalout_dataset_name).delete(quiet=True) - sas.sasdata(dualout_dataset_name).delete(quiet=True) + # Delete data sets, they will go away automatically, but does not hurt to delete them + if primalin_dataset_name: + sas.sasdata(primalin_dataset_name).delete(quiet=True) + sas.sasdata(primalout_dataset_name).delete(quiet=True) + sas.sasdata(dualout_dataset_name).delete(quiet=True) # Prepare the solver results results = self.results = self._create_results_from_status( @@ -597,6 +604,14 @@ def __init__(self, **kwds): else: self._python_api_exists = True + # Create the session only as its needed + self._sas_session = None + + def __del__(self): + # Close the session, if we created one + if self._sas_session: + self._sas_session.close() + def _apply_solver(self): """ "Prepare the options and run the solver. Then store the data to be returned.""" logger.debug("Running SAS Viya") @@ -626,7 +641,9 @@ def _apply_solver(self): # Connect to CAS server with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: - s = self._sas.CAS(**cas_opts) + s = self._sas_session + if s == None: + s = self._sas_session = self._sas.CAS(**cas_opts) try: # Load the optimization action set s.loadactionset("optimization") @@ -787,7 +804,6 @@ def _apply_solver(self): s.dropTable(name=primalout_table_name, quiet=True) if dualout_table_name: s.dropTable(name=dualout_table_name, quiet=True) - s.close() self._log = self._log_writer.log() self._rc = 0 diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 3a63e258600..1a6bbd80f1d 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -32,6 +32,14 @@ class SASTestAbc: solver_io = "_sas94" base_options = {} + @classmethod + def setUpClass(cls): + cls.opt_sas = SolverFactory("sas", solver_io=cls.solver_io) + + @classmethod + def tearDownClass(cls): + del cls.opt_sas + def setObj(self): X = self.instance.X self.instance.Obj = Objective( @@ -58,10 +66,7 @@ def setUp(self): instance.rc = Suffix(direction=Suffix.IMPORT) instance.dual = Suffix(direction=Suffix.IMPORT) - self.opt_sas = SolverFactory("sas", solver_io=self.solver_io) - def tearDown(self): - del self.opt_sas del self.instance def run_solver(self, **kwargs): From 619edf9805639ef2c95088c90f09d3eb5e8972c4 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 7 Aug 2023 14:31:58 -0400 Subject: [PATCH 007/150] Fix formatting issue in comment --- pyomo/solvers/plugins/solvers/SAS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index a5f74849813..f5840b5d6f3 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -152,7 +152,7 @@ def __init__(self, **kwds): super(SASAbc, self).set_problem_format(ProblemFormat.mps) def _presolve(self, *args, **kwds): - """ "Set things up for the actual solve.""" + """Set things up for the actual solve.""" # create a context in the temporary file manager for # this plugin - is "pop"ed in the _postsolve method. TempfileManager.push() From faddef1a12afdc8953a14ce78146a3d8ab7cef89 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 11 Aug 2023 03:23:50 -0400 Subject: [PATCH 008/150] Reset .gitignore to original state. --- .gitignore | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 7309ff1e8a8..09069552990 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,6 @@ .spyder* .ropeproject .vscode -.env -venv -sascfg_personal.py # Python generates numerous files when byte compiling / installing packages *.pyx *.pyc @@ -27,4 +24,4 @@ gurobi.log # Jupyterhub/Jupyterlab checkpoints .ipynb_checkpoints -cplex.log +cplex.log \ No newline at end of file From db18d03e5f4403d014f6fdaaa5a4ca494fee28c5 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Tue, 12 Mar 2024 05:43:11 -0400 Subject: [PATCH 009/150] Add session options, fix non-optimal return codes and version checking --- pyomo/solvers/plugins/solvers/SAS.py | 180 +++++++++++-------------- pyomo/solvers/tests/checks/test_SAS.py | 17 +-- 2 files changed, 87 insertions(+), 110 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index f5840b5d6f3..87ee31a08af 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -96,23 +96,6 @@ } -CAS_OPTION_NAMES = [ - "hostname", - "port", - "username", - "password", - "session", - "locale", - "name", - "nworkers", - "authinfo", - "protocol", - "path", - "ssl_ca_list", - "authcode", -] - - @SolverFactory.register("sas", doc="The SAS LP/MIP solver") class SAS(OptSolver): """The SAS optimization solver""" @@ -120,7 +103,7 @@ class SAS(OptSolver): def __new__(cls, *args, **kwds): mode = kwds.pop("solver_io", None) if mode != None: - return SolverFactory(mode) + return SolverFactory(mode, **kwds) else: # Choose solver factory automatically # based on what can be loaded. @@ -216,6 +199,7 @@ def _create_results_from_status(self, status, solution_status): results = SolverResults() results.solver.name = "SAS" results.solver.status = STATUS_TO_SOLVERSTATUS[status] + results.solver.hasSolution = False if results.solver.status == SolverStatus.ok: results.solver.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ solution_status @@ -226,11 +210,14 @@ def _create_results_from_status(self, status, solution_status): results.solver.status = TerminationCondition.to_solver_status( results.solver.termination_condition ) + if "OPTIMAL" in solution_status or "_SOL" in solution_status: + results.solver.hasSolution = True elif results.solver.status == SolverStatus.aborted: results.solver.termination_condition = TerminationCondition.userInterrupt - results.solver.message = ( - results.solver.termination_message - ) = SOLSTATUS_TO_MESSAGE["ABORTED"] + if solution_status != "ERROR": + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE[solution_status] else: results.solver.termination_condition = TerminationCondition.error results.solver.message = ( @@ -288,6 +275,9 @@ def __init__(self, **kwds): # Create the session only as its needed self._sas_session = None + # Store other options for the SAS session + self._session_options = kwds + def __del__(self): # Close the session, if we created one if self._sas_session: @@ -326,10 +316,6 @@ def _apply_solver(self): # Check if there are integer variables, this might be slow proc = "OPTMILP" if self._has_integer_variables() else "OPTLP" - # Remove CAS options in case they were specified - for opt in CAS_OPTION_NAMES: - self.options.pop(opt, None) - # Get the rootnode options decomp_str = self._create_statement_str("decomp") decompmaster_str = self._create_statement_str("decompmaster") @@ -373,9 +359,7 @@ def _apply_solver(self): sas_options = "option notes nonumber nodate nosource pagesize=max;" # Get the current SAS session, submit the code and return the results - sas = self._sas_session - if sas == None: - sas = self._sas_session = self._sas.SASsession() + sas = self._sas_session = self._sas.SASsession(**self._session_options) # Find the version of 9.4 we are using self._sasver = sas.sasver @@ -396,12 +380,13 @@ def _apply_solver(self): upload_pin = True # Using a function call to make it easier to moch the version check - version = self.sas_version().split("M", 1)[1][0] - if int(version) < 5: + major_version = self.sas_version()[0] + minor_version = self.sas_version().split("M", 1)[1][0] + if major_version == "9" and int(minor_version) < 5: raise NotImplementedError( "Support for SAS 9.4 M4 and earlier is no implemented." ) - elif int(version) == 5: + elif major_version == "9" and int(minor_version) == 5: # In 9.4M5 we have to create an MPS data set from an MPS file first # Earlier versions will not work because the MPS format in incompatible mps_dataset_name = "mps" + unique @@ -436,7 +421,7 @@ def _apply_solver(self): ) sas.sasdata(mps_dataset_name).delete(quiet=True) else: - # Since 9.4M6+ optlp/optmilp can read mps files directly + # Since 9.4M6+ optlp/optmilp can read mps files directly (this includes Viya-based local installs) res = sas.submit( """ {sas_options} @@ -512,12 +497,12 @@ def _apply_solver(self): results.problem.sense = ProblemSense.minimize # Prepare the solution information - if results.solver.termination_condition == TerminationCondition.optimal: + if results.solver.hasSolution: sol = results.solution.add() # Store status in solution sol.status = SolutionStatus.feasible - sol.termination_condition = TerminationCondition.optimal + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[self._macro.get("SOLUTION_STATUS", "ERROR")] # Store objective value in solution sol.objective["__default_objective__"] = {"Value": self._macro["OBJECTIVE"]} @@ -606,6 +591,7 @@ def __init__(self, **kwds): # Create the session only as its needed self._sas_session = None + self._session_options = kwds def __del__(self): # Close the session, if we created one @@ -619,13 +605,6 @@ def _apply_solver(self): # Set return code to issue an error if we get interrupted self._rc = -1 - # Extract CAS connection options - cas_opts = {} - for opt in CAS_OPTION_NAMES: - val = self.options.pop(opt, None) - if val != None: - cas_opts[opt] = val - # Figure out if the problem has integer variables with_opt = self.options.pop("with", None) if with_opt == "lp": @@ -643,7 +622,7 @@ def _apply_solver(self): with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: s = self._sas_session if s == None: - s = self._sas_session = self._sas.CAS(**cas_opts) + s = self._sas_session = self._sas.CAS(**self._session_options) try: # Load the optimization action set s.loadactionset("optimization") @@ -651,8 +630,9 @@ def _apply_solver(self): # Declare a unique table name for the mps table mpsdata_table_name = "mps" + unique - # Upload mps file to CAS - if stat(self._problem_files[0]).st_size >= 2 * 1024**3: + # Upload mps file to CAS, if the file is larger than 2 GB, we need to use convertMps instead of loadMps + # Note that technically it is 2 Gibibytes file size that trigger the issue, but 2 GB is the safer threshold + if stat(self._problem_files[0]).st_size > 2E9: # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). # Use convertMPS, first create file for upload. mpsWithIdFileName = TempfileManager.create_tempfile( @@ -731,62 +711,64 @@ def _apply_solver(self): r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") ) - if r.ProblemSummary["cValue1"][1] == "Maximization": - results.problem.sense = ProblemSense.maximize - else: - results.problem.sense = ProblemSense.minimize - - # Prepare the solution information - if ( - results.solver.termination_condition - == TerminationCondition.optimal - ): - sol = results.solution.add() - - # Store status in solution - sol.status = SolutionStatus.feasible - sol.termination_condition = TerminationCondition.optimal - - # Store objective value in solution - sol.objective["__default_objective__"] = { - "Value": r["objective"] - } - - if action == "solveMilp": - primal_out = s.CASTable(name=primalout_table_name) - # Use pandas functions for efficiency - primal_out = primal_out[["_VAR_", "_VALUE_"]] - sol.variable = {} - for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = {"Value": row[1]} + if results.solver.status != SolverStatus.error: + if r.ProblemSummary["cValue1"][1] == "Maximization": + results.problem.sense = ProblemSense.maximize else: - # Convert primal out data set to variable dictionary - # Use panda functions for efficiency - primal_out = s.CASTable(name=primalout_table_name) - primal_out = primal_out[ - ["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"] - ] - sol.variable = {} - for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = { - "Value": row[1], - "Status": row[2], - "rc": row[3], - } - - # Convert dual out data set to constraint dictionary - # Use pandas functions for efficiency - dual_out = s.CASTable(name=dualout_table_name) - dual_out = dual_out[ - ["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"] - ] - sol.constraint = {} - for row in dual_out.itertuples(index=False): - sol.constraint[row[0]] = { - "dual": row[1], - "Status": row[2], - "slack": row[3], - } + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if results.solver.hasSolution: + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[r.get("solutionStatus", "ERROR")] + + # Store objective value in solution + sol.objective["__default_objective__"] = { + "Value": r["objective"] + } + + if action == "solveMilp": + primal_out = s.CASTable(name=primalout_table_name) + # Use pandas functions for efficiency + primal_out = primal_out[["_VAR_", "_VALUE_"]] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {"Value": row[1]} + else: + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = s.CASTable(name=primalout_table_name) + primal_out = primal_out[ + ["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"] + ] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = { + "Value": row[1], + "Status": row[2], + "rc": row[3], + } + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = s.CASTable(name=dualout_table_name) + dual_out = dual_out[ + ["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"] + ] + sol.constraint = {} + for row in dual_out.itertuples(index=False): + sol.constraint[row[0]] = { + "dual": row[1], + "Status": row[2], + "slack": row[3], + } + else: + raise ValueError( + "The SAS solver returned an error status." + ) else: results = self.results = SolverResults() results.solver.name = "SAS" diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 1a6bbd80f1d..7b0e2cccd9a 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -30,11 +30,11 @@ class SASTestAbc: solver_io = "_sas94" - base_options = {} + session_options = {} @classmethod def setUpClass(cls): - cls.opt_sas = SolverFactory("sas", solver_io=cls.solver_io) + cls.opt_sas = SolverFactory("sas", solver_io=cls.solver_io, **cls.session_options) @classmethod def tearDownClass(cls): @@ -73,11 +73,6 @@ def run_solver(self, **kwargs): opt_sas = self.opt_sas instance = self.instance - # Add base options for connection data etc. - options = kwargs.get("options", {}) - if self.base_options: - kwargs["options"] = {**options, **self.base_options} - # Call the solver self.results = opt_sas.solve(instance, **kwargs) @@ -285,7 +280,7 @@ def test_solver_with_milp(self): class SASTestLP94(SASTestLP, unittest.TestCase): @mock.patch( "pyomo.solvers.plugins.solvers.SAS.SAS94.sas_version", - return_value="2sd45s39M4234232", + return_value="9.sd45s39M4234232", ) def test_solver_versionM4(self, sas): with self.assertRaises(NotImplementedError): @@ -293,7 +288,7 @@ def test_solver_versionM4(self, sas): @mock.patch( "pyomo.solvers.plugins.solvers.SAS.SAS94.sas_version", - return_value="234897293M5324u98", + return_value="9.34897293M5324u98", ) def test_solver_versionM5(self, sas): self.run_solver() @@ -317,7 +312,7 @@ def test_solver_error(self, submit_mock, symget_mock): @unittest.skipIf(not sas_available, "The SAS solver is not available") class SASTestLPCAS(SASTestLP, unittest.TestCase): solver_io = "_sascas" - base_options = CAS_OPTIONS + session_options = CAS_OPTIONS @mock.patch("pyomo.solvers.plugins.solvers.SAS.stat") def test_solver_large_file(self, os_stat): @@ -522,7 +517,7 @@ class SASTestMILP94(SASTestMILP, unittest.TestCase): @unittest.skipIf(not sas_available, "The SAS solver is not available") class SASTestMILPCAS(SASTestMILP, unittest.TestCase): solver_io = "_sascas" - base_options = CAS_OPTIONS + session_options = CAS_OPTIONS if __name__ == "__main__": From e17a2e5b77d33c6d63e2836667de97dd854d3c41 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Tue, 12 Mar 2024 07:26:58 -0400 Subject: [PATCH 010/150] Black formatting --- pyomo/solvers/plugins/solvers/SAS.py | 32 ++++++++++++++------------ pyomo/solvers/tests/checks/test_SAS.py | 4 +++- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 87ee31a08af..bd06f6a1ef7 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -204,9 +204,9 @@ def _create_results_from_status(self, status, solution_status): results.solver.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ solution_status ] - results.solver.message = ( - results.solver.termination_message - ) = SOLSTATUS_TO_MESSAGE[solution_status] + results.solver.message = results.solver.termination_message = ( + SOLSTATUS_TO_MESSAGE[solution_status] + ) results.solver.status = TerminationCondition.to_solver_status( results.solver.termination_condition ) @@ -215,14 +215,14 @@ def _create_results_from_status(self, status, solution_status): elif results.solver.status == SolverStatus.aborted: results.solver.termination_condition = TerminationCondition.userInterrupt if solution_status != "ERROR": - results.solver.message = ( - results.solver.termination_message - ) = SOLSTATUS_TO_MESSAGE[solution_status] + results.solver.message = results.solver.termination_message = ( + SOLSTATUS_TO_MESSAGE[solution_status] + ) else: results.solver.termination_condition = TerminationCondition.error - results.solver.message = ( - results.solver.termination_message - ) = SOLSTATUS_TO_MESSAGE["FAILED"] + results.solver.message = results.solver.termination_message = ( + SOLSTATUS_TO_MESSAGE["FAILED"] + ) return results @abstractmethod @@ -502,7 +502,9 @@ def _apply_solver(self): # Store status in solution sol.status = SolutionStatus.feasible - sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[self._macro.get("SOLUTION_STATUS", "ERROR")] + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + self._macro.get("SOLUTION_STATUS", "ERROR") + ] # Store objective value in solution sol.objective["__default_objective__"] = {"Value": self._macro["OBJECTIVE"]} @@ -632,7 +634,7 @@ def _apply_solver(self): # Upload mps file to CAS, if the file is larger than 2 GB, we need to use convertMps instead of loadMps # Note that technically it is 2 Gibibytes file size that trigger the issue, but 2 GB is the safer threshold - if stat(self._problem_files[0]).st_size > 2E9: + if stat(self._problem_files[0]).st_size > 2e9: # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). # Use convertMPS, first create file for upload. mpsWithIdFileName = TempfileManager.create_tempfile( @@ -723,7 +725,9 @@ def _apply_solver(self): # Store status in solution sol.status = SolutionStatus.feasible - sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[r.get("solutionStatus", "ERROR")] + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + r.get("solutionStatus", "ERROR") + ] # Store objective value in solution sol.objective["__default_objective__"] = { @@ -766,9 +770,7 @@ def _apply_solver(self): "slack": row[3], } else: - raise ValueError( - "The SAS solver returned an error status." - ) + raise ValueError("The SAS solver returned an error status.") else: results = self.results = SolverResults() results.solver.name = "SAS" diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 7b0e2cccd9a..922209ef88b 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -34,7 +34,9 @@ class SASTestAbc: @classmethod def setUpClass(cls): - cls.opt_sas = SolverFactory("sas", solver_io=cls.solver_io, **cls.session_options) + cls.opt_sas = SolverFactory( + "sas", solver_io=cls.solver_io, **cls.session_options + ) @classmethod def tearDownClass(cls): From 07df0c69a9d62e0174d94077f3fff323bea53c87 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Tue, 26 Mar 2024 09:17:14 -0400 Subject: [PATCH 011/150] Use TeeStream, simplify _apply_solver and other small fixes --- pyomo/solvers/plugins/solvers/SAS.py | 392 +++++++++++++------------ pyomo/solvers/tests/checks/test_SAS.py | 11 + 2 files changed, 208 insertions(+), 195 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index bd06f6a1ef7..bccb7d34077 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -1,13 +1,20 @@ -__all__ = ["SAS"] +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ import logging import sys from os import stat import uuid - -from io import StringIO from abc import ABC, abstractmethod -from contextlib import redirect_stdout +from io import StringIO from pyomo.opt.base import ProblemFormat, ResultsFormat, OptSolver from pyomo.opt.base.solvers import SolverFactory @@ -23,6 +30,8 @@ from pyomo.core.base import Var from pyomo.core.base.block import _BlockData from pyomo.core.kernel.block import IBlock +from pyomo.common.log import LogStream +from pyomo.common.tee import capture_output, TeeStream logger = logging.getLogger("pyomo.solvers") @@ -252,6 +261,7 @@ class SAS94(SASAbc): """ Solver interface for SAS 9.4 using saspy. See the saspy documentation about how to create a connection. + The swat connection options can be specified on the SolverFactory call. """ def __init__(self, **kwds): @@ -541,37 +551,12 @@ def _apply_solver(self): return Bunch(rc=self._rc, log=self._log) -class SASLogWriter: - """Helper class to take the log from stdout and put it also in a StringIO.""" - - def __init__(self, tee): - """Set up the two outputs.""" - self.tee = tee - self._log = StringIO() - self.stdout = sys.stdout - - def write(self, message): - """If the tee options is specified, write to both outputs.""" - if self.tee: - self.stdout.write(message) - self._log.write(message) - - def flush(self): - """Nothing to do, just here for compatibility reasons.""" - # Do nothing since we flush right away - pass - - def log(self): - """ "Get the log as a string.""" - return self._log.getvalue() - - @SolverFactory.register("_sascas", doc="SAS Viya CAS Server interface") class SASCAS(SASAbc): """ Solver interface connection to a SAS Viya CAS server using swat. See the documentation for the swat package about how to create a connection. - The swat connection options can be passed as options to the solve function. + The swat connection options can be specified on the SolverFactory call. """ def __init__(self, **kwds): @@ -600,6 +585,106 @@ def __del__(self): if self._sas_session: self._sas_session.close() + def _uploadMpsFile(self, s, unique): + # Declare a unique table name for the mps table + mpsdata_table_name = "mps" + unique + + # Upload mps file to CAS, if the file is larger than 2 GB, we need to use convertMps instead of loadMps + # Note that technically it is 2 Gibibytes file size that trigger the issue, but 2 GB is the safer threshold + if stat(self._problem_files[0]).st_size > 2e9: + # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). + # Use convertMPS, first create file for upload. + mpsWithIdFileName = TempfileManager.create_tempfile(".mps.csv", text=True) + with open(mpsWithIdFileName, "w") as mpsWithId: + mpsWithId.write("_ID_\tText\n") + with open(self._problem_files[0], "r") as f: + id = 0 + for line in f: + id += 1 + mpsWithId.write(str(id) + "\t" + line.rstrip() + "\n") + + # Upload .mps.csv file + mpscsv_table_name = "csv" + unique + s.upload_file( + mpsWithIdFileName, + casout={"name": mpscsv_table_name, "replace": True}, + importoptions={"filetype": "CSV", "delimiter": "\t"}, + ) + + # Convert .mps.csv file to .mps + s.optimization.convertMps( + data=mpscsv_table_name, + casOut={"name": mpsdata_table_name, "replace": True}, + format="FREE", + ) + + # Delete the table we don't need anymore + if mpscsv_table_name: + s.dropTable(name=mpscsv_table_name, quiet=True) + else: + # For small files (less than 2 GB), use loadMps + with open(self._problem_files[0], "r") as mps_file: + s.optimization.loadMps( + mpsFileString=mps_file.read(), + casout={"name": mpsdata_table_name, "replace": True}, + format="FREE", + ) + return mpsdata_table_name + + def _uploadPrimalin(self, s, unique): + # Upload warmstart file to CAS with a unique name + primalin_table_name = "pin" + unique + s.upload_file( + self._warm_start_file_name, + casout={"name": primalin_table_name, "replace": True}, + importoptions={"filetype": "CSV"}, + ) + self.options["primalin"] = primalin_table_name + return primalin_table_name + + def _retrieveSolution( + self, s, r, results, action, primalout_table_name, dualout_table_name + ): + # Create solution + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + r.get("solutionStatus", "ERROR") + ] + + # Store objective value in solution + sol.objective["__default_objective__"] = {"Value": r["objective"]} + + if action == "solveMilp": + primal_out = s.CASTable(name=primalout_table_name) + # Use pandas functions for efficiency + primal_out = primal_out[["_VAR_", "_VALUE_"]] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {"Value": row[1]} + else: + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = s.CASTable(name=primalout_table_name) + primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {"Value": row[1], "Status": row[2], "rc": row[3]} + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = s.CASTable(name=dualout_table_name) + dual_out = dual_out[["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"]] + sol.constraint = {} + for row in dual_out.itertuples(index=False): + sol.constraint[row[0]] = { + "dual": row[1], + "Status": row[2], + "slack": row[3], + } + def _apply_solver(self): """ "Prepare the options and run the solver. Then store the data to be returned.""" logger.debug("Running SAS Viya") @@ -620,175 +705,92 @@ def _apply_solver(self): # Get a unique identifier, always use the same with different prefixes unique = uuid.uuid4().hex[:16] + # Creat the output stream, we want to print to a log string as well as to the console + self._log = StringIO() + ostreams = [LogStream(level=logging.INFO, logger=logger)] + ostreams.append(self._log) + if self._tee: + ostreams.append(sys.stdout) + # Connect to CAS server - with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: - s = self._sas_session - if s == None: - s = self._sas_session = self._sas.CAS(**self._session_options) - try: - # Load the optimization action set - s.loadactionset("optimization") - - # Declare a unique table name for the mps table - mpsdata_table_name = "mps" + unique - - # Upload mps file to CAS, if the file is larger than 2 GB, we need to use convertMps instead of loadMps - # Note that technically it is 2 Gibibytes file size that trigger the issue, but 2 GB is the safer threshold - if stat(self._problem_files[0]).st_size > 2e9: - # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). - # Use convertMPS, first create file for upload. - mpsWithIdFileName = TempfileManager.create_tempfile( - ".mps.csv", text=True - ) - with open(mpsWithIdFileName, "w") as mpsWithId: - mpsWithId.write("_ID_\tText\n") - with open(self._problem_files[0], "r") as f: - id = 0 - for line in f: - id += 1 - mpsWithId.write(str(id) + "\t" + line.rstrip() + "\n") - - # Upload .mps.csv file - mpscsv_table_name = "csv" + unique - s.upload_file( - mpsWithIdFileName, - casout={"name": mpscsv_table_name, "replace": True}, - importoptions={"filetype": "CSV", "delimiter": "\t"}, - ) - - # Convert .mps.csv file to .mps - s.optimization.convertMps( - data=mpscsv_table_name, - casOut={"name": mpsdata_table_name, "replace": True}, - format="FREE", - ) - - # Delete the table we don't need anymore - if mpscsv_table_name: - s.dropTable(name=mpscsv_table_name, quiet=True) - else: - # For small files (less than 2 GB), use loadMps - with open(self._problem_files[0], "r") as mps_file: - s.optimization.loadMps( - mpsFileString=mps_file.read(), - casout={"name": mpsdata_table_name, "replace": True}, - format="FREE", + with TeeStream(*ostreams) as t: + with capture_output(output=t.STDOUT, capture_fd=False): + s = self._sas_session + if s == None: + s = self._sas_session = self._sas.CAS(**self._session_options) + try: + # Load the optimization action set + s.loadactionset("optimization") + + mpsdata_table_name = self._uploadMpsFile(s, unique) + + primalin_table_name = None + if self.warmstart_flag: + primalin_table_name = self._uploadPrimalin(s, unique) + + # Define output table names + primalout_table_name = "pout" + unique + dualout_table_name = None + + # Solve the problem in CAS + if action == "solveMilp": + r = s.optimization.solveMilp( + data={"name": mpsdata_table_name}, + primalOut={"name": primalout_table_name, "replace": True}, + **self.options + ) + else: + dualout_table_name = "dout" + unique + r = s.optimization.solveLp( + data={"name": mpsdata_table_name}, + primalOut={"name": primalout_table_name, "replace": True}, + dualOut={"name": dualout_table_name, "replace": True}, + **self.options ) - primalin_table_name = None - if self.warmstart_flag: - primalin_table_name = "pin" + unique - # Upload warmstart file to CAS - s.upload_file( - self._warm_start_file_name, - casout={"name": primalin_table_name, "replace": True}, - importoptions={"filetype": "CSV"}, - ) - self.options["primalin"] = primalin_table_name - - # Define output table names - primalout_table_name = "pout" + unique - dualout_table_name = None - - # Solve the problem in CAS - if action == "solveMilp": - r = s.optimization.solveMilp( - data={"name": mpsdata_table_name}, - primalOut={"name": primalout_table_name, "replace": True}, - **self.options - ) - else: - dualout_table_name = "dout" + unique - r = s.optimization.solveLp( - data={"name": mpsdata_table_name}, - primalOut={"name": primalout_table_name, "replace": True}, - dualOut={"name": dualout_table_name, "replace": True}, - **self.options - ) - - # Prepare the solver results - if r: - # Get back the primal and dual solution data sets - results = self.results = self._create_results_from_status( - r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") - ) - - if results.solver.status != SolverStatus.error: - if r.ProblemSummary["cValue1"][1] == "Maximization": - results.problem.sense = ProblemSense.maximize - else: - results.problem.sense = ProblemSense.minimize - - # Prepare the solution information - if results.solver.hasSolution: - sol = results.solution.add() - - # Store status in solution - sol.status = SolutionStatus.feasible - sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ - r.get("solutionStatus", "ERROR") - ] - - # Store objective value in solution - sol.objective["__default_objective__"] = { - "Value": r["objective"] - } - - if action == "solveMilp": - primal_out = s.CASTable(name=primalout_table_name) - # Use pandas functions for efficiency - primal_out = primal_out[["_VAR_", "_VALUE_"]] - sol.variable = {} - for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = {"Value": row[1]} + # Prepare the solver results + if r: + # Get back the primal and dual solution data sets + results = self.results = self._create_results_from_status( + r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") + ) + + if results.solver.status != SolverStatus.error: + if r.ProblemSummary["cValue1"][1] == "Maximization": + results.problem.sense = ProblemSense.maximize else: - # Convert primal out data set to variable dictionary - # Use panda functions for efficiency - primal_out = s.CASTable(name=primalout_table_name) - primal_out = primal_out[ - ["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"] - ] - sol.variable = {} - for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = { - "Value": row[1], - "Status": row[2], - "rc": row[3], - } - - # Convert dual out data set to constraint dictionary - # Use pandas functions for efficiency - dual_out = s.CASTable(name=dualout_table_name) - dual_out = dual_out[ - ["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"] - ] - sol.constraint = {} - for row in dual_out.itertuples(index=False): - sol.constraint[row[0]] = { - "dual": row[1], - "Status": row[2], - "slack": row[3], - } + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if results.solver.hasSolution: + self._retrieveSolution( + s, + r, + results, + action, + primalout_table_name, + dualout_table_name, + ) + else: + raise ValueError("The SAS solver returned an error status.") else: - raise ValueError("The SAS solver returned an error status.") - else: - results = self.results = SolverResults() - results.solver.name = "SAS" - results.solver.status = SolverStatus.error - raise ValueError( - "An option passed to the SAS solver caused a syntax error." - ) - - finally: - if mpsdata_table_name: - s.dropTable(name=mpsdata_table_name, quiet=True) - if primalin_table_name: - s.dropTable(name=primalin_table_name, quiet=True) - if primalout_table_name: - s.dropTable(name=primalout_table_name, quiet=True) - if dualout_table_name: - s.dropTable(name=dualout_table_name, quiet=True) - - self._log = self._log_writer.log() + results = self.results = SolverResults() + results.solver.name = "SAS" + results.solver.status = SolverStatus.error + raise ValueError( + "An option passed to the SAS solver caused a syntax error." + ) + + finally: + if mpsdata_table_name: + s.dropTable(name=mpsdata_table_name, quiet=True) + if primalin_table_name: + s.dropTable(name=primalin_table_name, quiet=True) + if primalout_table_name: + s.dropTable(name=primalout_table_name, quiet=True) + if dualout_table_name: + s.dropTable(name=dualout_table_name, quiet=True) + + self._log = self._log.getvalue() self._rc = 0 return Bunch(rc=self._rc, log=self._log) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 922209ef88b..75534e0e001 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + import os import pyomo.common.unittest as unittest from unittest import mock From f3b9b12f5fe4048472aaa7b2314f3448ea83b73f Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Apr 2024 16:48:23 -0600 Subject: [PATCH 012/150] Adding initial implementation of linear walker that only walks with respect to specified variables, treating others as data --- pyomo/repn/linear_wrt.py | 77 +++++++++++++++++++++++++++++ pyomo/repn/tests/test_linear_wrt.py | 40 +++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 pyomo/repn/linear_wrt.py create mode 100644 pyomo/repn/tests/test_linear_wrt.py diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py new file mode 100644 index 00000000000..0d86528056c --- /dev/null +++ b/pyomo/repn/linear_wrt.py @@ -0,0 +1,77 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.common.collections import ComponentSet +from pyomo.core import Var +from pyomo.core.expr.logical_expr import _flattened +from pyomo.core.expr.numeric_expr import ( + LinearExpression, + MonomialTermExpression, + SumExpression, +) +from pyomo.repn.linear import LinearBeforeChildDispatcher, LinearRepnVisitor +from pyomo.repn.util import ExprType + + +class MultiLevelLinearBeforeChildDispatcher(LinearBeforeChildDispatcher): + def __init__(self): + super().__init__() + self[Var] = self._before_var + self[MonomialTermExpression] = self._before_monomial + self[LinearExpression] = self._before_linear + self[SumExpression] = self._before_general_expression + + @staticmethod + def _before_linear(visitor, child): + return True, None + + @staticmethod + def _before_monomial(visitor, child): + return True, None + + @staticmethod + def _before_general_expression(visitor, child): + return True, None + + @staticmethod + def _before_var(visitor, child): + if child in visitor.wrt: + # This is a normal situation + print("NORMAL: %s" % child) + _id = id(child) + if _id not in visitor.var_map: + if child.fixed: + return False, ( + ExprType.CONSTANT, + visitor.check_constant(child.value, child), + ) + MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) + ans = visitor.Result() + ans.linear[_id] = 1 + return False, (ExprType.LINEAR, ans) + else: + print("DATA: %s" % child) + # We aren't treating this Var as a Var for the purposes of this walker + return False, (ExprType.CONSTANT, child) + + +_before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() + + +class MultilevelLinearRepnVisitor(LinearRepnVisitor): + def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): + super().__init__(subexpression_cache, var_map, var_order, sorter) + self.wrt = ComponentSet(_flattened(wrt)) + + def beforeChild(self, node, child, child_idx): + print("before child %s" % child) + print(child.__class__) + return _before_child_dispatcher[child.__class__](self, child) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py new file mode 100644 index 00000000000..29c8f69ad03 --- /dev/null +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -0,0 +1,40 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import pyomo.common.unittest as unittest +from pyomo.environ import Binary, ConcreteModel, Var +from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor +from pyomo.repn.tests.test_linear import VisitorConfig + + +class TestMultilevelLinearRepnVisitor(unittest.TestCase): + def make_model(self): + m = ConcreteModel() + m.x = Var(bounds=(0, 45)) + m.y = Var(domain=Binary) + + return m + + def test_walk_sum(self): + m = self.make_model() + e = m.x + m.y + cfg = VisitorConfig() + print("constructing") + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 1) + self.assertIs(repn.constant, m.y) + self.assertEqual(repn.multiplier, 1) From 7614ff0367558558d27bb48d9ea1393278cd9291 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 9 Apr 2024 14:44:15 -0600 Subject: [PATCH 013/150] Overriding finalizeResult to account for the fact that mult might not be constant --- pyomo/repn/linear.py | 31 ++++++++++++++++------------- pyomo/repn/linear_wrt.py | 14 +++++++++++++ pyomo/repn/tests/test_linear_wrt.py | 16 ++++++++++++++- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index ba08c7ef245..913b36f6f16 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -803,6 +803,22 @@ def exitNode(self, node, data): self, node, *data ) + def _factor_multiplier_into_linear_terms(self, ans, mult): + linear = ans.linear + zeros = [] + for vid, coef in linear.items(): + if coef: + linear[vid] = coef * mult + else: + zeros.append(vid) + for vid in zeros: + del linear[vid] + if ans.nonlinear is not None: + ans.nonlinear *= mult + if ans.constant: + ans.constant *= mult + ans.multiplier = 1 + def finalizeResult(self, result): ans = result[1] if ans.__class__ is self.Result: @@ -831,20 +847,7 @@ def finalizeResult(self, result): else: # mult not in {0, 1}: factor it into the constant, # linear coefficients, and nonlinear term - linear = ans.linear - zeros = [] - for vid, coef in linear.items(): - if coef: - linear[vid] = coef * mult - else: - zeros.append(vid) - for vid in zeros: - del linear[vid] - if ans.nonlinear is not None: - ans.nonlinear *= mult - if ans.constant: - ans.constant *= mult - ans.multiplier = 1 + self._factor_mult_into_linear_terms(ans, mult) return ans ans = self.Result() assert result[0] is _CONSTANT diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index 0d86528056c..46451d3d64c 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -10,6 +10,7 @@ # ___________________________________________________________________________ from pyomo.common.collections import ComponentSet +from pyomo.common.numeric_types import native_numeric_types from pyomo.core import Var from pyomo.core.expr.logical_expr import _flattened from pyomo.core.expr.numeric_expr import ( @@ -66,6 +67,7 @@ def _before_var(visitor, child): _before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() +# LinearSubsystemRepnVisitor class MultilevelLinearRepnVisitor(LinearRepnVisitor): def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): super().__init__(subexpression_cache, var_map, var_order, sorter) @@ -75,3 +77,15 @@ def beforeChild(self, node, child, child_idx): print("before child %s" % child) print(child.__class__) return _before_child_dispatcher[child.__class__](self, child) + + def finalizeResult(self, result): + ans = result[1] + if ans.__class__ is self.Result: + mult = ans.multiplier + if not mult.__class__ in native_numeric_types: + # mult is an expression--we should push it back into the other terms + self._factor_multiplier_into_linear_terms(ans, mult) + return ans + + # In all other cases, the base class implementation is correct + return super().finalizeResult(result) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 29c8f69ad03..384412b25ce 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -27,7 +27,6 @@ def test_walk_sum(self): m = self.make_model() e = m.x + m.y cfg = VisitorConfig() - print("constructing") visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) repn = visitor.walk_expression(e) @@ -38,3 +37,18 @@ def test_walk_sum(self): self.assertEqual(repn.linear[id(m.x)], 1) self.assertIs(repn.constant, m.y) self.assertEqual(repn.multiplier, 1) + + def test_bilinear_term(self): + m = self.make_model() + e = m.x * m.y + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertIs(repn.linear[id(m.x)], m.y) + self.assertEqual(repn.constant, 0) + self.assertEqual(repn.multiplier, 1) From 0451815e320b77689bb56ab263b4c1c4134174e5 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 14:30:14 -0600 Subject: [PATCH 014/150] Rewriting append and to_expression in LinearRepn to not assume that constant and multiplier are numeric types --- pyomo/repn/linear.py | 61 ++++++++++++------- pyomo/repn/linear_wrt.py | 4 +- pyomo/repn/tests/test_linear.py | 2 +- pyomo/repn/tests/test_linear_wrt.py | 91 ++++++++++++++++++++++++++++- 4 files changed, 131 insertions(+), 27 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 913b36f6f16..fd5eae13fc2 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -128,9 +128,10 @@ def to_expression(self, visitor): ans += e elif e.nargs() == 1: ans += e.arg(0) - if self.constant: + if self.constant.__class__ not in native_numeric_types or self.constant: ans += self.constant - if self.multiplier != 1: + if (self.multiplier.__class__ not in native_numeric_types or + self.multiplier != 1): ans *= self.multiplier return ans @@ -147,33 +148,49 @@ def append(self, other): callback). """ - # Note that self.multiplier will always be 1 (we only call append() - # within a sum, so there is no opportunity for self.multiplier to - # change). Omitting the assertion for efficiency. - # assert self.multiplier == 1 _type, other = other if _type is _CONSTANT: self.constant += other return mult = other.multiplier - if not mult: - # 0 * other, so there is nothing to add/change about - # self. We can just exit now. - return - if other.constant: - self.constant += mult * other.constant - if other.linear: - _merge_dict(self.linear, mult, other.linear) - if other.nonlinear is not None: - if mult != 1: + try: + _mult = bool(mult) + if not _mult: + return + if mult == 1: + _mult = False + except: + _mult = True + + const = other.constant + try: + _const = bool(const) + except: + _const = True + + if _mult: + if _const: + self.constant += mult * const + if other.linear: + _merge_dict(self.linear, mult, other.linear) + if other.nonlinear is not None: nl = mult * other.nonlinear - else: + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl + else: + if _const: + self.constant += const + if other.linear: + _merge_dict(self.linear, 1, other.linear) + if other.nonlinear is not None: nl = other.nonlinear - if self.nonlinear is None: - self.nonlinear = nl - else: - self.nonlinear += nl + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl def to_expression(visitor, arg): @@ -847,7 +864,7 @@ def finalizeResult(self, result): else: # mult not in {0, 1}: factor it into the constant, # linear coefficients, and nonlinear term - self._factor_mult_into_linear_terms(ans, mult) + self._factor_multiplier_into_linear_terms(ans, mult) return ans ans = self.Result() assert result[0] is _CONSTANT diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index 46451d3d64c..b58ff32ba71 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -74,15 +74,13 @@ def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): self.wrt = ComponentSet(_flattened(wrt)) def beforeChild(self, node, child, child_idx): - print("before child %s" % child) - print(child.__class__) return _before_child_dispatcher[child.__class__](self, child) def finalizeResult(self, result): ans = result[1] if ans.__class__ is self.Result: mult = ans.multiplier - if not mult.__class__ in native_numeric_types: + if mult.__class__ not in native_numeric_types: # mult is an expression--we should push it back into the other terms self._factor_multiplier_into_linear_terms(ans, mult) return ans diff --git a/pyomo/repn/tests/test_linear.py b/pyomo/repn/tests/test_linear.py index 0fd428fd8ee..badb7f407f5 100644 --- a/pyomo/repn/tests/test_linear.py +++ b/pyomo/repn/tests/test_linear.py @@ -1643,7 +1643,7 @@ def test_zero_elimination(self): self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 0) self.assertEqual(repn.linear, {}) - self.assertEqual(repn.nonlinear, None) + self.assertIsNone(repn.nonlinear) m.p = Param(mutable=True, within=Any, initialize=None) e = m.p * m.x[0] + m.p * m.x[1] * m.x[2] + m.p * log(m.x[3]) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 384412b25ce..6bfbd51c01e 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -10,7 +10,8 @@ # ___________________________________________________________________________ import pyomo.common.unittest as unittest -from pyomo.environ import Binary, ConcreteModel, Var +from pyomo.core.expr.compare import assertExpressionsEqual +from pyomo.environ import Binary, ConcreteModel, Var, log from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor from pyomo.repn.tests.test_linear import VisitorConfig @@ -52,3 +53,91 @@ def test_bilinear_term(self): self.assertIs(repn.linear[id(m.x)], m.y) self.assertEqual(repn.constant, 0) self.assertEqual(repn.multiplier, 1) + + def test_distributed_bilinear_term(self): + m = self.make_model() + e = m.y * (m.x + 7) + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertIs(repn.linear[id(m.x)], m.y) + assertExpressionsEqual( + self, + repn.constant, + m.y * 7 + ) + self.assertEqual(repn.multiplier, 1) + + def test_monomial(self): + m = self.make_model() + e = 45 * m.y + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.y]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 45) + self.assertEqual(repn.constant, 0) + self.assertEqual(repn.multiplier, 1) + + def test_constant(self): + m = self.make_model() + e = 45 * m.y + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 0) + assertExpressionsEqual( + self, + repn.constant, + 45 * m.y + ) + self.assertEqual(repn.multiplier, 1) + + def test_fixed_var(self): + m = self.make_model() + m.x.fix(42) + e = (m.y ** 2) * (m.x + m.x ** 2) + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 0) + assertExpressionsEqual( + self, + repn.constant, + (m.y ** 2) * 1806 + ) + self.assertEqual(repn.multiplier, 1) + + def test_nonlinear(self): + m = self.make_model() + e = (m.y * log(m.x)) * (m.y + 2) / m.x + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + print(repn.nonlinear) + assertExpressionsEqual( + self, + repn.nonlinear, + log(m.x) * (m.y *(m.y + 2))/m.x + ) From a1636f085c0dec9d8c7d320c1dfe021e25bde9d0 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 14:31:43 -0600 Subject: [PATCH 015/150] black --- pyomo/repn/linear.py | 6 ++++-- pyomo/repn/tests/test_linear_wrt.py | 30 +++++++---------------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index fd5eae13fc2..c3b84940a71 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -130,8 +130,10 @@ def to_expression(self, visitor): ans += e.arg(0) if self.constant.__class__ not in native_numeric_types or self.constant: ans += self.constant - if (self.multiplier.__class__ not in native_numeric_types or - self.multiplier != 1): + if ( + self.multiplier.__class__ not in native_numeric_types + or self.multiplier != 1 + ): ans *= self.multiplier return ans diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 6bfbd51c01e..fe159874186 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -66,11 +66,7 @@ def test_distributed_bilinear_term(self): self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertIs(repn.linear[id(m.x)], m.y) - assertExpressionsEqual( - self, - repn.constant, - m.y * 7 - ) + assertExpressionsEqual(self, repn.constant, m.y * 7) self.assertEqual(repn.multiplier, 1) def test_monomial(self): @@ -95,20 +91,16 @@ def test_constant(self): visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) repn = visitor.walk_expression(e) - + self.assertIsNone(repn.nonlinear) self.assertEqual(len(repn.linear), 0) - assertExpressionsEqual( - self, - repn.constant, - 45 * m.y - ) + assertExpressionsEqual(self, repn.constant, 45 * m.y) self.assertEqual(repn.multiplier, 1) def test_fixed_var(self): m = self.make_model() m.x.fix(42) - e = (m.y ** 2) * (m.x + m.x ** 2) + e = (m.y**2) * (m.x + m.x**2) cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) @@ -117,17 +109,13 @@ def test_fixed_var(self): self.assertIsNone(repn.nonlinear) self.assertEqual(len(repn.linear), 0) - assertExpressionsEqual( - self, - repn.constant, - (m.y ** 2) * 1806 - ) + assertExpressionsEqual(self, repn.constant, (m.y**2) * 1806) self.assertEqual(repn.multiplier, 1) def test_nonlinear(self): m = self.make_model() e = (m.y * log(m.x)) * (m.y + 2) / m.x - + cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) @@ -136,8 +124,4 @@ def test_nonlinear(self): self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) print(repn.nonlinear) - assertExpressionsEqual( - self, - repn.nonlinear, - log(m.x) * (m.y *(m.y + 2))/m.x - ) + assertExpressionsEqual(self, repn.nonlinear, log(m.x) * (m.y * (m.y + 2)) / m.x) From 549d8b00d25b5698f57100331a2fa7ab245a6741 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 14:58:03 -0600 Subject: [PATCH 016/150] fixing another to_expression spot where we assume the coefficient is numeric --- pyomo/repn/linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index c3b84940a71..2a578827afd 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -122,7 +122,7 @@ def to_expression(self, visitor): var_map = visitor.var_map with mutable_expression() as e: for vid, coef in self.linear.items(): - if coef: + if coef.__class__ not in native_numeric_types or coef: e += coef * var_map[vid] if e.nargs() > 1: ans += e From 02be1cc6636a08276787a4707f211775783bd100 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 15:24:40 -0600 Subject: [PATCH 017/150] Generalizing merge_dict for non-constant multipliers --- pyomo/repn/linear.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 2a578827afd..78534789fbe 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -62,7 +62,12 @@ def _merge_dict(dest_dict, mult, src_dict): - if mult == 1: + try: + _mult = mult != 1 + except: + _mult = True + + if not _mult: for vid, coef in src_dict.items(): if vid in dest_dict: dest_dict[vid] += coef From a91eb4d5a3dee95c11148f4d32a03c9fcc1e2888 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 15:25:06 -0600 Subject: [PATCH 018/150] Completely overriding finalizeResult because of non-constant coefficients --- pyomo/repn/linear_wrt.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index b58ff32ba71..42543340f48 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -84,6 +84,33 @@ def finalizeResult(self, result): # mult is an expression--we should push it back into the other terms self._factor_multiplier_into_linear_terms(ans, mult) return ans + if mult == 1: + for vid, coef in ans.linear.items(): + if coef.__class__ in native_numeric_types and not coef: + del ans.linear[vid] + elif not mult: + # the mulltiplier has cleared out the entire expression. + # Warn if this is suppressing a NaN (unusual, and + # non-standard, but we will wait to remove this behavior + # for the time being) + if ans.constant != ans.constant or any( + c != c for c in ans.linear.values() + ): + deprecation_warning( + f"Encountered {str(mult)}*nan in expression tree. " + "Mapping the NaN result to 0 for compatibility " + "with the lp_v1 writer. In the future, this NaN " + "will be preserved/emitted to comply with IEEE-754.", + version='6.6.0', + ) + return self.Result() + else: + # mult not in {0, 1}: factor it into the constant, + # linear coefficients, and nonlinear term + self._factor_multiplier_into_linear_terms(ans, mult) + return ans - # In all other cases, the base class implementation is correct - return super().finalizeResult(result) + ans = self.Result() + assert result[0] is ExprType.CONSTANT + ans.constant = result[1] + return ans From c9aa8b8565521963e49ee304ab3fbc6637bc774a Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 15:25:28 -0600 Subject: [PATCH 019/150] Testing to_expression --- pyomo/repn/tests/test_linear_wrt.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index fe159874186..89bc9ee6abf 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -38,6 +38,27 @@ def test_walk_sum(self): self.assertEqual(repn.linear[id(m.x)], 1) self.assertIs(repn.constant, m.y) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.y) + + def test_walk_triple_sum(self): + m = self.make_model() + m.z = Var() + e = m.x + m.z*m.y + m.z + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.x), repn.linear) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 1) + self.assertIs(repn.linear[id(m.y)], m.z) + self.assertIs(repn.constant, m.z) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.z * m.y + m.z) def test_bilinear_term(self): m = self.make_model() @@ -53,6 +74,7 @@ def test_bilinear_term(self): self.assertIs(repn.linear[id(m.x)], m.y) self.assertEqual(repn.constant, 0) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x) def test_distributed_bilinear_term(self): m = self.make_model() @@ -68,6 +90,7 @@ def test_distributed_bilinear_term(self): self.assertIs(repn.linear[id(m.x)], m.y) assertExpressionsEqual(self, repn.constant, m.y * 7) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x + m.y * 7) def test_monomial(self): m = self.make_model() @@ -83,6 +106,7 @@ def test_monomial(self): self.assertEqual(repn.linear[id(m.y)], 45) self.assertEqual(repn.constant, 0) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), 45 * m.y) def test_constant(self): m = self.make_model() @@ -96,6 +120,7 @@ def test_constant(self): self.assertEqual(len(repn.linear), 0) assertExpressionsEqual(self, repn.constant, 45 * m.y) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), 45 * m.y) def test_fixed_var(self): m = self.make_model() @@ -111,6 +136,7 @@ def test_fixed_var(self): self.assertEqual(len(repn.linear), 0) assertExpressionsEqual(self, repn.constant, (m.y**2) * 1806) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), (m.y**2) * 1806) def test_nonlinear(self): m = self.make_model() @@ -123,5 +149,6 @@ def test_nonlinear(self): self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) - print(repn.nonlinear) assertExpressionsEqual(self, repn.nonlinear, log(m.x) * (m.y * (m.y + 2)) / m.x) + assertExpressionsEqual(self, repn.to_expression(visitor), + log(m.x) * (m.y * (m.y + 2)) / m.x) From 9fb61265d4a9975683c045dec3b5f5713d4f89b2 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 15:26:04 -0600 Subject: [PATCH 020/150] Black --- pyomo/repn/tests/test_linear_wrt.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 89bc9ee6abf..35e7160a351 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -43,7 +43,7 @@ def test_walk_sum(self): def test_walk_triple_sum(self): m = self.make_model() m.z = Var() - e = m.x + m.z*m.y + m.z + e = m.x + m.z * m.y + m.z cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) @@ -150,5 +150,6 @@ def test_nonlinear(self): self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.nonlinear, log(m.x) * (m.y * (m.y + 2)) / m.x) - assertExpressionsEqual(self, repn.to_expression(visitor), - log(m.x) * (m.y * (m.y + 2)) / m.x) + assertExpressionsEqual( + self, repn.to_expression(visitor), log(m.x) * (m.y * (m.y + 2)) / m.x + ) From 4ec32ffd5ecb6c683c072124f8a8aaa3d57f3911 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Fri, 19 Apr 2024 13:56:32 -0600 Subject: [PATCH 021/150] Fixing even more places where we assume constant and coefficients are not expressions --- pyomo/repn/linear.py | 6 +- pyomo/repn/linear_wrt.py | 11 +-- pyomo/repn/tests/test_linear_wrt.py | 111 +++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 9 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 78534789fbe..079fbb86489 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -831,15 +831,15 @@ def _factor_multiplier_into_linear_terms(self, ans, mult): linear = ans.linear zeros = [] for vid, coef in linear.items(): - if coef: - linear[vid] = coef * mult + if coef.__class__ not in native_numeric_types or coef: + linear[vid] = mult * coef else: zeros.append(vid) for vid in zeros: del linear[vid] if ans.nonlinear is not None: ans.nonlinear *= mult - if ans.constant: + if ans.constant.__class__ not in native_numeric_types or ans.constant: ans.constant *= mult ans.multiplier = 1 diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index 42543340f48..8fa1f9b4546 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -46,7 +46,6 @@ def _before_general_expression(visitor, child): def _before_var(visitor, child): if child in visitor.wrt: # This is a normal situation - print("NORMAL: %s" % child) _id = id(child) if _id not in visitor.var_map: if child.fixed: @@ -59,7 +58,6 @@ def _before_var(visitor, child): ans.linear[_id] = 1 return False, (ExprType.LINEAR, ans) else: - print("DATA: %s" % child) # We aren't treating this Var as a Var for the purposes of this walker return False, (ExprType.CONSTANT, child) @@ -85,14 +83,17 @@ def finalizeResult(self, result): self._factor_multiplier_into_linear_terms(ans, mult) return ans if mult == 1: - for vid, coef in ans.linear.items(): - if coef.__class__ in native_numeric_types and not coef: - del ans.linear[vid] + zeros = [(vid, coef) for vid, coef in ans.linear.items() if + coef.__class__ in native_numeric_types and not coef] + for vid, coef in zeros: + del ans.linear[vid] elif not mult: # the mulltiplier has cleared out the entire expression. # Warn if this is suppressing a NaN (unusual, and # non-standard, but we will wait to remove this behavior # for the time being) + # ESJ TODO: This won't work either actually... + # I'm not sure how to do it. if ans.constant != ans.constant or any( c != c for c in ans.linear.values() ): diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 35e7160a351..d00668d5c00 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -9,9 +9,10 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from pyomo.common.log import LoggingIntercept import pyomo.common.unittest as unittest from pyomo.core.expr.compare import assertExpressionsEqual -from pyomo.environ import Binary, ConcreteModel, Var, log +from pyomo.environ import Any, Binary, ConcreteModel, log, Param, Var from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor from pyomo.repn.tests.test_linear import VisitorConfig @@ -153,3 +154,111 @@ def test_nonlinear(self): assertExpressionsEqual( self, repn.to_expression(visitor), log(m.x) * (m.y * (m.y + 2)) / m.x ) + + def test_finalize(self): + m = self.make_model() + m.z = Var() + m.w = Var() + + e = m.x + 2 * m.w**2 * m.y - m.x - m.w * m.z + + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + self.assertEqual(cfg.subexpr, {}) + self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) + self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.y), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.y)], + 2 * m.w ** 2 + ) + self.assertIn(id(m.z), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.z)], + -m.w + ) + self.assertEqual(repn.nonlinear, None) + + e *= 5 + + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + self.assertEqual(cfg.subexpr, {}) + self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) + self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.y), repn.linear) + print(repn.linear[id(m.y)]) + assertExpressionsEqual( + self, + repn.linear[id(m.y)], + 5 * (2 * m.w ** 2) + ) + self.assertIn(id(m.z), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.z)], + -5 * m.w + ) + self.assertEqual(repn.nonlinear, None) + + e = 5 * (m.w * m.y + m.z**2 + 3 * m.w * m.y**3) + + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + self.assertEqual(cfg.subexpr, {}) + self.assertEqual(cfg.var_map, {id(m.y): m.y, id(m.z): m.z}) + self.assertEqual(cfg.var_order, {id(m.y): 0, id(m.z): 1}) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.y)], + 5 * m.w + ) + assertExpressionsEqual(self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5) + + def test_errors_propogate_nan(self): + m = ConcreteModel() + m.p = Param(mutable=True, initialize=0, domain=Any) + m.x = Var() + m.y = Var() + m.z = Var() + m.y.fix(1) + + expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y + cfg = VisitorConfig() + with LoggingIntercept() as LOG: + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + self.assertEqual( + LOG.getvalue(), + "Exception encountered evaluating expression 'div(3, 0)'\n" + "\tmessage: division by zero\n" + "\texpression: 3/p\n", + ) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, + repn.constant, + 1 + m.z + ) + self.assertEqual(len(repn.linear), 1) + self.assertEqual(str(repn.linear[id(m.x)]), 'InvalidNumber(nan)') + self.assertEqual(repn.nonlinear, None) + + m.y.fix(None) + expr = m.z * log(m.y) + 3 + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertEqual(repn.linear, {}) + self.assertEqual(repn.nonlinear, None) From 259ee576ed2673fa30dacab41283e1c152b9ffe6 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 23 Apr 2024 15:09:30 -0600 Subject: [PATCH 022/150] Reverting my changes to linear--I'll override stuff that needs to handle expressions so that I don't kill performance --- pyomo/repn/linear.py | 101 ++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 64 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 079fbb86489..ba08c7ef245 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -62,12 +62,7 @@ def _merge_dict(dest_dict, mult, src_dict): - try: - _mult = mult != 1 - except: - _mult = True - - if not _mult: + if mult == 1: for vid, coef in src_dict.items(): if vid in dest_dict: dest_dict[vid] += coef @@ -127,18 +122,15 @@ def to_expression(self, visitor): var_map = visitor.var_map with mutable_expression() as e: for vid, coef in self.linear.items(): - if coef.__class__ not in native_numeric_types or coef: + if coef: e += coef * var_map[vid] if e.nargs() > 1: ans += e elif e.nargs() == 1: ans += e.arg(0) - if self.constant.__class__ not in native_numeric_types or self.constant: + if self.constant: ans += self.constant - if ( - self.multiplier.__class__ not in native_numeric_types - or self.multiplier != 1 - ): + if self.multiplier != 1: ans *= self.multiplier return ans @@ -155,49 +147,33 @@ def append(self, other): callback). """ + # Note that self.multiplier will always be 1 (we only call append() + # within a sum, so there is no opportunity for self.multiplier to + # change). Omitting the assertion for efficiency. + # assert self.multiplier == 1 _type, other = other if _type is _CONSTANT: self.constant += other return mult = other.multiplier - try: - _mult = bool(mult) - if not _mult: - return - if mult == 1: - _mult = False - except: - _mult = True - - const = other.constant - try: - _const = bool(const) - except: - _const = True - - if _mult: - if _const: - self.constant += mult * const - if other.linear: - _merge_dict(self.linear, mult, other.linear) - if other.nonlinear is not None: + if not mult: + # 0 * other, so there is nothing to add/change about + # self. We can just exit now. + return + if other.constant: + self.constant += mult * other.constant + if other.linear: + _merge_dict(self.linear, mult, other.linear) + if other.nonlinear is not None: + if mult != 1: nl = mult * other.nonlinear - if self.nonlinear is None: - self.nonlinear = nl - else: - self.nonlinear += nl - else: - if _const: - self.constant += const - if other.linear: - _merge_dict(self.linear, 1, other.linear) - if other.nonlinear is not None: + else: nl = other.nonlinear - if self.nonlinear is None: - self.nonlinear = nl - else: - self.nonlinear += nl + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl def to_expression(visitor, arg): @@ -827,22 +803,6 @@ def exitNode(self, node, data): self, node, *data ) - def _factor_multiplier_into_linear_terms(self, ans, mult): - linear = ans.linear - zeros = [] - for vid, coef in linear.items(): - if coef.__class__ not in native_numeric_types or coef: - linear[vid] = mult * coef - else: - zeros.append(vid) - for vid in zeros: - del linear[vid] - if ans.nonlinear is not None: - ans.nonlinear *= mult - if ans.constant.__class__ not in native_numeric_types or ans.constant: - ans.constant *= mult - ans.multiplier = 1 - def finalizeResult(self, result): ans = result[1] if ans.__class__ is self.Result: @@ -871,7 +831,20 @@ def finalizeResult(self, result): else: # mult not in {0, 1}: factor it into the constant, # linear coefficients, and nonlinear term - self._factor_multiplier_into_linear_terms(ans, mult) + linear = ans.linear + zeros = [] + for vid, coef in linear.items(): + if coef: + linear[vid] = coef * mult + else: + zeros.append(vid) + for vid in zeros: + del linear[vid] + if ans.nonlinear is not None: + ans.nonlinear *= mult + if ans.constant: + ans.constant *= mult + ans.multiplier = 1 return ans ans = self.Result() assert result[0] is _CONSTANT From 7886519da92c7da509aad346304df7daa5f9a2b1 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 23 Apr 2024 15:20:54 -0600 Subject: [PATCH 023/150] Adding a new LinearSubsystemRepn class to handle LinearRepns with 'constants' that are actually Pyomo expressions --- pyomo/repn/linear_wrt.py | 156 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 4 deletions(-) diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index 8fa1f9b4546..bbd4c6a7d25 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -9,6 +9,8 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import copy + from pyomo.common.collections import ComponentSet from pyomo.common.numeric_types import native_numeric_types from pyomo.core import Var @@ -16,10 +18,122 @@ from pyomo.core.expr.numeric_expr import ( LinearExpression, MonomialTermExpression, + mutable_expression, + ProductExpression, SumExpression, ) -from pyomo.repn.linear import LinearBeforeChildDispatcher, LinearRepnVisitor +from pyomo.repn.linear import ( + ExitNodeDispatcher, + _initialize_exit_node_dispatcher, + LinearBeforeChildDispatcher, + LinearRepn, + LinearRepnVisitor, +) from pyomo.repn.util import ExprType +from . import linear + +_CONSTANT = ExprType.CONSTANT + + +def _merge_dict(dest_dict, mult, src_dict): + if mult.__class__ not in native_numeric_types or mult != 1: + for vid, coef in src_dict.items(): + if vid in dest_dict: + dest_dict[vid] += mult * coef + else: + dest_dict[vid] = mult * coef + else: + for vid, coef in src_dict.items(): + if vid in dest_dict: + dest_dict[vid] += coef + else: + dest_dict[vid] = coef + + +class LinearSubsystemRepn(LinearRepn): + def to_expression(self, visitor): + if self.nonlinear is not None: + # We want to start with the nonlinear term (and use + # assignment) in case the term is a non-numeric node (like a + # relational expression) + ans = self.nonlinear + else: + ans = 0 + if self.linear: + var_map = visitor.var_map + with mutable_expression() as e: + for vid, coef in self.linear.items(): + if coef.__class__ not in native_numeric_types or coef: + e += coef * var_map[vid] + if e.nargs() > 1: + ans += e + elif e.nargs() == 1: + ans += e.arg(0) + if self.constant.__class__ not in native_numeric_types or self.constant: + ans += self.constant + if ( + self.multiplier.__class__ not in native_numeric_types + or self.multiplier != 1 + ): + ans *= self.multiplier + return ans + + def append(self, other): + """Append a child result from acceptChildResult + + Notes + ----- + This method assumes that the operator was "+". It is implemented + so that we can directly use a LinearRepn() as a `data` object in + the expression walker (thereby allowing us to use the default + implementation of acceptChildResult [which calls + `data.append()`] and avoid the function call for a custom + callback). + + """ + _type, other = other + if _type is _CONSTANT: + self.constant += other + return + + mult = other.multiplier + try: + _mult = bool(mult) + if not _mult: + return + if mult == 1: + _mult = False + except: + _mult = True + + const = other.constant + try: + _const = bool(const) + except: + _const = True + + if _mult: + if _const: + self.constant += mult * const + if other.linear: + _merge_dict(self.linear, mult, other.linear) + if other.nonlinear is not None: + nl = mult * other.nonlinear + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl + else: + if _const: + self.constant += const + if other.linear: + _merge_dict(self.linear, 1, other.linear) + if other.nonlinear is not None: + nl = other.nonlinear + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl class MultiLevelLinearBeforeChildDispatcher(LinearBeforeChildDispatcher): @@ -50,7 +164,7 @@ def _before_var(visitor, child): if _id not in visitor.var_map: if child.fixed: return False, ( - ExprType.CONSTANT, + _CONSTANT, visitor.check_constant(child.value, child), ) MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) @@ -59,14 +173,32 @@ def _before_var(visitor, child): return False, (ExprType.LINEAR, ans) else: # We aren't treating this Var as a Var for the purposes of this walker - return False, (ExprType.CONSTANT, child) + return False, (_CONSTANT, child) _before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() +_exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) +def _handle_product_constant_constant(visitor, node, arg1, arg2): + # ESJ: Can I do this? Just let the potential nans go through? + return _CONSTANT, arg1[1] * arg2[1] + +_exit_node_handlers[ProductExpression].update( + { + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, + } +) + + # LinearSubsystemRepnVisitor class MultilevelLinearRepnVisitor(LinearRepnVisitor): + Result = LinearSubsystemRepn + exit_node_handlers = _exit_node_handlers + exit_node_dispatcher = ExitNodeDispatcher( + _initialize_exit_node_dispatcher(_exit_node_handlers) + ) + def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): super().__init__(subexpression_cache, var_map, var_order, sorter) self.wrt = ComponentSet(_flattened(wrt)) @@ -74,6 +206,22 @@ def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): def beforeChild(self, node, child, child_idx): return _before_child_dispatcher[child.__class__](self, child) + def _factor_multiplier_into_linear_terms(self, ans, mult): + linear = ans.linear + zeros = [] + for vid, coef in linear.items(): + if coef.__class__ not in native_numeric_types or coef: + linear[vid] = mult * coef + else: + zeros.append(vid) + for vid in zeros: + del linear[vid] + if ans.nonlinear is not None: + ans.nonlinear *= mult + if ans.constant.__class__ not in native_numeric_types or ans.constant: + ans.constant *= mult + ans.multiplier = 1 + def finalizeResult(self, result): ans = result[1] if ans.__class__ is self.Result: @@ -112,6 +260,6 @@ def finalizeResult(self, result): return ans ans = self.Result() - assert result[0] is ExprType.CONSTANT + assert result[0] is _CONSTANT ans.constant = result[1] return ans From e09953a615370ec0f1c97c35d2ace2781377db0a Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 24 Apr 2024 09:54:11 -0600 Subject: [PATCH 024/150] Correcting nan propogation test assertion --- pyomo/repn/tests/test_linear_wrt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index d00668d5c00..43bc0a51914 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -241,15 +241,15 @@ def test_errors_propogate_nan(self): repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) self.assertEqual( LOG.getvalue(), - "Exception encountered evaluating expression 'div(3, 0)'\n" + "Exception encountered evaluating expression 'div(3*z, 0)'\n" "\tmessage: division by zero\n" - "\texpression: 3/p\n", + "\texpression: 3*z*x/p\n", ) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual( self, repn.constant, - 1 + m.z + m.y + m.z ) self.assertEqual(len(repn.linear), 1) self.assertEqual(str(repn.linear[id(m.x)]), 'InvalidNumber(nan)') From d51bb99711b20ce3661c2e112fd804f9fd45d9b3 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 24 Apr 2024 11:40:35 -0600 Subject: [PATCH 025/150] Adding handling of nan in assertExpressionsEqual --- pyomo/core/expr/compare.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pyomo/core/expr/compare.py b/pyomo/core/expr/compare.py index 790bc30aaee..e57e65a08f0 100644 --- a/pyomo/core/expr/compare.py +++ b/pyomo/core/expr/compare.py @@ -230,10 +230,14 @@ def assertExpressionsEqual(test, a, b, include_named_exprs=True, places=None): test.assertEqual(len(prefix_a), len(prefix_b)) for _a, _b in zip(prefix_a, prefix_b): test.assertIs(_a.__class__, _b.__class__) - if places is None: - test.assertEqual(_a, _b) + # If _a is nan, check _b is nan + if _a != _a: + test.assertTrue(_b != _b) else: - test.assertAlmostEqual(_a, _b, places=places) + if places is None: + test.assertEqual(_a, _b) + else: + test.assertAlmostEqual(_a, _b, places=places) except (PyomoException, AssertionError): test.fail( f"Expressions not equal:\n\t" @@ -292,10 +296,13 @@ def assertExpressionsStructurallyEqual( for _a, _b in zip(prefix_a, prefix_b): if _a.__class__ not in native_types and _b.__class__ not in native_types: test.assertIs(_a.__class__, _b.__class__) - if places is None: - test.assertEqual(_a, _b) + if _a != _a: + test.assertTrue(_b != _b) else: - test.assertAlmostEqual(_a, _b, places=places) + if places is None: + test.assertEqual(_a, _b) + else: + test.assertAlmostEqual(_a, _b, places=places) except (PyomoException, AssertionError): test.fail( f"Expressions not structurally equal:\n\t" From ec5879b5191aa1fbc525867d3ed48d0e55294e50 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 24 Apr 2024 11:44:22 -0600 Subject: [PATCH 026/150] nan propogation tests are finally passing --- pyomo/repn/tests/test_linear_wrt.py | 43 +++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 43bc0a51914..ef86aed47f8 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -15,6 +15,7 @@ from pyomo.environ import Any, Binary, ConcreteModel, log, Param, Var from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor from pyomo.repn.tests.test_linear import VisitorConfig +from pyomo.repn.util import InvalidNumber class TestMultilevelLinearRepnVisitor(unittest.TestCase): @@ -227,6 +228,34 @@ def test_finalize(self): ) assertExpressionsEqual(self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5) + def test_ANY_over_constant_division(self): + m = ConcreteModel() + m.p = Param(mutable=True, initialize=2, domain=Any) + m.x = Var() + m.y = Var() + m.z = Var() + # We aren't treating this as a Var, so we don't really care that it's fixed. + m.y.fix(1) + + expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, + repn.constant, + m.y + m.z + ) + self.assertEqual(len(repn.linear), 1) + print(repn.linear[id(m.x)]) + assertExpressionsEqual( + self, + repn.linear[id(m.x)], + 1 + 1.5 * m.z / m.y + ) + self.assertEqual(repn.nonlinear, None) + def test_errors_propogate_nan(self): m = ConcreteModel() m.p = Param(mutable=True, initialize=0, domain=Any) @@ -252,13 +281,23 @@ def test_errors_propogate_nan(self): m.y + m.z ) self.assertEqual(len(repn.linear), 1) - self.assertEqual(str(repn.linear[id(m.x)]), 'InvalidNumber(nan)') + self.assertIsInstance(repn.linear[id(m.x)], InvalidNumber) + assertExpressionsEqual( + self, + repn.linear[id(m.x)].value, + 1 + float('nan')/m.y + ) self.assertEqual(repn.nonlinear, None) m.y.fix(None) expr = m.z * log(m.y) + 3 repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertIsInstance(repn.constant, InvalidNumber) + assertExpressionsEqual( + self, + repn.constant.value, + float('nan')*m.z + 3 + ) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) From 4f5f50e64f6cddbc4ae9dd227deba5225575e320 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 29 Apr 2024 13:59:50 -0600 Subject: [PATCH 027/150] More sum tests --- pyomo/repn/tests/test_linear_wrt.py | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index ef86aed47f8..6853c0df7f6 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -62,6 +62,39 @@ def test_walk_triple_sum(self): self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.z * m.y + m.z) + def test_sum_two_of_the_same(self): + # This hits the mult == 1 and vid in dest_dict case in _merge_dict + m = self.make_model() + e = m.x + m.x + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 2) + self.assertEqual(repn.constant, 0) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), 2*m.x) + + def test_sum_with_mult_0(self): + m = self.make_model() + e = 0*m.x + m.x + m.y + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 1) + self.assertIs(repn.constant, m.y) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.y) + def test_bilinear_term(self): m = self.make_model() e = m.x * m.y From c9659c8c5a243959f7752f1d75a4f79ee53d6e4c Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 29 Apr 2024 14:24:51 -0600 Subject: [PATCH 028/150] Tests for everything in append --- pyomo/repn/tests/test_linear_wrt.py | 65 +++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 6853c0df7f6..a9a31ce8232 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -95,6 +95,71 @@ def test_sum_with_mult_0(self): self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.y) + def test_sum_nonlinear_to_linear(self): + m = self.make_model() + e = m.y * m.x**2 + m.y * m.x + 3 + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + assertExpressionsEqual( + self, + repn.nonlinear, + m.y * m.x ** 2 + ) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertIs(repn.linear[id(m.x)], m.y) + self.assertEqual(repn.constant, 3) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x ** 2 + + m.y * m.x + 3) + + def test_sum_nonlinear_to_nonlinear(self): + m = self.make_model() + e = m.x ** 3 + 3 + m.x**2 + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + assertExpressionsEqual( + self, + repn.nonlinear, + m.x ** 3 + m.x ** 2 + ) + self.assertEqual(repn.constant, 3) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x ** 3 + + m.x ** 2 + 3) + + def test_sum_to_linear_expr(self): + m = self.make_model() + e = m.x + m.y * (m.x + 5) + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.x)], + 1 + m.y + ) + assertExpressionsEqual( + self, + repn.constant, + m.y * 5 + ) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, + repn.to_expression(visitor), (1 + m.y) * m.x + m.y * 5 + ) + def test_bilinear_term(self): m = self.make_model() e = m.x * m.y From 7f224c8c1f4c0ae79b35d75269441042b8429186 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 30 Apr 2024 13:43:52 -0600 Subject: [PATCH 029/150] Not blindly evaluating unary functions, but that actually causes me a whole conundrum about fixed variables --- pyomo/repn/linear_wrt.py | 24 ++++++++++++- pyomo/repn/tests/test_linear_wrt.py | 55 +++++++++++++++++++++++------ 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index bbd4c6a7d25..c5cd6c130b0 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -16,11 +16,13 @@ from pyomo.core import Var from pyomo.core.expr.logical_expr import _flattened from pyomo.core.expr.numeric_expr import ( + AbsExpression, LinearExpression, MonomialTermExpression, mutable_expression, ProductExpression, SumExpression, + UnaryFunctionExpression, ) from pyomo.repn.linear import ( ExitNodeDispatcher, @@ -183,13 +185,33 @@ def _before_var(visitor, child): def _handle_product_constant_constant(visitor, node, arg1, arg2): # ESJ: Can I do this? Just let the potential nans go through? return _CONSTANT, arg1[1] * arg2[1] + _exit_node_handlers[ProductExpression].update( { (_CONSTANT, _CONSTANT): _handle_product_constant_constant, } ) - + +def _handle_unary_constant(visitor, node, arg): + # We override this because we can't blindly use apply_node_operation in this case + if arg.__class__ not in native_numeric_types: + return _CONSTANT, node.create_node_with_local_data( + (linear.to_expression(visitor, arg),)) + # otherwise do the usual: + ans = apply_node_operation(node, (arg[1],)) + # Unary includes sqrt() which can return complex numbers + if ans.__class__ in native_complex_types: + ans = complex_number_error(ans, visitor, node) + return _CONSTANT, ans + +_exit_node_handlers[UnaryFunctionExpression].update( + { + (_CONSTANT,): _handle_unary_constant + } +) +_exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] + # LinearSubsystemRepnVisitor class MultilevelLinearRepnVisitor(LinearRepnVisitor): diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index a9a31ce8232..fa7aaf7799b 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -23,6 +23,7 @@ def make_model(self): m = ConcreteModel() m.x = Var(bounds=(0, 45)) m.y = Var(domain=Binary) + m.z = Var() return m @@ -44,7 +45,6 @@ def test_walk_sum(self): def test_walk_triple_sum(self): m = self.make_model() - m.z = Var() e = m.x + m.z * m.y + m.z cfg = VisitorConfig() @@ -81,7 +81,7 @@ def test_sum_two_of_the_same(self): def test_sum_with_mult_0(self): m = self.make_model() - e = 0*m.x + m.x + m.y + e = 0*m.x + m.x - m.y cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) @@ -91,13 +91,17 @@ def test_sum_with_mult_0(self): self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertEqual(repn.linear[id(m.x)], 1) - self.assertIs(repn.constant, m.y) + assertExpressionsEqual( + self, + repn.constant, + - m.y + ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.y) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x - m.y) def test_sum_nonlinear_to_linear(self): m = self.make_model() - e = m.y * m.x**2 + m.y * m.x + 3 + e = m.y * m.x**2 + m.y * m.x - 3 cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) @@ -111,10 +115,10 @@ def test_sum_nonlinear_to_linear(self): self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertIs(repn.linear[id(m.x)], m.y) - self.assertEqual(repn.constant, 3) + self.assertEqual(repn.constant, -3) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x ** 2 - + m.y * m.x + 3) + + m.y * m.x - 3) def test_sum_nonlinear_to_nonlinear(self): m = self.make_model() @@ -256,7 +260,6 @@ def test_nonlinear(self): def test_finalize(self): m = self.make_model() - m.z = Var() m.w = Var() e = m.x + 2 * m.w**2 * m.y - m.x - m.w * m.z @@ -330,8 +333,8 @@ def test_ANY_over_constant_division(self): m = ConcreteModel() m.p = Param(mutable=True, initialize=2, domain=Any) m.x = Var() - m.y = Var() m.z = Var() + m.y = Var() # We aren't treating this as a Var, so we don't really care that it's fixed. m.y.fix(1) @@ -358,8 +361,8 @@ def test_errors_propogate_nan(self): m = ConcreteModel() m.p = Param(mutable=True, initialize=0, domain=Any) m.x = Var() - m.y = Var() m.z = Var() + m.y = Var() m.y.fix(1) expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y @@ -399,3 +402,35 @@ def test_errors_propogate_nan(self): ) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) + + def test_negation_constant(self): + m = self.make_model() + e = - (m.y * m.z + 17) + + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, + repn.constant, + - 1 * (m.y * m.z + 17) + ) + self.assertIsNone(repn.nonlinear) + + def test_product_nonlinear(self): + m = self.make_model() + e = (m.x ** 2) * (log(m.y) * m.z ** 4) * m.y + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.z]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + print(repn.nonlinear) + assertExpressionsEqual( + self, + repn.nonlinear, + (m.x ** 2) * (m.z ** 4 * log(m.y)) * m.y + ) From 20dd5aaf73bc279273c145f2ae6979c920bf5855 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 2 May 2024 15:57:06 -0600 Subject: [PATCH 030/150] Renaming my walker and repn to something sane, but that meant changing the meaning of the 'wrt' argument --- ...{linear_wrt.py => parameterized_linear.py} | 31 +-- ...ar_wrt.py => test_parameterized_linear.py} | 179 ++++++------------ 2 files changed, 77 insertions(+), 133 deletions(-) rename pyomo/repn/{linear_wrt.py => parameterized_linear.py} (94%) rename pyomo/repn/tests/{test_linear_wrt.py => test_parameterized_linear.py} (73%) diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/parameterized_linear.py similarity index 94% rename from pyomo/repn/linear_wrt.py rename to pyomo/repn/parameterized_linear.py index c5cd6c130b0..9df0ea458db 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/parameterized_linear.py @@ -52,7 +52,7 @@ def _merge_dict(dest_dict, mult, src_dict): dest_dict[vid] = coef -class LinearSubsystemRepn(LinearRepn): +class ParameterizedLinearRepn(LinearRepn): def to_expression(self, visitor): if self.nonlinear is not None: # We want to start with the nonlinear term (and use @@ -160,7 +160,7 @@ def _before_general_expression(visitor, child): @staticmethod def _before_var(visitor, child): - if child in visitor.wrt: + if child not in visitor.wrt: # This is a normal situation _id = id(child) if _id not in visitor.var_map: @@ -185,19 +185,19 @@ def _before_var(visitor, child): def _handle_product_constant_constant(visitor, node, arg1, arg2): # ESJ: Can I do this? Just let the potential nans go through? return _CONSTANT, arg1[1] * arg2[1] - + _exit_node_handlers[ProductExpression].update( - { - (_CONSTANT, _CONSTANT): _handle_product_constant_constant, - } + {(_CONSTANT, _CONSTANT): _handle_product_constant_constant} ) + def _handle_unary_constant(visitor, node, arg): # We override this because we can't blindly use apply_node_operation in this case if arg.__class__ not in native_numeric_types: return _CONSTANT, node.create_node_with_local_data( - (linear.to_expression(visitor, arg),)) + (linear.to_expression(visitor, arg),) + ) # otherwise do the usual: ans = apply_node_operation(node, (arg[1],)) # Unary includes sqrt() which can return complex numbers @@ -205,17 +205,15 @@ def _handle_unary_constant(visitor, node, arg): ans = complex_number_error(ans, visitor, node) return _CONSTANT, ans + _exit_node_handlers[UnaryFunctionExpression].update( - { - (_CONSTANT,): _handle_unary_constant - } + {(_CONSTANT,): _handle_unary_constant} ) _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] -# LinearSubsystemRepnVisitor -class MultilevelLinearRepnVisitor(LinearRepnVisitor): - Result = LinearSubsystemRepn +class ParameterizedLinearRepnVisitor(LinearRepnVisitor): + Result = ParameterizedLinearRepn exit_node_handlers = _exit_node_handlers exit_node_dispatcher = ExitNodeDispatcher( _initialize_exit_node_dispatcher(_exit_node_handlers) @@ -253,8 +251,11 @@ def finalizeResult(self, result): self._factor_multiplier_into_linear_terms(ans, mult) return ans if mult == 1: - zeros = [(vid, coef) for vid, coef in ans.linear.items() if - coef.__class__ in native_numeric_types and not coef] + zeros = [ + (vid, coef) + for vid, coef in ans.linear.items() + if coef.__class__ in native_numeric_types and not coef + ] for vid, coef in zeros: del ans.linear[vid] elif not mult: diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_parameterized_linear.py similarity index 73% rename from pyomo/repn/tests/test_linear_wrt.py rename to pyomo/repn/tests/test_parameterized_linear.py index fa7aaf7799b..32f58dbfc13 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -13,12 +13,12 @@ import pyomo.common.unittest as unittest from pyomo.core.expr.compare import assertExpressionsEqual from pyomo.environ import Any, Binary, ConcreteModel, log, Param, Var -from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor +from pyomo.repn.parameterized_linear import ParameterizedLinearRepnVisitor from pyomo.repn.tests.test_linear import VisitorConfig from pyomo.repn.util import InvalidNumber -class TestMultilevelLinearRepnVisitor(unittest.TestCase): +class TestParameterizedLinearRepnVisitor(unittest.TestCase): def make_model(self): m = ConcreteModel() m.x = Var(bounds=(0, 45)) @@ -31,7 +31,7 @@ def test_walk_sum(self): m = self.make_model() e = m.x + m.y cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -48,7 +48,7 @@ def test_walk_triple_sum(self): e = m.x + m.z * m.y + m.z cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]) repn = visitor.walk_expression(e) @@ -67,7 +67,7 @@ def test_sum_two_of_the_same(self): m = self.make_model() e = m.x + m.x cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]) repn = visitor.walk_expression(e) @@ -77,25 +77,21 @@ def test_sum_two_of_the_same(self): self.assertEqual(repn.linear[id(m.x)], 2) self.assertEqual(repn.constant, 0) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.to_expression(visitor), 2*m.x) + assertExpressionsEqual(self, repn.to_expression(visitor), 2 * m.x) def test_sum_with_mult_0(self): m = self.make_model() - e = 0*m.x + m.x - m.y - + e = 0 * m.x + m.x - m.y + cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) self.assertIsNone(repn.nonlinear) self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertEqual(repn.linear[id(m.x)], 1) - assertExpressionsEqual( - self, - repn.constant, - - m.y - ) + assertExpressionsEqual(self, repn.constant, -m.y) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.to_expression(visitor), m.x - m.y) @@ -104,71 +100,56 @@ def test_sum_nonlinear_to_linear(self): e = m.y * m.x**2 + m.y * m.x - 3 cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) - assertExpressionsEqual( - self, - repn.nonlinear, - m.y * m.x ** 2 - ) + assertExpressionsEqual(self, repn.nonlinear, m.y * m.x**2) self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertIs(repn.linear[id(m.x)], m.y) self.assertEqual(repn.constant, -3) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x ** 2 - + m.y * m.x - 3) + assertExpressionsEqual( + self, repn.to_expression(visitor), m.y * m.x**2 + m.y * m.x - 3 + ) def test_sum_nonlinear_to_nonlinear(self): m = self.make_model() - e = m.x ** 3 + 3 + m.x**2 + e = m.x**3 + 3 + m.x**2 cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) - assertExpressionsEqual( - self, - repn.nonlinear, - m.x ** 3 + m.x ** 2 - ) + assertExpressionsEqual(self, repn.nonlinear, m.x**3 + m.x**2) self.assertEqual(repn.constant, 3) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.to_expression(visitor), m.x ** 3 - + m.x ** 2 + 3) + assertExpressionsEqual( + self, repn.to_expression(visitor), m.x**3 + m.x**2 + 3 + ) def test_sum_to_linear_expr(self): m = self.make_model() e = m.x + m.y * (m.x + 5) cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) - assertExpressionsEqual( - self, - repn.linear[id(m.x)], - 1 + m.y - ) - assertExpressionsEqual( - self, - repn.constant, - m.y * 5 - ) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + m.y) + assertExpressionsEqual(self, repn.constant, m.y * 5) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual( - self, - repn.to_expression(visitor), (1 + m.y) * m.x + m.y * 5 + self, repn.to_expression(visitor), (1 + m.y) * m.x + m.y * 5 ) def test_bilinear_term(self): m = self.make_model() e = m.x * m.y cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -184,7 +165,7 @@ def test_distributed_bilinear_term(self): m = self.make_model() e = m.y * (m.x + 7) cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -200,7 +181,7 @@ def test_monomial(self): m = self.make_model() e = 45 * m.y cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.y]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x, m.z]) repn = visitor.walk_expression(e) @@ -216,7 +197,7 @@ def test_constant(self): m = self.make_model() e = 45 * m.y cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -232,7 +213,7 @@ def test_fixed_var(self): e = (m.y**2) * (m.x + m.x**2) cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -247,7 +228,7 @@ def test_nonlinear(self): e = (m.y * log(m.x)) * (m.y + 2) / m.x cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -265,7 +246,7 @@ def test_finalize(self): e = m.x + 2 * m.w**2 * m.y - m.x - m.w * m.z cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.w]).walk_expression(e) self.assertEqual(cfg.subexpr, {}) self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) @@ -273,23 +254,15 @@ def test_finalize(self): self.assertEqual(repn.constant, 0) self.assertEqual(len(repn.linear), 2) self.assertIn(id(m.y), repn.linear) - assertExpressionsEqual( - self, - repn.linear[id(m.y)], - 2 * m.w ** 2 - ) + assertExpressionsEqual(self, repn.linear[id(m.y)], 2 * m.w**2) self.assertIn(id(m.z), repn.linear) - assertExpressionsEqual( - self, - repn.linear[id(m.z)], - -m.w - ) + assertExpressionsEqual(self, repn.linear[id(m.z)], -m.w) self.assertEqual(repn.nonlinear, None) e *= 5 cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.w]).walk_expression(e) self.assertEqual(cfg.subexpr, {}) self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) @@ -298,23 +271,15 @@ def test_finalize(self): self.assertEqual(len(repn.linear), 2) self.assertIn(id(m.y), repn.linear) print(repn.linear[id(m.y)]) - assertExpressionsEqual( - self, - repn.linear[id(m.y)], - 5 * (2 * m.w ** 2) - ) + assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * (2 * m.w**2)) self.assertIn(id(m.z), repn.linear) - assertExpressionsEqual( - self, - repn.linear[id(m.z)], - -5 * m.w - ) + assertExpressionsEqual(self, repn.linear[id(m.z)], -5 * m.w) self.assertEqual(repn.nonlinear, None) e = 5 * (m.w * m.y + m.z**2 + 3 * m.w * m.y**3) cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.w]).walk_expression(e) self.assertEqual(cfg.subexpr, {}) self.assertEqual(cfg.var_map, {id(m.y): m.y, id(m.z): m.z}) self.assertEqual(cfg.var_order, {id(m.y): 0, id(m.z): 1}) @@ -322,12 +287,10 @@ def test_finalize(self): self.assertEqual(repn.constant, 0) self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.y), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * m.w) assertExpressionsEqual( - self, - repn.linear[id(m.y)], - 5 * m.w + self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5 ) - assertExpressionsEqual(self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5) def test_ANY_over_constant_division(self): m = ConcreteModel() @@ -340,21 +303,15 @@ def test_ANY_over_constant_division(self): expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression( + expr + ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual( - self, - repn.constant, - m.y + m.z - ) + assertExpressionsEqual(self, repn.constant, m.y + m.z) self.assertEqual(len(repn.linear), 1) print(repn.linear[id(m.x)]) - assertExpressionsEqual( - self, - repn.linear[id(m.x)], - 1 + 1.5 * m.z / m.y - ) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z / m.y) self.assertEqual(repn.nonlinear, None) def test_errors_propogate_nan(self): @@ -368,7 +325,9 @@ def test_errors_propogate_nan(self): expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y cfg = VisitorConfig() with LoggingIntercept() as LOG: - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression( + expr + ) self.assertEqual( LOG.getvalue(), "Exception encountered evaluating expression 'div(3*z, 0)'\n" @@ -376,61 +335,45 @@ def test_errors_propogate_nan(self): "\texpression: 3*z*x/p\n", ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual( - self, - repn.constant, - m.y + m.z - ) + assertExpressionsEqual(self, repn.constant, m.y + m.z) self.assertEqual(len(repn.linear), 1) self.assertIsInstance(repn.linear[id(m.x)], InvalidNumber) - assertExpressionsEqual( - self, - repn.linear[id(m.x)].value, - 1 + float('nan')/m.y - ) + assertExpressionsEqual(self, repn.linear[id(m.x)].value, 1 + float('nan') / m.y) self.assertEqual(repn.nonlinear, None) m.y.fix(None) expr = m.z * log(m.y) + 3 - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression( + expr + ) self.assertEqual(repn.multiplier, 1) self.assertIsInstance(repn.constant, InvalidNumber) - assertExpressionsEqual( - self, - repn.constant.value, - float('nan')*m.z + 3 - ) + assertExpressionsEqual(self, repn.constant.value, float('nan') * m.z + 3) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) def test_negation_constant(self): m = self.make_model() - e = - (m.y * m.z + 17) + e = -(m.y * m.z + 17) cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression(e) self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual( - self, - repn.constant, - - 1 * (m.y * m.z + 17) - ) + assertExpressionsEqual(self, repn.constant, -1 * (m.y * m.z + 17)) self.assertIsNone(repn.nonlinear) - + def test_product_nonlinear(self): m = self.make_model() - e = (m.x ** 2) * (log(m.y) * m.z ** 4) * m.y + e = (m.x**2) * (log(m.y) * m.z**4) * m.y cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.z]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 0) print(repn.nonlinear) assertExpressionsEqual( - self, - repn.nonlinear, - (m.x ** 2) * (m.z ** 4 * log(m.y)) * m.y + self, repn.nonlinear, (m.x**2) * (m.z**4 * log(m.y)) * m.y ) From b8d91c86473fd8709fc6b430fecd4bcd2fbddff6 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Sun, 12 May 2024 11:54:04 -0600 Subject: [PATCH 031/150] Fixing before_var handler so that we always use the values of fixed Vars regardless of if they are parameters or Vars from the perspective of the walker --- pyomo/repn/parameterized_linear.py | 33 ++++++++++--------- pyomo/repn/tests/test_parameterized_linear.py | 13 +++++--- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 9df0ea458db..b180ee91862 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -160,22 +160,23 @@ def _before_general_expression(visitor, child): @staticmethod def _before_var(visitor, child): - if child not in visitor.wrt: + _id = id(child) + if _id not in visitor.var_map: + if child.fixed: + return False, ( + _CONSTANT, + visitor.check_constant(child.value, child), + ) + if child in visitor.wrt: + # psueudo-constant + # We aren't treating this Var as a Var for the purposes of this walker + return False, (_CONSTANT, child) # This is a normal situation - _id = id(child) - if _id not in visitor.var_map: - if child.fixed: - return False, ( - _CONSTANT, - visitor.check_constant(child.value, child), - ) - MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) - ans = visitor.Result() - ans.linear[_id] = 1 - return False, (ExprType.LINEAR, ans) - else: - # We aren't treating this Var as a Var for the purposes of this walker - return False, (_CONSTANT, child) + # TODO: override record var to not record things in wrt + MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) + ans = visitor.Result() + ans.linear[_id] = 1 + return False, (ExprType.LINEAR, ans) _before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() @@ -259,7 +260,7 @@ def finalizeResult(self, result): for vid, coef in zeros: del ans.linear[vid] elif not mult: - # the mulltiplier has cleared out the entire expression. + # the multiplier has cleared out the entire expression. # Warn if this is suppressing a NaN (unusual, and # non-standard, but we will wait to remove this behavior # for the time being) diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 32f58dbfc13..b5e0a1a0348 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -298,7 +298,8 @@ def test_ANY_over_constant_division(self): m.x = Var() m.z = Var() m.y = Var() - # We aren't treating this as a Var, so we don't really care that it's fixed. + # We will use the fixed value regardless of the fact that we aren't + # treating this as a Var. m.y.fix(1) expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y @@ -308,10 +309,10 @@ def test_ANY_over_constant_division(self): ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.constant, m.y + m.z) + assertExpressionsEqual(self, repn.constant, 1 + m.z) self.assertEqual(len(repn.linear), 1) print(repn.linear[id(m.x)]) - assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z / m.y) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z) self.assertEqual(repn.nonlinear, None) def test_errors_propogate_nan(self): @@ -335,10 +336,10 @@ def test_errors_propogate_nan(self): "\texpression: 3*z*x/p\n", ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.constant, m.y + m.z) + assertExpressionsEqual(self, repn.constant, 1 + m.z) self.assertEqual(len(repn.linear), 1) self.assertIsInstance(repn.linear[id(m.x)], InvalidNumber) - assertExpressionsEqual(self, repn.linear[id(m.x)].value, 1 + float('nan') / m.y) + assertExpressionsEqual(self, repn.linear[id(m.x)].value, 1 + float('nan')) self.assertEqual(repn.nonlinear, None) m.y.fix(None) @@ -347,6 +348,8 @@ def test_errors_propogate_nan(self): expr ) self.assertEqual(repn.multiplier, 1) + # TODO: Is this expected to just wrap up into a single InvalidNumber? + print(repn.constant) self.assertIsInstance(repn.constant, InvalidNumber) assertExpressionsEqual(self, repn.constant.value, float('nan') * m.z + 3) self.assertEqual(repn.linear, {}) From b4101b7bf7eb88fde08133839aa080056deab853 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Sun, 12 May 2024 12:01:14 -0600 Subject: [PATCH 032/150] Extending the ExprType enum to include 'pseudo constant', bwahahaha --- pyomo/repn/parameterized_linear.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index b180ee91862..2163b544184 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -10,8 +10,10 @@ # ___________________________________________________________________________ import copy +import enum from pyomo.common.collections import ComponentSet +from pyomo.common.enums import ExtendedEnumType from pyomo.common.numeric_types import native_numeric_types from pyomo.core import Var from pyomo.core.expr.logical_expr import _flattened @@ -34,7 +36,12 @@ from pyomo.repn.util import ExprType from . import linear -_CONSTANT = ExprType.CONSTANT + +class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): + __base_enum__ = ExprType + PSUEDO_CONSTANT = 50 + +_CONSTANT = ParameterizedExprType.CONSTANT def _merge_dict(dest_dict, mult, src_dict): From c4c3de08b8db83a211b5395f54faf842243d4ac3 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 13 May 2024 14:26:11 -0600 Subject: [PATCH 033/150] starting to add pseudo-constant handlers but they don't work yet --- pyomo/repn/parameterized_linear.py | 95 ++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 2163b544184..29e6583d19d 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -19,9 +19,12 @@ from pyomo.core.expr.logical_expr import _flattened from pyomo.core.expr.numeric_expr import ( AbsExpression, + DivisionExpression, LinearExpression, MonomialTermExpression, + NegationExpression, mutable_expression, + PowExpression, ProductExpression, SumExpression, UnaryFunctionExpression, @@ -41,7 +44,10 @@ class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): __base_enum__ = ExprType PSUEDO_CONSTANT = 50 +_PSEUDO_CONSTANT = ParameterizedExprType.PSUEDO_CONSTANT _CONSTANT = ParameterizedExprType.CONSTANT +_LINEAR = ParameterizedExprType.LINEAR +_GENERAL = ParameterizedExprType.GENERAL def _merge_dict(dest_dict, mult, src_dict): @@ -177,7 +183,7 @@ def _before_var(visitor, child): if child in visitor.wrt: # psueudo-constant # We aren't treating this Var as a Var for the purposes of this walker - return False, (_CONSTANT, child) + return False, (_PSEUDO_CONSTANT, child) # This is a normal situation # TODO: override record var to not record things in wrt MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) @@ -189,14 +195,93 @@ def _before_var(visitor, child): _before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() _exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) +# +# NEGATION handler +# + +def _handle_negation_pseudo_constant(visitor, node, arg): + return (_PSEUDO_CONSTANT, -1 * arg[1]) -def _handle_product_constant_constant(visitor, node, arg1, arg2): - # ESJ: Can I do this? Just let the potential nans go through? - return _CONSTANT, arg1[1] * arg2[1] + +_exit_node_handlers[NegationExpression].update( + {(_PSEUDO_CONSTANT,): _handle_negation_pseudo_constant,} +) + + +# +# PRODUCT handler +# + + +def _handle_product_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, arg1[1] * arg2[1] + + +def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, arg1[1] * arg2[1] _exit_node_handlers[ProductExpression].update( - {(_CONSTANT, _CONSTANT): _handle_product_constant_constant} + { + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _CONSTANT): _handle_product_pseudo_constant_constant, + (_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, + (_PSEUDO_CONSTANT, _LINEAR): linear._handle_product_constant_ANY, + (_LINEAR, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, + (_PSEUDO_CONSTANT, _GENERAL): linear._handle_product_constant_ANY, + (_GENERAL, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, + } +) +_exit_node_handlers[MonomialTermExpression].update(_exit_node_handlers[ProductExpression]) + +# +# DIVISION handlers +# + +def _handle_division_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, arg1[1] / arg2[1] + + +def _handle_division_pseudo_constant_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, arg[1] / arg2[1] + + +def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): + arg1[1].multiplier = arg1[1].multiplier / arg2[1] + return arg1 + + +_exit_node_handlers[DivisionExpression].update( + { + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _CONSTANT): _handle_division_pseudo_constant_constant, + (_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, + (_LINEAR, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, + (_GENERAL, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, + } +) + +# +# EXPONENTIATION handlers +# + +def _handle_pow_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, node.create_node_with_local_data( + linear.to_expression(visitor, arg1), linear.to_expression(visitor, arg2)) + + +def _handle_pow_ANY_pseudo_constant(visitor, node, arg1, arg2): + # TODO + pass + + +_exit_node_handlers[PowExpression].update( + { + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, + (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, + (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_ANY_pseudo_constant, + } ) From 2a9abeae16467245d5fbc1bc2246488f6d25958c Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 10:29:43 -0600 Subject: [PATCH 034/150] Full draft of new exit node handlers for psuedo constant expressions, all tests passing --- pyomo/repn/parameterized_linear.py | 83 ++++++++++--------- pyomo/repn/tests/test_parameterized_linear.py | 8 +- 2 files changed, 45 insertions(+), 46 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 29e6583d19d..9556cae1cf7 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -44,6 +44,7 @@ class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): __base_enum__ = ExprType PSUEDO_CONSTANT = 50 + _PSEUDO_CONSTANT = ParameterizedExprType.PSUEDO_CONSTANT _CONSTANT = ParameterizedExprType.CONSTANT _LINEAR = ParameterizedExprType.LINEAR @@ -65,6 +66,13 @@ def _merge_dict(dest_dict, mult, src_dict): dest_dict[vid] = coef +def to_expression(visitor, arg): + if arg[0] in (_CONSTANT, _PSEUDO_CONSTANT): + return arg[1] + else: + return arg[1].to_expression(visitor) + + class ParameterizedLinearRepn(LinearRepn): def to_expression(self, visitor): if self.nonlinear is not None: @@ -107,7 +115,7 @@ def append(self, other): """ _type, other = other - if _type is _CONSTANT: + if _type in (_CONSTANT, _PSEUDO_CONSTANT): self.constant += other return @@ -176,10 +184,7 @@ def _before_var(visitor, child): _id = id(child) if _id not in visitor.var_map: if child.fixed: - return False, ( - _CONSTANT, - visitor.check_constant(child.value, child), - ) + return False, (_CONSTANT, visitor.check_constant(child.value, child)) if child in visitor.wrt: # psueudo-constant # We aren't treating this Var as a Var for the purposes of this walker @@ -199,12 +204,13 @@ def _before_var(visitor, child): # NEGATION handler # + def _handle_negation_pseudo_constant(visitor, node, arg): return (_PSEUDO_CONSTANT, -1 * arg[1]) _exit_node_handlers[NegationExpression].update( - {(_PSEUDO_CONSTANT,): _handle_negation_pseudo_constant,} + {(_PSEUDO_CONSTANT,): _handle_negation_pseudo_constant} ) @@ -213,17 +219,13 @@ def _handle_negation_pseudo_constant(visitor, node, arg): # -def _handle_product_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg1[1] * arg2[1] - - def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): return _PSEUDO_CONSTANT, arg1[1] * arg2[1] _exit_node_handlers[ProductExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, (_PSEUDO_CONSTANT, _CONSTANT): _handle_product_pseudo_constant_constant, (_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, (_PSEUDO_CONSTANT, _LINEAR): linear._handle_product_constant_ANY, @@ -232,18 +234,17 @@ def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): (_GENERAL, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, } ) -_exit_node_handlers[MonomialTermExpression].update(_exit_node_handlers[ProductExpression]) +_exit_node_handlers[MonomialTermExpression].update( + _exit_node_handlers[ProductExpression] +) # # DIVISION handlers # -def _handle_division_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg1[1] / arg2[1] - def _handle_division_pseudo_constant_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg[1] / arg2[1] + return _PSEUDO_CONSTANT, arg1[1] / arg2[1] def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): @@ -253,7 +254,7 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): _exit_node_handlers[DivisionExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, (_PSEUDO_CONSTANT, _CONSTANT): _handle_division_pseudo_constant_constant, (_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, (_LINEAR, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, @@ -265,42 +266,44 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): # EXPONENTIATION handlers # -def _handle_pow_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, node.create_node_with_local_data( - linear.to_expression(visitor, arg1), linear.to_expression(visitor, arg2)) + +def _handle_pow_pseudo_constant_constant(visitor, node, arg1, arg2): + print("creating node") + print(to_expression(visitor, arg1)) + print(to_expression(visitor, arg2)) + return _PSEUDO_CONSTANT, to_expression(visitor, arg1) ** to_expression( + visitor, arg2 + ) -def _handle_pow_ANY_pseudo_constant(visitor, node, arg1, arg2): - # TODO - pass +def _handle_pow_ANY_psuedo_constant(visitor, node, arg1, arg2): + return linear._handle_pow_nonlinear(visitor, node, arg1, arg2) _exit_node_handlers[PowExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, - (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, - (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, - (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_ANY_pseudo_constant, + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, + (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_constant, + (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, + (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, + (_GENERAL, _PSEUDO_CONSTANT): _handle_pow_ANY_psuedo_constant, } ) +# +# ABS and UNARY handlers +# + -def _handle_unary_constant(visitor, node, arg): +def _handle_unary_pseudo_constant(visitor, node, arg): # We override this because we can't blindly use apply_node_operation in this case - if arg.__class__ not in native_numeric_types: - return _CONSTANT, node.create_node_with_local_data( - (linear.to_expression(visitor, arg),) - ) - # otherwise do the usual: - ans = apply_node_operation(node, (arg[1],)) - # Unary includes sqrt() which can return complex numbers - if ans.__class__ in native_complex_types: - ans = complex_number_error(ans, visitor, node) - return _CONSTANT, ans + return _PSEUDO_CONSTANT, node.create_node_with_local_data( + (to_expression(visitor, arg),) + ) _exit_node_handlers[UnaryFunctionExpression].update( - {(_CONSTANT,): _handle_unary_constant} + {(_PSEUDO_CONSTANT,): _handle_unary_pseudo_constant} ) _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] @@ -376,6 +379,6 @@ def finalizeResult(self, result): return ans ans = self.Result() - assert result[0] is _CONSTANT + assert result[0] in (_CONSTANT, _PSEUDO_CONSTANT) ans.constant = result[1] return ans diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index b5e0a1a0348..068afe16929 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -124,9 +124,7 @@ def test_sum_nonlinear_to_nonlinear(self): assertExpressionsEqual(self, repn.nonlinear, m.x**3 + m.x**2) self.assertEqual(repn.constant, 3) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual( - self, repn.to_expression(visitor), m.x**3 + m.x**2 + 3 - ) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x**3 + m.x**2 + 3) def test_sum_to_linear_expr(self): m = self.make_model() @@ -288,9 +286,7 @@ def test_finalize(self): self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.y), repn.linear) assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * m.w) - assertExpressionsEqual( - self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5 - ) + assertExpressionsEqual(self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5) def test_ANY_over_constant_division(self): m = ConcreteModel() From 0d99834c446045efbf58a615cb2c4ae81f23f06a Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 13:59:38 -0600 Subject: [PATCH 035/150] Fixing a bug where psuedo constants were getting labeled as constants in walker_exitNode --- pyomo/repn/parameterized_linear.py | 27 +++-- pyomo/repn/tests/test_parameterized_linear.py | 105 +++++++++++++++++- 2 files changed, 120 insertions(+), 12 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 9556cae1cf7..67fb4a7421e 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -74,6 +74,16 @@ def to_expression(visitor, arg): class ParameterizedLinearRepn(LinearRepn): + def walker_exitNode(self): + if self.nonlinear is not None: + return _GENERAL, self + elif self.linear: + return _LINEAR, self + elif self.constant.__class__ in native_numeric_types: + return _CONSTANT, self.multiplier * self.constant + else: + return _PSEUDO_CONSTANT, self.multiplier * self.constant + def to_expression(self, visitor): if self.nonlinear is not None: # We want to start with the nonlinear term (and use @@ -268,16 +278,17 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): def _handle_pow_pseudo_constant_constant(visitor, node, arg1, arg2): - print("creating node") - print(to_expression(visitor, arg1)) - print(to_expression(visitor, arg2)) return _PSEUDO_CONSTANT, to_expression(visitor, arg1) ** to_expression( visitor, arg2 ) -def _handle_pow_ANY_psuedo_constant(visitor, node, arg1, arg2): - return linear._handle_pow_nonlinear(visitor, node, arg1, arg2) +def _handle_pow_nonlinear(visitor, node, arg1, arg2): + # ESJ: We override this because we need our own to_expression implementation + # if pseudo constants are involved. + ans = visitor.Result() + ans.nonlinear = to_expression(visitor, arg1) ** to_expression(visitor, arg2) + return _GENERAL, ans _exit_node_handlers[PowExpression].update( @@ -285,8 +296,10 @@ def _handle_pow_ANY_psuedo_constant(visitor, node, arg1, arg2): (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_constant, (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, - (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, - (_GENERAL, _PSEUDO_CONSTANT): _handle_pow_ANY_psuedo_constant, + (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_nonlinear, + (_PSEUDO_CONSTANT, _LINEAR): _handle_pow_nonlinear, + (_GENERAL, _PSEUDO_CONSTANT): _handle_pow_nonlinear, + (_PSEUDO_CONSTANT, _GENERAL): _handle_pow_nonlinear, } ) diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 068afe16929..de4de301a83 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -268,7 +268,6 @@ def test_finalize(self): self.assertEqual(repn.constant, 0) self.assertEqual(len(repn.linear), 2) self.assertIn(id(m.y), repn.linear) - print(repn.linear[id(m.y)]) assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * (2 * m.w**2)) self.assertIn(id(m.z), repn.linear) assertExpressionsEqual(self, repn.linear[id(m.z)], -5 * m.w) @@ -307,7 +306,6 @@ def test_ANY_over_constant_division(self): self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.constant, 1 + m.z) self.assertEqual(len(repn.linear), 1) - print(repn.linear[id(m.x)]) assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z) self.assertEqual(repn.nonlinear, None) @@ -344,8 +342,6 @@ def test_errors_propogate_nan(self): expr ) self.assertEqual(repn.multiplier, 1) - # TODO: Is this expected to just wrap up into a single InvalidNumber? - print(repn.constant) self.assertIsInstance(repn.constant, InvalidNumber) assertExpressionsEqual(self, repn.constant.value, float('nan') * m.z + 3) self.assertEqual(repn.linear, {}) @@ -372,7 +368,106 @@ def test_product_nonlinear(self): self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 0) - print(repn.nonlinear) assertExpressionsEqual( self, repn.nonlinear, (m.x**2) * (m.z**4 * log(m.y)) * m.y ) + + def test_division_pseudo_constant_constant(self): + m = self.make_model() + e = m.x / 4 + m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 1) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, m.x / 4) + self.assertIsNone(repn.nonlinear) + + e = 4 / m.x + m.y + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 1) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, 4 / m.x) + self.assertIsNone(repn.nonlinear) + + e = m.z / m.x + m.y + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x, m.z]).walk_expression(e) + + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 1) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, m.z / m.x) + self.assertIsNone(repn.nonlinear) + + def test_division_ANY_psuedo_constant(self): + m = self.make_model() + e = (m.x + 3 * m.z) / m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.x), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 / m.y) + self.assertIn(id(m.z), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.z)], (1 / m.y) * 3) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertIsNone(repn.nonlinear) + + def test_pow_ANY_psuedo_constant(self): + m = self.make_model() + e = (m.x**2 + 3 * m.z) ** m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, (m.x**2 + 3 * m.z) ** m.y) + + def test_pow_psuedo_constant_ANY(self): + m = self.make_model() + e = m.y ** (m.x**2 + 3 * m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, m.y ** (m.x**2 + 3 * m.z)) + + def test_pow_linear_pseudo_constant(self): + m = self.make_model() + e = (m.x + 3 * m.z) ** m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, (m.x + 3 * m.z) ** m.y) + + def test_pow_pseudo_constant_linear(self): + m = self.make_model() + e = m.y ** (m.x + 3 * m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, m.y ** (m.x + 3 * m.z)) From f13d3ab5905f9bba4c67af6b9ea6cd9a68b38d52 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 16:17:53 -0600 Subject: [PATCH 036/150] Making the new walker compliant with IEEE 754 when multiplying 0 and nan --- pyomo/repn/parameterized_linear.py | 15 ++---- pyomo/repn/tests/test_parameterized_linear.py | 53 +++++++++++++++++++ 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 67fb4a7421e..6748c406bb7 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -200,7 +200,6 @@ def _before_var(visitor, child): # We aren't treating this Var as a Var for the purposes of this walker return False, (_PSEUDO_CONSTANT, child) # This is a normal situation - # TODO: override record var to not record things in wrt MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) ans = visitor.Result() ans.linear[_id] = 1 @@ -228,6 +227,8 @@ def _handle_negation_pseudo_constant(visitor, node, arg): # PRODUCT handler # +def _handle_product_constant_constant(visitor, node, arg1, arg2): + return _CONSTANT, arg1[1] * arg2[1] def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): return _PSEUDO_CONSTANT, arg1[1] * arg2[1] @@ -235,6 +236,7 @@ def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): _exit_node_handlers[ProductExpression].update( { + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, (_PSEUDO_CONSTANT, _CONSTANT): _handle_product_pseudo_constant_constant, (_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, @@ -372,18 +374,11 @@ def finalizeResult(self, result): # Warn if this is suppressing a NaN (unusual, and # non-standard, but we will wait to remove this behavior # for the time being) - # ESJ TODO: This won't work either actually... - # I'm not sure how to do it. if ans.constant != ans.constant or any( c != c for c in ans.linear.values() ): - deprecation_warning( - f"Encountered {str(mult)}*nan in expression tree. " - "Mapping the NaN result to 0 for compatibility " - "with the lp_v1 writer. In the future, this NaN " - "will be preserved/emitted to comply with IEEE-754.", - version='6.6.0', - ) + # There's a nan in here, so we keep it + self._factor_multiplier_into_linear_terms(ans, mult) return self.Result() else: # mult not in {0, 1}: factor it into the constant, diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index de4de301a83..2148e3053f4 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -471,3 +471,56 @@ def test_pow_pseudo_constant_linear(self): self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 0) assertExpressionsEqual(self, repn.nonlinear, m.y ** (m.x + 3 * m.z)) + + def test_0_mult(self): + m = self.make_model() + m.p = Var() + m.p.fix(0) + e = m.p * (m.y ** 2 + m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertEqual(repn.constant, 0) + + def test_0_mult_nan(self): + m = self.make_model() + m.p = Param(initialize=0, mutable=True) + m.y.domain = Any + m.y.fix(float('nan')) + e = m.p * (m.y ** 2 + m.x) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertIsInstance(repn.constant, InvalidNumber) + assertExpressionsEqual( + self, + repn.constant.value, + 0 * (float('nan') + m.x) + ) + + def test_0_mult_nan_param(self): + m = self.make_model() + m.p = Param(initialize=0, mutable=True) + m.y.fix(float('nan')) + e = m.p * (m.y ** 2) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertIsInstance(repn.constant, InvalidNumber) + assertExpressionsEqual( + self, + repn.constant.value, + 0 * float('nan') + ) From 42ff2932f5c88ad5025971593ced3e48345240b7 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 16:18:21 -0600 Subject: [PATCH 037/150] Black --- pyomo/repn/parameterized_linear.py | 2 ++ pyomo/repn/tests/test_parameterized_linear.py | 20 ++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 6748c406bb7..5928d1f18d9 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -227,9 +227,11 @@ def _handle_negation_pseudo_constant(visitor, node, arg): # PRODUCT handler # + def _handle_product_constant_constant(visitor, node, arg1, arg2): return _CONSTANT, arg1[1] * arg2[1] + def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): return _PSEUDO_CONSTANT, arg1[1] * arg2[1] diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 2148e3053f4..1b0ab630462 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -476,7 +476,7 @@ def test_0_mult(self): m = self.make_model() m.p = Var() m.p.fix(0) - e = m.p * (m.y ** 2 + m.z) + e = m.p * (m.y**2 + m.z) cfg = VisitorConfig() repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]).walk_expression(e) @@ -491,7 +491,7 @@ def test_0_mult_nan(self): m.p = Param(initialize=0, mutable=True) m.y.domain = Any m.y.fix(float('nan')) - e = m.p * (m.y ** 2 + m.x) + e = m.p * (m.y**2 + m.x) cfg = VisitorConfig() repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) @@ -500,17 +500,13 @@ def test_0_mult_nan(self): self.assertEqual(repn.multiplier, 1) self.assertIsNone(repn.nonlinear) self.assertIsInstance(repn.constant, InvalidNumber) - assertExpressionsEqual( - self, - repn.constant.value, - 0 * (float('nan') + m.x) - ) - + assertExpressionsEqual(self, repn.constant.value, 0 * (float('nan') + m.x)) + def test_0_mult_nan_param(self): m = self.make_model() m.p = Param(initialize=0, mutable=True) m.y.fix(float('nan')) - e = m.p * (m.y ** 2) + e = m.p * (m.y**2) cfg = VisitorConfig() repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) @@ -519,8 +515,4 @@ def test_0_mult_nan_param(self): self.assertEqual(repn.multiplier, 1) self.assertIsNone(repn.nonlinear) self.assertIsInstance(repn.constant, InvalidNumber) - assertExpressionsEqual( - self, - repn.constant.value, - 0 * float('nan') - ) + assertExpressionsEqual(self, repn.constant.value, 0 * float('nan')) From fb78c898b9f1c8fe1aa3e45fa1bb43b923fbe518 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 16:21:49 -0600 Subject: [PATCH 038/150] Fixing some typos --- pyomo/repn/parameterized_linear.py | 6 +++--- pyomo/repn/tests/test_parameterized_linear.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 5928d1f18d9..8200b4cf01e 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -42,10 +42,10 @@ class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): __base_enum__ = ExprType - PSUEDO_CONSTANT = 50 + PSEUDO_CONSTANT = 50 -_PSEUDO_CONSTANT = ParameterizedExprType.PSUEDO_CONSTANT +_PSEUDO_CONSTANT = ParameterizedExprType.PSEUDO_CONSTANT _CONSTANT = ParameterizedExprType.CONSTANT _LINEAR = ParameterizedExprType.LINEAR _GENERAL = ParameterizedExprType.GENERAL @@ -196,7 +196,7 @@ def _before_var(visitor, child): if child.fixed: return False, (_CONSTANT, visitor.check_constant(child.value, child)) if child in visitor.wrt: - # psueudo-constant + # pseudo-constant # We aren't treating this Var as a Var for the purposes of this walker return False, (_PSEUDO_CONSTANT, child) # This is a normal situation diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 1b0ab630462..7d99acf8bb8 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -309,7 +309,7 @@ def test_ANY_over_constant_division(self): assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z) self.assertEqual(repn.nonlinear, None) - def test_errors_propogate_nan(self): + def test_errors_propagate_nan(self): m = ConcreteModel() m.p = Param(mutable=True, initialize=0, domain=Any) m.x = Var() From 95fa5245de86ebef4ad45f0ebff122a28ebf7f2e Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 09:21:07 -0600 Subject: [PATCH 039/150] Fixing more typos --- pyomo/repn/tests/test_parameterized_linear.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 7d99acf8bb8..4e92c5f11f2 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -408,7 +408,7 @@ def test_division_pseudo_constant_constant(self): assertExpressionsEqual(self, repn.constant, m.z / m.x) self.assertIsNone(repn.nonlinear) - def test_division_ANY_psuedo_constant(self): + def test_division_ANY_pseudo_constant(self): m = self.make_model() e = (m.x + 3 * m.z) / m.y @@ -424,7 +424,7 @@ def test_division_ANY_psuedo_constant(self): self.assertEqual(repn.constant, 0) self.assertIsNone(repn.nonlinear) - def test_pow_ANY_psuedo_constant(self): + def test_pow_ANY_pseudo_constant(self): m = self.make_model() e = (m.x**2 + 3 * m.z) ** m.y @@ -436,7 +436,7 @@ def test_pow_ANY_psuedo_constant(self): self.assertEqual(repn.constant, 0) assertExpressionsEqual(self, repn.nonlinear, (m.x**2 + 3 * m.z) ** m.y) - def test_pow_psuedo_constant_ANY(self): + def test_pow_pseudo_constant_ANY(self): m = self.make_model() e = m.y ** (m.x**2 + 3 * m.z) From 794a3bf2ccaa5561c8def704b6c7b1111c0746e4 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 10:59:12 -0600 Subject: [PATCH 040/150] Distributing 0 if there are nans present during finalizeResult --- pyomo/repn/parameterized_linear.py | 10 +++++----- pyomo/repn/tests/test_parameterized_linear.py | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 8200b4cf01e..c1647e44732 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -372,15 +372,15 @@ def finalizeResult(self, result): for vid, coef in zeros: del ans.linear[vid] elif not mult: - # the multiplier has cleared out the entire expression. - # Warn if this is suppressing a NaN (unusual, and - # non-standard, but we will wait to remove this behavior - # for the time being) + # the multiplier has cleared out the entire expression. Check + # if this is suppressing a NaN because we can't clear everything + # out if it is if ans.constant != ans.constant or any( c != c for c in ans.linear.values() ): - # There's a nan in here, so we keep it + # There's a nan in here, so we distribute the 0 self._factor_multiplier_into_linear_terms(ans, mult) + return ans return self.Result() else: # mult not in {0, 1}: factor it into the constant, diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 4e92c5f11f2..fd2f2aaec68 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -516,3 +516,23 @@ def test_0_mult_nan_param(self): self.assertIsNone(repn.nonlinear) self.assertIsInstance(repn.constant, InvalidNumber) assertExpressionsEqual(self, repn.constant.value, 0 * float('nan')) + + def test_0_mult_linear_with_nan(self): + m = self.make_model() + m.p = Param(initialize=0, mutable=True) + m.x.domain = Any + m.x.fix(float('nan')) + e = m.p * (3 * m.x * m.y + m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.y), repn.linear) + self.assertIsInstance(repn.linear[id(m.y)], InvalidNumber) + assertExpressionsEqual(self, repn.linear[id(m.y)].value, 0 * 3 * float('nan')) + self.assertIn(id(m.z), repn.linear) + self.assertEqual(repn.linear[id(m.z)], 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertEqual(repn.constant, 0) From 2c4a1cbd9cc309e1925a484444a468e09fd4fed6 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 11:23:03 -0600 Subject: [PATCH 041/150] Testing duplicate in ParameterizedLinearRepn, changing the string representation to not be the same as LinearRepn --- pyomo/repn/parameterized_linear.py | 6 ++++++ pyomo/repn/tests/test_parameterized_linear.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index c1647e44732..eb6dd619168 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -74,6 +74,12 @@ def to_expression(visitor, arg): class ParameterizedLinearRepn(LinearRepn): + def __str__(self): + return ( + f"ParameterizedLinearRepn(mult={self.multiplier}, const={self.constant}, " + f"linear={self.linear}, nonlinear={self.nonlinear})" + ) + def walker_exitNode(self): if self.nonlinear is not None: return _GENERAL, self diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index fd2f2aaec68..624f8390d16 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -424,6 +424,20 @@ def test_division_ANY_pseudo_constant(self): self.assertEqual(repn.constant, 0) self.assertIsNone(repn.nonlinear) + def test_duplicate(self): + m = self.make_model() + e = (1 + m.x) ** 2 + m.y + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]) + visitor.max_exponential_expansion = 2 + repn = visitor.walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIs(repn.constant, m.y) + assertExpressionsEqual(self, repn.nonlinear, (m.x + 1) * (m.x + 1)) + def test_pow_ANY_pseudo_constant(self): m = self.make_model() e = (m.x**2 + 3 * m.z) ** m.y From 7d4270108a46eca83d288d1d78a4593f4a8f1ad4 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 11:28:56 -0600 Subject: [PATCH 042/150] NFC: fixing some comment typos --- pyomo/repn/parameterized_linear.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index eb6dd619168..f976523a37f 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -216,7 +216,7 @@ def _before_var(visitor, child): _exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) # -# NEGATION handler +# NEGATION handlers # @@ -230,7 +230,7 @@ def _handle_negation_pseudo_constant(visitor, node, arg): # -# PRODUCT handler +# PRODUCT handlers # From 71d8801f277d7b40510aeff23e0f46c3839121bd Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 11:40:18 -0600 Subject: [PATCH 043/150] Whoops, one last class name change I missed --- pyomo/repn/parameterized_linear.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index f976523a37f..0633c285155 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -175,7 +175,7 @@ def append(self, other): self.nonlinear += nl -class MultiLevelLinearBeforeChildDispatcher(LinearBeforeChildDispatcher): +class ParameterizedLinearBeforeChildDispatcher(LinearBeforeChildDispatcher): def __init__(self): super().__init__() self[Var] = self._before_var @@ -206,13 +206,13 @@ def _before_var(visitor, child): # We aren't treating this Var as a Var for the purposes of this walker return False, (_PSEUDO_CONSTANT, child) # This is a normal situation - MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) + ParameterizedLinearBeforeChildDispatcher._record_var(visitor, child) ans = visitor.Result() ans.linear[_id] = 1 return False, (ExprType.LINEAR, ans) -_before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() +_before_child_dispatcher = ParameterizedLinearBeforeChildDispatcher() _exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) # From 91f0aaa7fe7e8e14940fddd8a037981b2245d7cc Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 22 May 2024 12:54:54 -0600 Subject: [PATCH 044/150] Addressing John's comments --- pyomo/repn/parameterized_linear.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 0633c285155..892edd5643d 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -37,12 +37,12 @@ LinearRepnVisitor, ) from pyomo.repn.util import ExprType -from . import linear +import pyomo.repn.linear as linear class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): __base_enum__ = ExprType - PSEUDO_CONSTANT = 50 + PSEUDO_CONSTANT = 5 _PSEUDO_CONSTANT = ParameterizedExprType.PSEUDO_CONSTANT @@ -123,7 +123,7 @@ def append(self, other): Notes ----- This method assumes that the operator was "+". It is implemented - so that we can directly use a LinearRepn() as a `data` object in + so that we can directly use a ParameterizedLinearRepn() as a `data` object in the expression walker (thereby allowing us to use the default implementation of acceptChildResult [which calls `data.append()`] and avoid the function call for a custom @@ -131,7 +131,7 @@ def append(self, other): """ _type, other = other - if _type in (_CONSTANT, _PSEUDO_CONSTANT): + if _type is _CONSTANT or type is _PSEUDO_CONSTANT: self.constant += other return @@ -235,6 +235,8 @@ def _handle_negation_pseudo_constant(visitor, node, arg): def _handle_product_constant_constant(visitor, node, arg1, arg2): + # [ESJ 5/22/24]: Overriding this handler to exclude the deprecation path for + # 0 * nan. It doesn't need overridden when that deprecation path goes away. return _CONSTANT, arg1[1] * arg2[1] From 5aebb02fce467f10bfc4cacc653e8dbce3894cfa Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 22 May 2024 13:46:29 -0600 Subject: [PATCH 045/150] Whoops, bad typo --- pyomo/repn/parameterized_linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 892edd5643d..ae0856cfe76 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -131,7 +131,7 @@ def append(self, other): """ _type, other = other - if _type is _CONSTANT or type is _PSEUDO_CONSTANT: + if _type is _CONSTANT or _type is _PSEUDO_CONSTANT: self.constant += other return From 35edf83785e4db3071080903b92ccd6c809c32bd Mon Sep 17 00:00:00 2001 From: Eslick Date: Thu, 23 May 2024 14:33:53 -0400 Subject: [PATCH 046/150] Change the way models in main are handeled --- pyomo/contrib/viewer/model_select.py | 14 ++++++----- pyomo/contrib/viewer/pyomo_viewer.py | 2 +- pyomo/contrib/viewer/tests/test_qt.py | 16 ++++++------ pyomo/contrib/viewer/ui.py | 35 ++++++++++++++++++++++++--- pyomo/contrib/viewer/ui_data.py | 13 +++++++++- 5 files changed, 61 insertions(+), 19 deletions(-) diff --git a/pyomo/contrib/viewer/model_select.py b/pyomo/contrib/viewer/model_select.py index e9c82740708..1e65e91a089 100644 --- a/pyomo/contrib/viewer/model_select.py +++ b/pyomo/contrib/viewer/model_select.py @@ -60,31 +60,33 @@ def select_model(self): items = self.tableWidget.selectedItems() if len(items) == 0: return - self.ui_data.model = self.models[items[0].row()] + self.ui_data.model_var_name_in_main = self.models[items[0].row()][1] + self.ui_data.model = self.models[items[0].row()][0] self.close() def update_models(self): import __main__ - s = __main__.__dict__ + s = dir(__main__) keys = [] for k in s: - if isinstance(s[k], pyo.Block): + if isinstance(getattr(__main__, k), pyo.Block): keys.append(k) self.tableWidget.clearContents() self.tableWidget.setRowCount(len(keys)) self.models = [] for row, k in enumerate(sorted(keys)): + model = getattr(__main__, k) item = myqt.QTableWidgetItem() item.setText(k) self.tableWidget.setItem(row, 0, item) item = myqt.QTableWidgetItem() try: - item.setText(s[k].name) + item.setText(model.name) except: item.setText("None") self.tableWidget.setItem(row, 1, item) item = myqt.QTableWidgetItem() - item.setText(str(type(s[k]))) + item.setText(str(type(model))) self.tableWidget.setItem(row, 2, item) - self.models.append(s[k]) + self.models.append((model, k)) diff --git a/pyomo/contrib/viewer/pyomo_viewer.py b/pyomo/contrib/viewer/pyomo_viewer.py index 6a24e12aa61..e4f75c86840 100644 --- a/pyomo/contrib/viewer/pyomo_viewer.py +++ b/pyomo/contrib/viewer/pyomo_viewer.py @@ -41,7 +41,7 @@ class QtApp( model except NameError: model=None - ui, model = get_mainwindow(model=model, ask_close=False) + ui = get_mainwindow(model=model, ask_close=False) ui.setWindowTitle('Pyomo Model Viewer -- {}')""" _kernel_cmd_hide_ui = """try: diff --git a/pyomo/contrib/viewer/tests/test_qt.py b/pyomo/contrib/viewer/tests/test_qt.py index e71921500f9..b7250729cd9 100644 --- a/pyomo/contrib/viewer/tests/test_qt.py +++ b/pyomo/contrib/viewer/tests/test_qt.py @@ -103,7 +103,7 @@ def blackbox(a, b): @unittest.skipIf(not available, "Qt packages are not available.") def test_get_mainwindow(qtbot): m = get_model() - mw, m = get_mainwindow(model=m, testing=True) + mw = get_mainwindow(model=m, testing=True) assert hasattr(mw, "menuBar") assert isinstance(mw.variables, ModelBrowser) assert isinstance(mw.constraints, ModelBrowser) @@ -113,13 +113,13 @@ def test_get_mainwindow(qtbot): @unittest.skipIf(not available, "Qt packages are not available.") def test_close_mainwindow(qtbot): - mw, m = get_mainwindow(model=None, testing=True) + mw = get_mainwindow(model=None, testing=True) mw.exit_action() @unittest.skipIf(not available, "Qt packages are not available.") def test_show_model_select_no_models(qtbot): - mw, m = get_mainwindow(model=None, testing=True) + mw = get_mainwindow(model=None, testing=True) ms = mw.show_model_select() ms.update_models() ms.select_model() @@ -128,7 +128,7 @@ def test_show_model_select_no_models(qtbot): @unittest.skipIf(not available, "Qt packages are not available.") def test_model_information(qtbot): m = get_model() - mw, m = get_mainwindow(model=m, testing=True) + mw = get_mainwindow(model=m, testing=True) mw.model_information() assert isinstance(mw._dialog, QMessageBox) text = mw._dialog.text() @@ -149,7 +149,7 @@ def test_model_information(qtbot): @unittest.skipIf(not available, "Qt packages are not available.") def test_tree_expand_collapse(qtbot): m = get_model() - mw, m = get_mainwindow(model=m, testing=True) + mw = get_mainwindow(model=m, testing=True) mw.variables.treeView.expandAll() mw.variables.treeView.collapseAll() @@ -157,7 +157,7 @@ def test_tree_expand_collapse(qtbot): @unittest.skipIf(not available, "Qt packages are not available.") def test_residual_table(qtbot): m = get_model() - mw, m = get_mainwindow(model=m, testing=True) + mw = get_mainwindow(model=m, testing=True) mw.residuals_restart() mw.ui_data.calculate_expressions() mw.residuals.calculate() @@ -184,7 +184,7 @@ def test_residual_table(qtbot): @unittest.skipIf(not available, "Qt packages are not available.") def test_var_tree(qtbot): m = get_model() - mw, m = get_mainwindow(model=m, testing=True) + mw = get_mainwindow(model=m, testing=True) qtbot.addWidget(mw) mw.variables.treeView.expandAll() root_index = mw.variables.datmodel.index(0, 0) @@ -218,7 +218,7 @@ def test_var_tree(qtbot): @unittest.skipIf(not available, "Qt packages are not available.") def test_bad_view(qtbot): m = get_model() - mw, m = get_mainwindow(model=m, testing=True) + mw = get_mainwindow(model=m, testing=True) err = None try: mw.badTree = mw._tree_restart( diff --git a/pyomo/contrib/viewer/ui.py b/pyomo/contrib/viewer/ui.py index 374af8a26f0..75b786b9796 100644 --- a/pyomo/contrib/viewer/ui.py +++ b/pyomo/contrib/viewer/ui.py @@ -79,16 +79,31 @@ def get_mainwindow(model=None, show=True, ask_close=True, testing=False): (ui, model): ui is the MainWindow widget, and model is the linked Pyomo model. If no model is provided a new ConcreteModel is created """ + model_name = None if model is None: - model = pyo.ConcreteModel(name="Default") - ui = MainWindow(model=model, ask_close=ask_close, testing=testing) + import __main__ + if "model" in dir(__main__): + if isinstance(getattr(__main__, "model"), pyo.Block): + model = getattr(__main__, "model") + model_name = "model" + for s in dir(__main__): + if isinstance(getattr(__main__, s), pyo.Block): + model = getattr(__main__, s) + model_name = s + break + ui = MainWindow( + model=model, + model_var_name_in_main=model_name, + ask_close=ask_close, + testing=testing, + ) try: get_ipython().events.register("post_execute", ui.refresh_on_execute) except AttributeError: pass # not in ipy kernel, so is fine to not register callback if show: ui.show() - return ui, model + return ui class MainWindow(_MainWindow, _MainWindowUI): @@ -97,6 +112,7 @@ def __init__(self, *args, **kwargs): main = self.main = kwargs.pop("main", None) ask_close = self.ask_close = kwargs.pop("ask_close", True) self.testing = kwargs.pop("testing", False) + model_var_name_in_main = kwargs.pop("model_var_name_in_main", None) flags = kwargs.pop("flags", 0) self.ui_data = UIData(model=model) super().__init__(*args, **kwargs) @@ -128,6 +144,7 @@ def __init__(self, *args, **kwargs): self.actionCalculateExpressions.triggered.connect( self.ui_data.calculate_expressions ) + self.ui_data.model_var_name_in_main = model_var_name_in_main self.actionTile.triggered.connect(self.mdiArea.tileSubWindows) self.actionCascade.triggered.connect(self.mdiArea.cascadeSubWindows) self.actionTabs.triggered.connect(self.toggle_tabs) @@ -256,6 +273,18 @@ def refresh_on_execute(self): ipython kernel. The main purpose of this right now it to refresh the UI display so that it matches the current state of the model. """ + if self.ui_data.model_var_name_in_main is not None: + import __main__ + + try: + mname = self.ui_data.model_var_name_in_main + mid = id(getattr(__main__, mname)) + if id(self.ui_data.model) != mid: + self.ui_data.model = getattr(__main__, mname) + self.update_model + return + except AttributeError: + pass for w in self._refresh_list: try: w.refresh() diff --git a/pyomo/contrib/viewer/ui_data.py b/pyomo/contrib/viewer/ui_data.py index c716cfeedf6..8d83be91e5f 100644 --- a/pyomo/contrib/viewer/ui_data.py +++ b/pyomo/contrib/viewer/ui_data.py @@ -39,16 +39,27 @@ class UIDataNoUi(object): UIData. The class is split this way for testing when PyQt is not available. """ - def __init__(self, model=None): + def __init__(self, model=None, model_var_name_in_main=None): """ This class holds the basic UI setup, but doesn't depend on Qt. It shouldn't really be used except for testing when Qt is not available. Args: model: The Pyomo model to view + model_var_name_in_main: if this is set, check that the model variable + which points to a model object in __main__ has the same id when + the UI is refreshed due to a command being executed in jupyter + notebook or QtConsole, if not the same id, then update the model + Since the model viewer is not necessarily pointed at a model in the + __main__ namespace only set this if you want the model to auto + update. Since the model selector dialog lets you choose models + from the __main__ namespace it sets this when you select a model. + This is useful if you run a script repeatedly that replaces a model + preventing you from looking at a previous version of the model. """ super().__init__() self._model = None + self.model_var_name_in_main = model_var_name_in_main self._begin_update = False self.value_cache = ComponentMap() self.value_cache_units = ComponentMap() From 01b5ef233935a1b7f958b49b8c13dacd35c21fde Mon Sep 17 00:00:00 2001 From: Eslick Date: Thu, 23 May 2024 14:58:31 -0400 Subject: [PATCH 047/150] add arg --- pyomo/contrib/viewer/ui.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pyomo/contrib/viewer/ui.py b/pyomo/contrib/viewer/ui.py index 75b786b9796..ac96e58eea9 100644 --- a/pyomo/contrib/viewer/ui.py +++ b/pyomo/contrib/viewer/ui.py @@ -66,7 +66,9 @@ class _MainWindow(object): _log.error(_err) -def get_mainwindow(model=None, show=True, ask_close=True, testing=False): +def get_mainwindow( + model=None, show=True, ask_close=True, model_var_name_in_main=None, testing=False +): """ Create a UI MainWindow. @@ -79,18 +81,19 @@ def get_mainwindow(model=None, show=True, ask_close=True, testing=False): (ui, model): ui is the MainWindow widget, and model is the linked Pyomo model. If no model is provided a new ConcreteModel is created """ - model_name = None + model_name = model_var_name_in_main if model is None: import __main__ - if "model" in dir(__main__): - if isinstance(getattr(__main__, "model"), pyo.Block): - model = getattr(__main__, "model") - model_name = "model" - for s in dir(__main__): - if isinstance(getattr(__main__, s), pyo.Block): - model = getattr(__main__, s) - model_name = s - break + + if model_name in dir(__main__): + if isinstance(getattr(__main__, model_name), pyo.Block): + model = getattr(__main__, model_name) + else: + for s in dir(__main__): + if isinstance(getattr(__main__, s), pyo.Block): + model = getattr(__main__, s) + model_name = s + break ui = MainWindow( model=model, model_var_name_in_main=model_name, From a7f21c5c9c36b39f0982c3db12aaaa29258240a6 Mon Sep 17 00:00:00 2001 From: Eslick Date: Thu, 23 May 2024 15:03:52 -0400 Subject: [PATCH 048/150] Update the doc --- pyomo/contrib/viewer/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pyomo/contrib/viewer/README.md b/pyomo/contrib/viewer/README.md index cfc50b54ce2..93d773e3829 100644 --- a/pyomo/contrib/viewer/README.md +++ b/pyomo/contrib/viewer/README.md @@ -42,6 +42,24 @@ ui = get_mainwindow(model=model) # Do model things, the viewer will stay in sync with the Pyomo model ``` +If you are working in Jupyter notebook, Jupyter qtconsole, or other Jupyter- +based IDEs, and your model is in the __main__ namespace (this is the usual case), +you can specify the model by its variable name as below. The advantage of this +is that if you replace the model with a new model having the same variable name, +the UI will automatically update without having to manually reset the model pointer. + +```python +%gui qt #Enables IPython's GUI event loop integration. +# Execute the above in its own cell and wait for it to finish before moving on. +from pyomo.contrib.viewer.ui import get_mainwindow +import pyomo.environ as pyo + +model = pyo.ConcreteModel() # could import an existing model here +ui = get_mainwindow(model_var_name_in_main="model") + +# Do model things, the viewer will stay in sync with the Pyomo model +``` + **Note:** the ```%gui qt``` cell must be executed in its own cell and execution must complete before running any other cells (you can't use "run all"). From 67fbe0bb958c7ea1f47bba71a38e4bf094f511a7 Mon Sep 17 00:00:00 2001 From: jasherma Date: Wed, 29 May 2024 19:18:56 -0400 Subject: [PATCH 049/150] Make PyROS temporarily adjust Pyomo NL writer tol --- pyomo/contrib/pyros/util.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 3b0187af7dd..8f86f0cc179 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -38,6 +38,7 @@ from pyomo.core.expr import value from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression from pyomo.repn.standard_repn import generate_standard_repn +from pyomo.repn.plugins import nl_writer as pyomo_nl_writer from pyomo.core.expr.visitor import ( identify_variables, identify_mutable_parameters, @@ -1809,6 +1810,16 @@ def call_solver(model, solver, config, timing_obj, timer_name, err_msg): timing_obj.start_timer(timer_name) tt_timer.tic(msg=None) + # tentative: reduce risk of InfeasibleConstraintException + # occurring due to discrepancies between Pyomo NL writer + # tolerance and (default) subordinate solver (e.g. IPOPT) + # feasibility tolerances. + # e.g., a Var fixed outside bounds beyond the Pyomo NL writer + # tolerance, but still within the default IPOPT feasibility + # tolerance + current_nl_writer_tol = pyomo_nl_writer.TOL + pyomo_nl_writer.TOL = 1e-4 + try: results = solver.solve( model, @@ -1827,6 +1838,8 @@ def call_solver(model, solver, config, timing_obj, timer_name, err_msg): results.solver, TIC_TOC_SOLVE_TIME_ATTR, tt_timer.toc(msg=None, delta=True) ) finally: + pyomo_nl_writer.TOL = current_nl_writer_tol + timing_obj.stop_timer(timer_name) revert_solver_max_time_adjustment( solver, orig_setting, custom_setting_present, config From 7460625becccdcd5adf2f3e8b228d76884d28c01 Mon Sep 17 00:00:00 2001 From: jasherma Date: Wed, 29 May 2024 19:40:20 -0400 Subject: [PATCH 050/150] Add test for adjustment of NL writer tolerance --- pyomo/contrib/pyros/tests/test_grcs.py | 72 +++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index f7efec4d6e7..5e323ad7a78 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -42,6 +42,7 @@ from pyomo.contrib.pyros.util import get_vars_from_component from pyomo.contrib.pyros.util import identify_objective_functions from pyomo.common.collections import Bunch +from pyomo.repn.plugins import nl_writer as pyomo_nl_writer import time import math from pyomo.contrib.pyros.util import time_code @@ -68,7 +69,7 @@ from pyomo.common.dependencies import numpy as np, numpy_available from pyomo.common.dependencies import scipy as sp, scipy_available from pyomo.environ import maximize as pyo_max -from pyomo.common.errors import ApplicationError +from pyomo.common.errors import ApplicationError, InfeasibleConstraintException from pyomo.opt import ( SolverResults, SolverStatus, @@ -4616,6 +4617,75 @@ def test_discrete_separation_subsolver_error(self): ), ) + def test_pyros_nl_writer_tol(self): + """ + Test PyROS subsolver call routine behavior + with respect to the NL writer tolerance is as + expected. + """ + m = ConcreteModel() + m.q = Param(initialize=1, mutable=True) + m.x1 = Var(initialize=1, bounds=(0, 1)) + m.x2 = Var(initialize=2, bounds=(0, m.q)) + m.obj = Objective(expr=m.x1 + m.x2) + + # fixed just inside the PyROS-specified NL writer tolerance. + m.x1.fix(m.x1.upper + 9.9e-5) + + current_nl_writer_tol = pyomo_nl_writer.TOL + ipopt_solver = SolverFactory("ipopt") + pyros_solver = SolverFactory("pyros") + + pyros_solver.solve( + model=m, + first_stage_variables=[m.x1], + second_stage_variables=[m.x2], + uncertain_params=[m.q], + uncertainty_set=BoxSet([[0, 1]]), + local_solver=ipopt_solver, + global_solver=ipopt_solver, + decision_rule_order=0, + solve_master_globally=False, + bypass_global_separation=True, + ) + + self.assertEqual( + pyomo_nl_writer.TOL, + current_nl_writer_tol, + msg="Pyomo NL writer tolerance not restored as expected.", + ) + + # fixed just outside the PyROS-specified NL writer tolerance. + # this should be exceptional. + m.x1.fix(m.x1.upper + 1.01e-4) + + err_msg = ( + "model contains a trivially infeasible variable.*x1" + ".*fixed.*outside bounds" + ) + with self.assertRaisesRegex(InfeasibleConstraintException, err_msg): + pyros_solver.solve( + model=m, + first_stage_variables=[m.x1], + second_stage_variables=[m.x2], + uncertain_params=[m.q], + uncertainty_set=BoxSet([[0, 1]]), + local_solver=ipopt_solver, + global_solver=ipopt_solver, + decision_rule_order=0, + solve_master_globally=False, + bypass_global_separation=True, + ) + + self.assertEqual( + pyomo_nl_writer.TOL, + current_nl_writer_tol, + msg=( + "Pyomo NL writer tolerance not restored as expected " + "after exceptional test." + ), + ) + @unittest.skipUnless( baron_license_is_valid, "Global NLP solver is not available and licensed." ) From 4f2cda6bf374a21d8bb16098140edfcd41154112 Mon Sep 17 00:00:00 2001 From: jasherma Date: Wed, 29 May 2024 20:07:08 -0400 Subject: [PATCH 051/150] Fix IPOPT subsolver wall time limit restoration --- pyomo/contrib/pyros/util.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 8f86f0cc179..ecabca8f115 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -378,7 +378,14 @@ def revert_solver_max_time_adjustment( elif isinstance(solver, SolverFactory.get_class("baron")): options_key = "MaxTime" elif isinstance(solver, SolverFactory.get_class("ipopt")): - options_key = "max_cpu_time" + options_key = ( + # IPOPT 3.14.0+ added support for specifying + # wall time limit explicitly; this is preferred + # over CPU time limit + "max_wall_time" + if solver.version() >= (3, 14, 0, 0) + else "max_cpu_time" + ) elif isinstance(solver, SolverFactory.get_class("scip")): options_key = "limits/time" else: From edeaf1d2c75ae999596072ef130253024f14b3dd Mon Sep 17 00:00:00 2001 From: jasherma Date: Wed, 29 May 2024 22:07:01 -0400 Subject: [PATCH 052/150] Add IPOPT availability check to new test --- pyomo/contrib/pyros/tests/test_grcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index 5e323ad7a78..f2954750a16 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -4617,6 +4617,7 @@ def test_discrete_separation_subsolver_error(self): ), ) + @unittest.skipUnless(ipopt_available, "IPOPT is not available.") def test_pyros_nl_writer_tol(self): """ Test PyROS subsolver call routine behavior From 96355df4552fb1ba8b96231b8320c2876707801f Mon Sep 17 00:00:00 2001 From: Atalay Kutlay Date: Sat, 1 Jun 2024 23:50:59 -0400 Subject: [PATCH 053/150] bug: Sort indices before sending to solver --- pyomo/contrib/appsi/solvers/highs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/appsi/solvers/highs.py b/pyomo/contrib/appsi/solvers/highs.py index c948444839d..57a7b1eac72 100644 --- a/pyomo/contrib/appsi/solvers/highs.py +++ b/pyomo/contrib/appsi/solvers/highs.py @@ -481,7 +481,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): indices_to_remove.append(con_ndx) self._mutable_helpers.pop(con, None) self._solver_model.deleteRows( - len(indices_to_remove), np.array(indices_to_remove) + len(indices_to_remove), np.sort(np.array(indices_to_remove)) ) con_ndx = 0 new_con_map = dict() From 6834c906bc299575f7588bb3b07558b63c5d7050 Mon Sep 17 00:00:00 2001 From: Miranda Rose Mundt Date: Wed, 5 Jun 2024 10:32:35 -0600 Subject: [PATCH 054/150] Add in cfgfile information; turn off SAS Viya tests for now --- pyomo/solvers/tests/checks/test_SAS.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 75534e0e001..843ca3f2d2b 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -29,6 +29,8 @@ from pyomo.opt import SolverFactory, check_available_solvers import warnings +CFGFILE = os.environ.get("SAS_CFG_FILE_PATH", None) + CAS_OPTIONS = { "hostname": os.environ.get("CASHOST", None), "port": os.environ.get("CASPORT", None), @@ -42,11 +44,12 @@ class SASTestAbc: solver_io = "_sas94" session_options = {} + cfgfile = CFGFILE @classmethod def setUpClass(cls): cls.opt_sas = SolverFactory( - "sas", solver_io=cls.solver_io, **cls.session_options + "sas", solver_io=cls.solver_io, cfgfile=cls.cfgfile, **cls.session_options ) @classmethod @@ -322,7 +325,8 @@ def test_solver_error(self, submit_mock, symget_mock): self.assertEqual(results.solver.status, SolverStatus.error) -@unittest.skipIf(not sas_available, "The SAS solver is not available") +#@unittest.skipIf(not sas_available, "The SAS solver is not available") +@unittest.skip("Tests not yet configured for SAS Viya interface.") class SASTestLPCAS(SASTestLP, unittest.TestCase): solver_io = "_sascas" session_options = CAS_OPTIONS @@ -528,6 +532,7 @@ class SASTestMILP94(SASTestMILP, unittest.TestCase): @unittest.skipIf(not sas_available, "The SAS solver is not available") +@unittest.skip("Tests not yet configured for SAS Viya interface.") class SASTestMILPCAS(SASTestMILP, unittest.TestCase): solver_io = "_sascas" session_options = CAS_OPTIONS From 6d5c931bf00b9ac9b0144314584eff2b52d8b92c Mon Sep 17 00:00:00 2001 From: Miranda Rose Mundt Date: Wed, 5 Jun 2024 10:38:20 -0600 Subject: [PATCH 055/150] Apply black to test_SAS --- pyomo/solvers/tests/checks/test_SAS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 843ca3f2d2b..00b4d7a9b3e 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -325,7 +325,7 @@ def test_solver_error(self, submit_mock, symget_mock): self.assertEqual(results.solver.status, SolverStatus.error) -#@unittest.skipIf(not sas_available, "The SAS solver is not available") +# @unittest.skipIf(not sas_available, "The SAS solver is not available") @unittest.skip("Tests not yet configured for SAS Viya interface.") class SASTestLPCAS(SASTestLP, unittest.TestCase): solver_io = "_sascas" @@ -531,7 +531,7 @@ class SASTestMILP94(SASTestMILP, unittest.TestCase): pass -@unittest.skipIf(not sas_available, "The SAS solver is not available") +# @unittest.skipIf(not sas_available, "The SAS solver is not available") @unittest.skip("Tests not yet configured for SAS Viya interface.") class SASTestMILPCAS(SASTestMILP, unittest.TestCase): solver_io = "_sascas" From 60dcc7a3f9875ab7e32a2d9540edfcfa243f054d Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Wed, 5 Jun 2024 13:14:16 -0600 Subject: [PATCH 056/150] Import specific library from uuid --- pyomo/solvers/plugins/solvers/SAS.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index bccb7d34077..eb5014fbd1a 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -12,7 +12,7 @@ import logging import sys from os import stat -import uuid +from uuid import uuid4 from abc import ABC, abstractmethod from io import StringIO @@ -334,7 +334,7 @@ def _apply_solver(self): rootnode_str = self._create_statement_str("rootnode") # Get a unique identifier, always use the same with different prefixes - unique = uuid.uuid4().hex[:16] + unique = uuid4().hex[:16] # Create unique filename for output datasets primalout_dataset_name = "pout" + unique @@ -703,7 +703,7 @@ def _apply_solver(self): action = "solveMilp" if self._has_integer_variables() else "solveLp" # Get a unique identifier, always use the same with different prefixes - unique = uuid.uuid4().hex[:16] + unique = uuid4().hex[:16] # Creat the output stream, we want to print to a log string as well as to the console self._log = StringIO() From f4a17e1debd5e69ae0cc0c1372dc70192910cf8e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sat, 8 Jun 2024 17:11:31 -0600 Subject: [PATCH 057/150] Update ConstraintData to only store the expression (and not lower, body, upper) --- pyomo/core/base/constraint.py | 296 ++++++++++-------------------- pyomo/core/kernel/constraint.py | 3 + pyomo/core/tests/unit/test_con.py | 248 +++++++++++++++++-------- 3 files changed, 273 insertions(+), 274 deletions(-) diff --git a/pyomo/core/base/constraint.py b/pyomo/core/base/constraint.py index e12860991c2..b3846b44a0a 100644 --- a/pyomo/core/base/constraint.py +++ b/pyomo/core/base/constraint.py @@ -64,6 +64,7 @@ InequalityExpression, RangedExpression, } +_strict_relational_exprs = {True, (False, True), (True, False), (True, True)} _rule_returned_none_error = """Constraint '%s': rule returned None. Constraint rules must return either a valid expression, a 2- or 3-member @@ -151,7 +152,7 @@ class ConstraintData(ActiveComponentData): _active A boolean that indicates whether this data is active """ - __slots__ = ('_body', '_lower', '_upper', '_expr') + __slots__ = ('_expr',) # Set to true when a constraint class stores its expression # in linear canonical form @@ -167,126 +168,139 @@ def __init__(self, expr=None, component=None): self._component = weakref_ref(component) if (component is not None) else None self._active = True - self._body = None - self._lower = None - self._upper = None self._expr = None if expr is not None: self.set_value(expr) def __call__(self, exception=True): """Compute the value of the body of this constraint.""" - return value(self.body, exception=exception) + body = self.normalize_constraint()[1] + if body.__class__ not in native_numeric_types: + body = value(self.body, exception=exception) + return body + + def normalize_constraint(self): + expr = self._expr + if expr.__class__ is RangedExpression: + lb, body, ub = ans = expr.args + if ( + lb.__class__ not in native_types + and lb.is_potentially_variable() + and not lb.is_fixed() + ): + raise ValueError( + f"Constraint '{self.name}' is a Ranged Inequality with a " + "variable lower bound. Cannot normalize the " + "constraint or send it to a solver." + ) + if ( + ub.__class__ not in native_types + and ub.is_potentially_variable() + and not ub.is_fixed() + ): + raise ValueError( + f"Constraint '{self.name}' is a Ranged Inequality with a " + "variable upper bound. Cannot normalize the " + "constraint or send it to a solver." + ) + return ans + elif expr is not None: + lhs, rhs = expr.args + if rhs.__class__ in native_types or not rhs.is_potentially_variable(): + return rhs if expr.__class__ is EqualityExpression else None, lhs, rhs + if lhs.__class__ in native_types or not lhs.is_potentially_variable(): + return lhs, rhs, lhs if expr.__class__ is EqualityExpression else None + return 0 if expr.__class__ is EqualityExpression else None, lhs - rhs, 0 + return None, None, None @property def body(self): """Access the body of a constraint expression.""" - if self._body is not None: - return self._body - # The incoming RangedInequality had a potentially variable - # bound. The "body" is fine, but the bounds may not be - # (although the responsibility for those checks lies with the - # lower/upper properties) - body = self._expr.arg(1) - if body.__class__ in native_types and body is not None: - return as_numeric(body) - return body - - def _get_range_bound(self, range_arg): - # Equalities and simple inequalities can always be (directly) - # reformulated at construction time to force constant bounds. - # The only time we need to defer the determination of bounds is - # for ranged inequalities that contain non-constant bounds (so - # we *know* that the expr will have 3 args) - # - # It is possible that there is no expression at all (so catch that) - if self._expr is None: - return None - bound = self._expr.arg(range_arg) - if not is_fixed(bound): - raise ValueError( - "Constraint '%s' is a Ranged Inequality with a " - "variable %s bound. Cannot normalize the " - "constraint or send it to a solver." - % (self.name, {0: 'lower', 2: 'upper'}[range_arg]) - ) - return bound + try: + ans = self.normalize_constraint()[1] + except ValueError: + if self._expr.__class__ is RangedExpression: + _, ans, _ = self._expr.args + else: + raise + if ans.__class__ in native_numeric_types: + return as_numeric(ans) + return ans @property def lower(self): """Access the lower bound of a constraint expression.""" - bound = self._lower if self._body is not None else self._get_range_bound(0) - # Historically, constraint.lower was guaranteed to return a type - # derived from Pyomo NumericValue (or None). Replicate that - # functionality, although clients should in almost all cases - # move to using ConstraintData.lb instead of accessing - # lower/body/upper to avoid the unnecessary creation (and - # inevitable destruction) of the NumericConstant wrappers. - if bound is None: - return None - return as_numeric(bound) + ans = self.normalize_constraint()[0] + if ans.__class__ in native_numeric_types: + # Historically, constraint.lower was guaranteed to return a type + # derived from Pyomo NumericValue (or None). Replicate that + # functionality, although clients should in almost all cases + # move to using ConstraintData.lb instead of accessing + # lower/body/upper to avoid the unnecessary creation (and + # inevitable destruction) of the NumericConstant wrappers. + return as_numeric(ans) + return ans @property def upper(self): """Access the upper bound of a constraint expression.""" - bound = self._upper if self._body is not None else self._get_range_bound(2) - # Historically, constraint.upper was guaranteed to return a type - # derived from Pyomo NumericValue (or None). Replicate that - # functionality, although clients should in almost all cases - # move to using ConstraintData.ub instead of accessing - # lower/body/upper to avoid the unnecessary creation (and - # inevitable destruction) of the NumericConstant wrappers. - if bound is None: - return None - return as_numeric(bound) + ans = self.normalize_constraint()[2] + if ans.__class__ in native_numeric_types: + # Historically, constraint.lower was guaranteed to return a type + # derived from Pyomo NumericValue (or None). Replicate that + # functionality, although clients should in almost all cases + # move to using ConstraintData.lb instead of accessing + # lower/body/upper to avoid the unnecessary creation (and + # inevitable destruction) of the NumericConstant wrappers. + return as_numeric(ans) + return ans @property def lb(self): """Access the value of the lower bound of a constraint expression.""" - bound = self._lower if self._body is not None else self._get_range_bound(0) + bound = self.normalize_constraint()[0] + if bound is None: + return None if bound.__class__ not in native_numeric_types: - if bound is None: - return None bound = float(value(bound)) + # Note that "bound != bound" catches float('nan') if bound in _nonfinite_values or bound != bound: - # Note that "bound != bound" catches float('nan') if bound == -_inf: return None - else: - raise ValueError( - "Constraint '%s' created with an invalid non-finite " - "lower bound (%s)." % (self.name, bound) - ) + raise ValueError( + f"Constraint '{self.name}' created with an invalid non-finite " + f"lower bound ({bound})." + ) return bound @property def ub(self): """Access the value of the upper bound of a constraint expression.""" - bound = self._upper if self._body is not None else self._get_range_bound(2) + bound = self.normalize_constraint()[2] + if bound is None: + return None if bound.__class__ not in native_numeric_types: - if bound is None: - return None bound = float(value(bound)) + # Note that "bound != bound" catches float('nan') if bound in _nonfinite_values or bound != bound: - # Note that "bound != bound" catches float('nan') if bound == _inf: return None - else: - raise ValueError( - "Constraint '%s' created with an invalid non-finite " - "upper bound (%s)." % (self.name, bound) - ) + raise ValueError( + f"Constraint '{self.name}' created with an invalid non-finite " + f"upper bound ({bound})." + ) return bound @property def equality(self): """A boolean indicating whether this is an equality constraint.""" - if self._expr.__class__ is EqualityExpression: + expr = self.expr + if expr.__class__ is EqualityExpression: return True - elif self._expr.__class__ is RangedExpression: + elif expr.__class__ is RangedExpression: # TODO: this is a very restrictive form of structural equality. - lb = self._expr.arg(0) - if lb is not None and lb is self._expr.arg(2): + lb = expr.arg(0) + if lb is not None and lb is expr.arg(2): return True return False @@ -317,15 +331,22 @@ def expr(self): def get_value(self): """Get the expression on this constraint.""" - return self._expr + return self.expr def set_value(self, expr): """Set the expression on this constraint.""" # Clear any previously-cached normalized constraint - self._lower = self._upper = self._body = self._expr = None - + self._expr = None if expr.__class__ in _known_relational_expressions: + if getattr(expr, 'strict', False) in _strict_relational_exprs: + raise ValueError( + "Constraint '%s' encountered a strict " + "inequality expression ('>' or '< '). All" + " constraints must be formulated using " + "using '<=', '>=', or '=='." % (self.name,) + ) self._expr = expr + elif expr.__class__ is tuple: # or expr_type is list: for arg in expr: if ( @@ -422,120 +443,6 @@ def set_value(self, expr): "\n (0, model.price[item], 50)" % (self.name, str(expr)) ) raise ValueError(msg) - # - # Normalize the incoming expressions, if we can - # - args = self._expr.args - if self._expr.__class__ is InequalityExpression: - if self._expr.strict: - raise ValueError( - "Constraint '%s' encountered a strict " - "inequality expression ('>' or '< '). All" - " constraints must be formulated using " - "using '<=', '>=', or '=='." % (self.name,) - ) - if ( - args[1] is None - or args[1].__class__ in native_numeric_types - or not args[1].is_potentially_variable() - ): - self._body = args[0] - self._upper = args[1] - elif ( - args[0] is None - or args[0].__class__ in native_numeric_types - or not args[0].is_potentially_variable() - ): - self._lower = args[0] - self._body = args[1] - else: - self._body = args[0] - args[1] - self._upper = 0 - elif self._expr.__class__ is EqualityExpression: - if args[0] is None or args[1] is None: - # Error check: ensure equality does not have infinite RHS - raise ValueError( - "Equality constraint '%s' defined with " - "non-finite term (%sHS == None)." - % (self.name, 'L' if args[0] is None else 'R') - ) - if ( - args[0].__class__ in native_numeric_types - or not args[0].is_potentially_variable() - ): - self._lower = self._upper = args[0] - self._body = args[1] - elif ( - args[1].__class__ in native_numeric_types - or not args[1].is_potentially_variable() - ): - self._lower = self._upper = args[1] - self._body = args[0] - else: - self._lower = self._upper = 0 - self._body = args[0] - args[1] - # The following logic is caught below when checking for - # invalid non-finite bounds: - # - # if self._lower.__class__ in native_numeric_types and \ - # not math.isfinite(self._lower): - # raise ValueError( - # "Equality constraint '%s' defined with " - # "non-finite term." % (self.name)) - elif self._expr.__class__ is RangedExpression: - if any(self._expr.strict): - raise ValueError( - "Constraint '%s' encountered a strict " - "inequality expression ('>' or '< '). All" - " constraints must be formulated using " - "using '<=', '>=', or '=='." % (self.name,) - ) - if all( - ( - arg is None - or arg.__class__ in native_numeric_types - or not arg.is_potentially_variable() - ) - for arg in (args[0], args[2]) - ): - self._lower, self._body, self._upper = args - else: - # Defensive programming: we currently only support three - # relational expression types. This will only be hit if - # someone defines a fourth... - raise DeveloperError( - "Unrecognized relational expression type: %s" - % (self._expr.__class__.__name__,) - ) - - # We have historically forced the body to be a numeric expression. - # TODO: remove this requirement - if self._body.__class__ in native_types and self._body is not None: - self._body = as_numeric(self._body) - - # We have historically mapped incoming inf to None - if self._lower.__class__ in native_numeric_types: - bound = self._lower - if bound in _nonfinite_values or bound != bound: - # Note that "bound != bound" catches float('nan') - if bound == -_inf: - self._lower = None - else: - raise ValueError( - "Constraint '%s' created with an invalid non-finite " - "lower bound (%s)." % (self.name, self._lower) - ) - if self._upper.__class__ in native_numeric_types: - bound = self._upper - if bound in _nonfinite_values or bound != bound: - # Note that "bound != bound" catches float('nan') - if bound == _inf: - self._upper = None - else: - raise ValueError( - "Constraint '%s' created with an invalid non-finite " - "upper bound (%s)." % (self.name, self._upper) - ) def lslack(self): """ @@ -911,6 +818,7 @@ class SimpleConstraint(metaclass=RenamedClass): { 'add', 'set_value', + 'normalize_constraint', 'body', 'lower', 'upper', diff --git a/pyomo/core/kernel/constraint.py b/pyomo/core/kernel/constraint.py index 6aa4abc4bfe..6b8c4c619f5 100644 --- a/pyomo/core/kernel/constraint.py +++ b/pyomo/core/kernel/constraint.py @@ -177,6 +177,9 @@ class _MutableBoundsConstraintMixin(object): # Define some of the IConstraint abstract methods # + def normalize_constraint(self): + return self.lower, self.body, self.upper + @property def lower(self): """The expression for the lower bound of the constraint""" diff --git a/pyomo/core/tests/unit/test_con.py b/pyomo/core/tests/unit/test_con.py index 15f190e281e..7274adae397 100644 --- a/pyomo/core/tests/unit/test_con.py +++ b/pyomo/core/tests/unit/test_con.py @@ -84,21 +84,55 @@ def rule(model): self.assertEqual(model.c.upper, 0) def test_tuple_construct_inf_equality(self): - model = self.create_model(abstract=True) - - def rule(model): - return (model.x, float('inf')) - - model.c = Constraint(rule=rule) - self.assertRaises(ValueError, model.create_instance) - - model = self.create_model(abstract=True) - - def rule(model): - return (float('inf'), model.x) + model = self.create_model(abstract=True).create_instance() - model.c = Constraint(rule=rule) - self.assertRaises(ValueError, model.create_instance) + model.c = Constraint(expr=(model.x, float('inf'))) + self.assertEqual(model.c.equality, True) + self.assertEqual(model.c.lower, float('inf')) + self.assertIs(model.c.body, model.x) + self.assertEqual(model.c.upper, float('inf')) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'c' created with an invalid non-finite lower bound \(inf\).", + ): + model.c.lb + self.assertEqual(model.c.ub, None) + + model.d = Constraint(expr=(float('inf'), model.x)) + self.assertEqual(model.d.equality, True) + self.assertEqual(model.d.lower, float('inf')) + self.assertIs(model.d.body, model.x) + self.assertEqual(model.d.upper, float('inf')) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'd' created with an invalid non-finite lower bound \(inf\).", + ): + model.d.lb + self.assertEqual(model.d.ub, None) + + model.e = Constraint(expr=(model.x, float('-inf'))) + self.assertEqual(model.e.equality, True) + self.assertEqual(model.e.lower, float('-inf')) + self.assertIs(model.e.body, model.x) + self.assertEqual(model.e.upper, float('-inf')) + self.assertEqual(model.e.lb, None) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'e' created with an invalid non-finite upper bound \(-inf\).", + ): + model.e.ub + + model.f = Constraint(expr=(float('-inf'), model.x)) + self.assertEqual(model.f.equality, True) + self.assertEqual(model.f.lower, float('-inf')) + self.assertIs(model.f.body, model.x) + self.assertEqual(model.f.upper, float('-inf')) + self.assertEqual(model.f.lb, None) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'f' created with an invalid non-finite upper bound \(-inf\).", + ): + model.f.ub def test_tuple_construct_1sided_inequality(self): model = self.create_model() @@ -134,9 +168,11 @@ def rule(model): model.c = Constraint(rule=rule) self.assertEqual(model.c.equality, False) - self.assertEqual(model.c.lower, None) + self.assertEqual(model.c.lower, float('-inf')) self.assertIs(model.c.body, model.y) self.assertEqual(model.c.upper, 1) + self.assertEqual(model.c.lb, None) + self.assertEqual(model.c.ub, 1) model = self.create_model() @@ -148,7 +184,9 @@ def rule(model): self.assertEqual(model.c.equality, False) self.assertEqual(model.c.lower, 0) self.assertIs(model.c.body, model.y) - self.assertEqual(model.c.upper, None) + self.assertEqual(model.c.upper, float('inf')) + self.assertEqual(model.c.lb, 0) + self.assertEqual(model.c.ub, None) def test_tuple_construct_unbounded_inequality(self): model = self.create_model() @@ -171,9 +209,11 @@ def rule(model): model.c = Constraint(rule=rule) self.assertEqual(model.c.equality, False) - self.assertEqual(model.c.lower, None) + self.assertEqual(model.c.lower, float('-inf')) self.assertIs(model.c.body, model.y) - self.assertEqual(model.c.upper, None) + self.assertEqual(model.c.upper, float('inf')) + self.assertEqual(model.c.lb, None) + self.assertEqual(model.c.ub, None) def test_tuple_construct_invalid_1sided_inequality(self): model = self.create_model(abstract=True) @@ -229,7 +269,11 @@ def rule(model): ): instance.c.lower self.assertIs(instance.c.body, instance.y) - self.assertEqual(instance.c.upper, 1) + with self.assertRaisesRegex( + ValueError, + "Constraint 'c' is a Ranged Inequality with a variable lower bound", + ): + instance.c.upper instance.x.fix(3) self.assertEqual(value(instance.c.lower), 3) @@ -240,7 +284,11 @@ def rule(model): model.c = Constraint(rule=rule) instance = model.create_instance() - self.assertEqual(instance.c.lower, 0) + with self.assertRaisesRegex( + ValueError, + "Constraint 'c' is a Ranged Inequality with a variable upper bound", + ): + instance.c.lower self.assertIs(instance.c.body, instance.y) with self.assertRaisesRegex( ValueError, @@ -276,21 +324,23 @@ def rule(model): self.assertEqual(model.c.upper, 0) def test_expr_construct_inf_equality(self): - model = self.create_model(abstract=True) - - def rule(model): - return model.x == float('inf') - - model.c = Constraint(rule=rule) - self.assertRaises(ValueError, model.create_instance) + model = self.create_model(abstract=True).create_instance() - model = self.create_model(abstract=True) - - def rule(model): - return float('inf') == model.x + model.c = Constraint(expr=model.x == float('inf')) + self.assertEqual(model.c.ub, None) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'c' created with an invalid non-finite lower bound \(inf\).", + ): + model.c.lb - model.c = Constraint(rule=rule) - self.assertRaises(ValueError, model.create_instance) + model.d = Constraint(expr=model.x == float('-inf')) + self.assertEqual(model.d.lb, None) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'd' created with an invalid non-finite upper bound \(-inf\).", + ): + model.d.ub def test_expr_construct_1sided_inequality(self): model = self.create_model() @@ -350,9 +400,11 @@ def rule(model): model.c = Constraint(rule=rule) self.assertEqual(model.c.equality, False) - self.assertEqual(model.c.lower, None) + self.assertIs(model.c.lower, None) self.assertIs(model.c.body, model.y) - self.assertEqual(model.c.upper, None) + self.assertEqual(model.c.upper, float('inf')) + self.assertIs(model.c.ub, None) + self.assertIs(model.c.lb, None) model = self.create_model() @@ -362,9 +414,11 @@ def rule(model): model.c = Constraint(rule=rule) self.assertEqual(model.c.equality, False) - self.assertEqual(model.c.lower, None) + self.assertEqual(model.c.lower, float('-inf')) self.assertIs(model.c.body, model.y) self.assertEqual(model.c.upper, None) + self.assertIs(model.c.ub, None) + self.assertIs(model.c.lb, None) model = self.create_model() @@ -374,9 +428,11 @@ def rule(model): model.c = Constraint(rule=rule) self.assertEqual(model.c.equality, False) - self.assertEqual(model.c.lower, None) + self.assertEqual(model.c.lower, float('-inf')) self.assertIs(model.c.body, model.y) self.assertEqual(model.c.upper, None) + self.assertIs(model.c.ub, None) + self.assertIs(model.c.lb, None) model = self.create_model() @@ -388,40 +444,40 @@ def rule(model): self.assertEqual(model.c.equality, False) self.assertEqual(model.c.lower, None) self.assertIs(model.c.body, model.y) - self.assertEqual(model.c.upper, None) + self.assertEqual(model.c.upper, float('inf')) + self.assertIs(model.c.ub, None) + self.assertIs(model.c.lb, None) def test_expr_construct_invalid_unbounded_inequality(self): - model = self.create_model(abstract=True) - - def rule(model): - return model.y <= float('-inf') - - model.c = Constraint(rule=rule) - self.assertRaises(ValueError, model.create_instance) - - model = self.create_model(abstract=True) - - def rule(model): - return float('inf') <= model.y - - model.c = Constraint(rule=rule) - self.assertRaises(ValueError, model.create_instance) - - model = self.create_model(abstract=True) - - def rule(model): - return model.y >= float('inf') + model = self.create_model(abstract=True).create_instance() - model.c = Constraint(rule=rule) - self.assertRaises(ValueError, model.create_instance) + model.c = Constraint(expr=model.y <= float('-inf')) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'c' created with an invalid non-finite upper bound \(-inf\).", + ): + model.c.ub - model = self.create_model(abstract=True) + model.d = Constraint(expr=float('inf') <= model.y) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'd' created with an invalid non-finite lower bound \(inf\).", + ): + model.d.lb - def rule(model): - return float('-inf') >= model.y + model.e = Constraint(expr=model.y >= float('inf')) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'e' created with an invalid non-finite lower bound \(inf\).", + ): + model.e.lb - model.c = Constraint(rule=rule) - self.assertRaises(ValueError, model.create_instance) + model.f = Constraint(expr=float('-inf') >= model.y) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'f' created with an invalid non-finite upper bound \(-inf\).", + ): + model.f.ub def test_expr_construct_invalid(self): m = ConcreteModel() @@ -484,9 +540,6 @@ def test_nondata_bounds(self): model.e2 = Expression() model.e3 = Expression() model.c.set_value((model.e1, model.e2, model.e3)) - self.assertIsNone(model.c._lower) - self.assertIsNone(model.c._body) - self.assertIsNone(model.c._upper) self.assertIs(model.c.lower, model.e1) self.assertIs(model.c.body, model.e2) self.assertIs(model.c.upper, model.e3) @@ -507,7 +560,7 @@ def test_nondata_bounds(self): self.assertIs(model.c.body.expr, model.v[2]) with self.assertRaisesRegex( ValueError, - "Constraint 'c' is a Ranged Inequality with a variable upper bound", + "Constraint 'c' is a Ranged Inequality with a variable lower bound", ): model.c.upper @@ -1574,10 +1627,30 @@ def rule1(model): self.assertIs(instance.c.body, instance.x) with self.assertRaisesRegex( ValueError, - "Constraint 'c' is a Ranged Inequality with a variable upper bound", + "Constraint 'c' is a Ranged Inequality with a variable lower bound", ): instance.c.upper # + def rule1(model): + return (0, model.x, model.z) + + model = AbstractModel() + model.x = Var() + model.z = Var() + model.c = Constraint(rule=rule1) + instance = model.create_instance() + with self.assertRaisesRegex( + ValueError, + "Constraint 'c' is a Ranged Inequality with a variable upper bound", + ): + instance.c.lower + self.assertIs(instance.c.body, instance.x) + with self.assertRaisesRegex( + ValueError, + "Constraint 'c' is a Ranged Inequality with a variable upper bound", + ): + instance.c.upper + def test_expression_constructor_coverage(self): def rule1(model): @@ -1807,23 +1880,39 @@ def test_potentially_variable_bounds(self): r"Constraint 'c' is a Ranged Inequality with a variable lower bound", ): m.c.lower - self.assertIs(m.c.upper, m.u) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'c' is a Ranged Inequality with a variable lower bound", + ): + self.assertIs(m.c.upper, m.u) with self.assertRaisesRegex( ValueError, r"Constraint 'c' is a Ranged Inequality with a variable lower bound", ): m.c.lb - self.assertEqual(m.c.ub, 10) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'c' is a Ranged Inequality with a variable lower bound", + ): + self.assertEqual(m.c.ub, 10) m.l = 15 m.u.expr = m.x - self.assertIs(m.c.lower, m.l) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'c' is a Ranged Inequality with a variable upper bound", + ): + self.assertIs(m.c.lower, m.l) with self.assertRaisesRegex( ValueError, r"Constraint 'c' is a Ranged Inequality with a variable upper bound", ): m.c.upper - self.assertEqual(m.c.lb, 15) + with self.assertRaisesRegex( + ValueError, + r"Constraint 'c' is a Ranged Inequality with a variable upper bound", + ): + self.assertEqual(m.c.lb, 15) with self.assertRaisesRegex( ValueError, r"Constraint 'c' is a Ranged Inequality with a variable upper bound", @@ -1890,17 +1979,16 @@ def test_tuple_expression(self): ): m.c = (m.x, None) + # You can create it with an infinite value, but then one of the + # bounds will fail: + m.c = (m.x, float('inf')) + self.assertIsNone(m.c.ub) with self.assertRaisesRegex( ValueError, r"Constraint 'c' created with an invalid " r"non-finite lower bound \(inf\)", ): - m.c = (m.x, float('inf')) - - with self.assertRaisesRegex( - ValueError, r"Equality constraint 'c' defined with non-finite term" - ): - m.c = EqualityExpression((m.x, None)) + m.c.lb if __name__ == "__main__": From 25027f9fe76e6513a3491976b5c4a2e2f5a14f4a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sat, 8 Jun 2024 17:30:43 -0600 Subject: [PATCH 058/150] Update FBBT to work with raw relational expressions --- .../contrib/fbbt/expression_bounds_walker.py | 6 +- pyomo/contrib/fbbt/fbbt.py | 105 ++++++++++++++---- pyomo/contrib/fbbt/interval.py | 18 +-- 3 files changed, 94 insertions(+), 35 deletions(-) diff --git a/pyomo/contrib/fbbt/expression_bounds_walker.py b/pyomo/contrib/fbbt/expression_bounds_walker.py index cb287d54df5..3cb32fcbf29 100644 --- a/pyomo/contrib/fbbt/expression_bounds_walker.py +++ b/pyomo/contrib/fbbt/expression_bounds_walker.py @@ -232,15 +232,15 @@ def _handle_unknowable_bounds(visitor, node, arg): def _handle_equality(visitor, node, arg1, arg2): - return eq(*arg1, *arg2) + return eq(*arg1, *arg2, feasibility_tol=visitor.feasibility_tol) def _handle_inequality(visitor, node, arg1, arg2): - return ineq(*arg1, *arg2) + return ineq(*arg1, *arg2, feasibility_tol=visitor.feasibility_tol) def _handle_ranged(visitor, node, arg1, arg2, arg3): - return ranged(*arg1, *arg2, *arg3) + return ranged(*arg1, *arg2, *arg3, feasibility_tol=visitor.feasibility_tol) def _handle_expr_if(visitor, node, arg1, arg2, arg3): diff --git a/pyomo/contrib/fbbt/fbbt.py b/pyomo/contrib/fbbt/fbbt.py index 1507c4a3cc5..62cd90d9c87 100644 --- a/pyomo/contrib/fbbt/fbbt.py +++ b/pyomo/contrib/fbbt/fbbt.py @@ -12,6 +12,7 @@ from collections import defaultdict from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.contrib.fbbt.expression_bounds_walker import ExpressionBoundsVisitor +import pyomo.core.expr.relational_expr as relational_expr import pyomo.core.expr.numeric_expr as numeric_expr from pyomo.core.expr.visitor import ( ExpressionValueVisitor, @@ -80,6 +81,24 @@ class FBBTException(PyomoException): pass +def _prop_bnds_leaf_to_root_equality(visitor, node, arg1, arg2): + bnds_dict = visitor.bnds_dict + bnds_dict[node] = interval.eq( + *bnds_dict[arg1], *bnds_dict[arg2], visitor.feasibility_tol + ) + +def _prop_bnds_leaf_to_root_inequality(visitor, node, arg1, arg2): + bnds_dict = visitor.bnds_dict + bnds_dict[node] = interval.ineq( + *bnds_dict[arg1], *bnds_dict[arg2], visitor.feasibility_tol + ) + +def _prop_bnds_leaf_to_root_ranged(visitor, node, arg1, arg2, arg3): + bnds_dict = visitor.bnds_dict + bnds_dict[node] = interval.ranged( + *bnds_dict[arg1], *bnds_dict[arg2], *bnds_dict[arg3], visitor.feasibility_tol + ) + def _prop_bnds_leaf_to_root_ProductExpression(visitor, node, arg1, arg2): """ @@ -367,6 +386,9 @@ def _prop_bnds_leaf_to_root_NamedExpression(visitor, node, expr): numeric_expr.UnaryFunctionExpression: _prop_bnds_leaf_to_root_UnaryFunctionExpression, numeric_expr.LinearExpression: _prop_bnds_leaf_to_root_SumExpression, numeric_expr.AbsExpression: _prop_bnds_leaf_to_root_abs, + relational_expr.EqualityExpression: _prop_bnds_leaf_to_root_equality, + relational_expr.InequalityExpression: _prop_bnds_leaf_to_root_inequality, + relational_expr.RangedExpression: _prop_bnds_leaf_to_root_ranged, ExpressionData: _prop_bnds_leaf_to_root_NamedExpression, ScalarExpression: _prop_bnds_leaf_to_root_NamedExpression, ObjectiveData: _prop_bnds_leaf_to_root_NamedExpression, @@ -375,6 +397,51 @@ def _prop_bnds_leaf_to_root_NamedExpression(visitor, node, expr): ) +def _prop_bnds_root_to_leaf_equality(node, bnds_dict, feasibility_tol): + assert bnds_dict[node][1] # This expression is feasible + arg1, arg2 = node.args + lb1, ub1 = bnds_dict[arg1] + lb2, ub2 = bnds_dict[arg2] + bnds_dict[arg1] = bnds_dict[arg2] = max(lb1, lb2), min(ub1, ub2) + + +def _prop_bnds_root_to_leaf_inequality(node, bnds_dict, feasibility_tol): + assert bnds_dict[node][1] # This expression is feasible + arg1, arg2 = node.args + lb1, ub1 = bnds_dict[arg1] + lb2, ub2 = bnds_dict[arg2] + if lb1 > lb2: + bnds_dict[arg2] = lb1, ub2 + if ub1 > ub2: + bnds_dict[arg1] = lb1, ub2 + + +def _prop_bnds_root_to_leaf_ranged(node, bnds_dict, feasibility_tol): + assert bnds_dict[node][1] # This expression is feasible + arg1, arg2, arg3 = node.args + lb1, ub1 = bnds_dict[arg1] + lb2, ub2 = bnds_dict[arg2] + lb3, ub3 = bnds_dict[arg3] + if lb1 > lb2: + bnds_dict[arg2] = lb1, ub2 + lb2 = lb1 + if lb2 > lb3: + bnds_dict[arg3] = lb2, ub3 + if ub2 > ub3: + bnds_dict[arg2] = lb2, ub3 + ub2 = ub3 + if ub1 > ub2: + bnds_dict[arg1] = lb1, ub2 + + +def _prop_bnds_root_to_leaf_equality(node, bnds_dict, feasibility_tol): + assert bnds_dict[node][1] # This expression is feasible + arg1, arg2 = node.args + lb1, ub1 = bnds_dict[arg1] + lb2, ub2 = bnds_dict[arg2] + bnds_dict[arg1] = bnds_dict[arg2] = max(lb1, lb2), min(ub1, ub2) + + def _prop_bnds_root_to_leaf_ProductExpression(node, bnds_dict, feasibility_tol): """ @@ -953,6 +1020,15 @@ def _prop_bnds_root_to_leaf_NamedExpression(node, bnds_dict, feasibility_tol): _prop_bnds_root_to_leaf_map[ObjectiveData] = _prop_bnds_root_to_leaf_NamedExpression _prop_bnds_root_to_leaf_map[ScalarObjective] = _prop_bnds_root_to_leaf_NamedExpression +_prop_bnds_root_to_leaf_map[relational_expr.EqualityExpression] = ( + _prop_bnds_root_to_leaf_equality +) +_prop_bnds_root_to_leaf_map[relational_expr.InequalityExpression] = ( + _prop_bnds_root_to_leaf_inequality +) +_prop_bnds_root_to_leaf_map[relational_expr.RangedExpression] = ( + _prop_bnds_root_to_leaf_ranged +) def _check_and_reset_bounds(var, lb, ub): """ @@ -1250,36 +1326,19 @@ def _fbbt_con(con, config): # a walker to propagate bounds from the variables to the root visitorA = _FBBTVisitorLeafToRoot(bnds_dict, feasibility_tol=config.feasibility_tol) - visitorA.walk_expression(con.body) - - # Now we need to replace the bounds in bnds_dict for the root - # node with the bounds on the constraint (if those bounds are - # better). - _lb = value(con.lower) - _ub = value(con.upper) - if _lb is None: - _lb = -interval.inf - if _ub is None: - _ub = interval.inf + visitorA.walk_expression(con.expr) - lb, ub = bnds_dict[con.body] + always_feasible, feasible = bnds_dict[con.expr] # check if the constraint is infeasible - if lb > _ub + config.feasibility_tol or ub < _lb - config.feasibility_tol: + if not feasible: raise InfeasibleConstraintException( 'Detected an infeasible constraint during FBBT: {0}'.format(str(con)) ) # check if the constraint is always satisfied - if config.deactivate_satisfied_constraints: - if lb >= _lb - config.feasibility_tol and ub <= _ub + config.feasibility_tol: - con.deactivate() - - if _lb > lb: - lb = _lb - if _ub < ub: - ub = _ub - bnds_dict[con.body] = (lb, ub) + if config.deactivate_satisfied_constraints and always_feasible: + con.deactivate() # Now, propagate bounds back from the root to the variables visitorB = _FBBTVisitorRootToLeaf( @@ -1287,7 +1346,7 @@ def _fbbt_con(con, config): integer_tol=config.integer_tol, feasibility_tol=config.feasibility_tol, ) - visitorB.dfs_postorder_stack(con.body) + visitorB.dfs_postorder_stack(con.expr) new_var_bounds = ComponentMap() for _node, _bnds in bnds_dict.items(): diff --git a/pyomo/contrib/fbbt/interval.py b/pyomo/contrib/fbbt/interval.py index a12d1a4529f..b9117961990 100644 --- a/pyomo/contrib/fbbt/interval.py +++ b/pyomo/contrib/fbbt/interval.py @@ -57,7 +57,7 @@ def BoolFlag(val): return _true if val else _false -def ineq(xl, xu, yl, yu): +def ineq(xl, xu, yl, yu, feasibility_tol): """Compute the "bounds" on an InequalityExpression Note this is *not* performing interval arithmetic: we are @@ -67,9 +67,9 @@ def ineq(xl, xu, yl, yu): """ ans = [] - if yl < xu: + if yl < xu - feasibility_tol: ans.append(_false) - if xl <= yu: + if xl <= yu + feasibility_tol: ans.append(_true) assert ans if len(ans) == 1: @@ -77,7 +77,7 @@ def ineq(xl, xu, yl, yu): return tuple(ans) -def eq(xl, xu, yl, yu): +def eq(xl, xu, yl, yu, feasibility_tol): """Compute the "bounds" on an EqualityExpression Note this is *not* performing interval arithmetic: we are @@ -87,9 +87,9 @@ def eq(xl, xu, yl, yu): """ ans = [] - if xl != xu or yl != yu or xl != yl: + if abs(xl - xu) > feasibility_tol or abs(yl - yu) > feasibility_tol or abs(xl - yl) > feasibility_tol: ans.append(_false) - if xl <= yu and yl <= xu: + if xl <= yu + feasibility_tol and yl <= xu + feasibility_tol: ans.append(_true) assert ans if len(ans) == 1: @@ -97,7 +97,7 @@ def eq(xl, xu, yl, yu): return tuple(ans) -def ranged(xl, xu, yl, yu, zl, zu): +def ranged(xl, xu, yl, yu, zl, zu, feasibility_tol): """Compute the "bounds" on a RangedExpression Note this is *not* performing interval arithmetic: we are @@ -106,8 +106,8 @@ def ranged(xl, xu, yl, yu, zl, zu): `z` and `z`, `y` can be outside the range `x` and `z`, or both. """ - lb = ineq(xl, xu, yl, yu) - ub = ineq(yl, yu, zl, zu) + lb = ineq(xl, xu, yl, yu, feasibility_tol) + ub = ineq(yl, yu, zl, zu, feasibility_tol) ans = [] if not lb[0] or not ub[0]: ans.append(_false) From 1b20539f12f4eefc7f2679de1f83f59bff0ebd4a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sat, 8 Jun 2024 17:34:54 -0600 Subject: [PATCH 059/150] Update transformations to not rely on Constraint lower/body/upper --- pyomo/core/plugins/transform/add_slack_vars.py | 13 ++++++++----- pyomo/gdp/plugins/bilinear.py | 7 ++++--- pyomo/gdp/plugins/cuttingplane.py | 6 ++++-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pyomo/core/plugins/transform/add_slack_vars.py b/pyomo/core/plugins/transform/add_slack_vars.py index 39903384729..31c1107d692 100644 --- a/pyomo/core/plugins/transform/add_slack_vars.py +++ b/pyomo/core/plugins/transform/add_slack_vars.py @@ -150,26 +150,29 @@ def _apply_to_impl(self, instance, **kwds): if not cons.active: continue cons_name = cons.getname(fully_qualified=True) - if cons.lower is not None: + lower = cons.lower + body = cons.body + upper = cons.upper + if lower is not None: # we add positive slack variable to body: # declare positive slack varName = "_slack_plus_" + cons_name posSlack = Var(within=NonNegativeReals) xblock.add_component(varName, posSlack) # add positive slack to body expression - cons._body += posSlack + body += posSlack # penalize slack in objective obj_expr += posSlack - if cons.upper is not None: + if upper is not None: # we subtract a positive slack variable from the body: # declare slack varName = "_slack_minus_" + cons_name negSlack = Var(within=NonNegativeReals) xblock.add_component(varName, negSlack) # add negative slack to body expression - cons._body -= negSlack + body -= negSlack # add slack to objective obj_expr += negSlack - + cons.set_value((lower, body, upper)) # make a new objective that minimizes sum of slack variables xblock._slack_objective = Objective(expr=obj_expr) diff --git a/pyomo/gdp/plugins/bilinear.py b/pyomo/gdp/plugins/bilinear.py index 67390801348..70b6e83b52f 100644 --- a/pyomo/gdp/plugins/bilinear.py +++ b/pyomo/gdp/plugins/bilinear.py @@ -77,9 +77,10 @@ def _transformBlock(self, block, instance): for component in block.component_data_objects( Constraint, active=True, descend_into=False ): - expr = self._transformExpression(component.body, instance) - instance.bilinear_data_.c_body[id(component)] = component.body - component._body = expr + lb, body, ub = component.normalize_constraint() + expr = self._transformExpression(body, instance) + instance.bilinear_data_.c_body[id(component)] = body + component.set_value((lb, expr, ub)) def _transformExpression(self, expr, instance): if expr.polynomial_degree() > 2: diff --git a/pyomo/gdp/plugins/cuttingplane.py b/pyomo/gdp/plugins/cuttingplane.py index 6c77a582987..a757f23c826 100644 --- a/pyomo/gdp/plugins/cuttingplane.py +++ b/pyomo/gdp/plugins/cuttingplane.py @@ -400,7 +400,8 @@ def back_off_constraint_with_calculated_cut_violation( val = value(transBlock_rHull.infeasibility_objective) - TOL if val <= 0: logger.info("\tBacking off cut by %s" % val) - cut._body += abs(val) + lb, body, ub = cut.normalize_constraint() + cut.set_value((lb, body + abs(val), ub)) # else there is nothing to do: restore the objective transBlock_rHull.del_component(transBlock_rHull.infeasibility_objective) transBlock_rHull.separation_objective.activate() @@ -424,7 +425,8 @@ def back_off_constraint_by_fixed_tolerance( this callback TOL: An absolute tolerance to be added to make cut more conservative. """ - cut._body += TOL + lb, body, ub = cut.normalize_constraint() + cut.set_value((lb, body + TOL, ub)) @TransformationFactory.register( From 8714e4ca7653da39488a198f7c3271d516b20427 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sat, 8 Jun 2024 17:35:37 -0600 Subject: [PATCH 060/150] Update solver interfaces to not rely on Constraint lower/body/upper --- pyomo/contrib/solver/persistent.py | 37 +++---------------- .../plugins/solvers/persistent_solver.py | 4 +- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/pyomo/contrib/solver/persistent.py b/pyomo/contrib/solver/persistent.py index 71322b7043e..65da81a0c08 100644 --- a/pyomo/contrib/solver/persistent.py +++ b/pyomo/contrib/solver/persistent.py @@ -111,8 +111,8 @@ def add_constraints(self, cons: List[ConstraintData]): raise ValueError( 'constraint {name} has already been added'.format(name=con.name) ) - self._active_constraints[con] = (con.lower, con.body, con.upper) - tmp = collect_vars_and_named_exprs(con.body) + self._active_constraints[con] = con.expr + tmp = collect_vars_and_named_exprs(con.expr) named_exprs, variables, fixed_vars, external_functions = tmp self._check_for_new_vars(variables) self._named_expressions[con] = [(e, e.expr) for e in named_exprs] @@ -417,40 +417,13 @@ def update(self, timer: HierarchicalTimer = None): cons_to_remove_and_add = {} need_to_set_objective = False if config.update_constraints: - cons_to_update = [] - sos_to_update = [] for c in current_cons_dict.keys(): - if c not in new_cons_set: - cons_to_update.append(c) + if c not in new_cons_set and c.expr is not self._active_constraints[c]: + cons_to_remove_and_add[c] = None + sos_to_update = [] for c in current_sos_dict.keys(): if c not in new_sos_set: sos_to_update.append(c) - for c in cons_to_update: - lower, body, upper = self._active_constraints[c] - new_lower, new_body, new_upper = c.lower, c.body, c.upper - if new_body is not body: - cons_to_remove_and_add[c] = None - continue - if new_lower is not lower: - if ( - type(new_lower) is NumericConstant - and type(lower) is NumericConstant - and new_lower.value == lower.value - ): - pass - else: - cons_to_remove_and_add[c] = None - continue - if new_upper is not upper: - if ( - type(new_upper) is NumericConstant - and type(upper) is NumericConstant - and new_upper.value == upper.value - ): - pass - else: - cons_to_remove_and_add[c] = None - continue self.remove_sos_constraints(sos_to_update) self.add_sos_constraints(sos_to_update) timer.stop('cons') diff --git a/pyomo/solvers/plugins/solvers/persistent_solver.py b/pyomo/solvers/plugins/solvers/persistent_solver.py index 3c2a9e52eab..ef96bfa339f 100644 --- a/pyomo/solvers/plugins/solvers/persistent_solver.py +++ b/pyomo/solvers/plugins/solvers/persistent_solver.py @@ -262,7 +262,9 @@ def _add_and_collect_column_data(self, var, obj_coef, constraints, coefficients) coeff_list = list() constr_list = list() for val, c in zip(coefficients, constraints): - c._body += val * var + lb, body, ub = c.normalize_constraint() + body += val * var + c.set_value((lb, body, ub)) self._vars_referenced_by_con[c].add(var) cval = _convert_to_const(val) From d84a8a019c715082fbbe4167b66adb5f87ce113d Mon Sep 17 00:00:00 2001 From: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> Date: Mon, 10 Jun 2024 12:21:52 -0600 Subject: [PATCH 061/150] Update test_SAS.py --- pyomo/solvers/tests/checks/test_SAS.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 00b4d7a9b3e..6dd662bdb21 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -526,7 +526,8 @@ def test_solver_warmstart_capable(self): self.assertTrue(self.opt_sas.warm_start_capable()) -@unittest.skipIf(not sas_available, "The SAS solver is not available") +# @unittest.skipIf(not sas_available, "The SAS solver is not available") +@unittest.skip("MILP94 tests disabled.") class SASTestMILP94(SASTestMILP, unittest.TestCase): pass From 2bfabd5523ec53539da2a91cd86fc2e97cefed01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Magalh=C3=A3es?= Date: Thu, 13 Jun 2024 12:35:33 +0200 Subject: [PATCH 062/150] Fixed bug. Added tests to ensure the functionality is kept in the future. --- examples/pyomo/tutorials/set.dat | 3 + examples/pyomo/tutorials/set.out | 8 +- examples/pyomo/tutorials/set.py | 7 + pyomo/core/base/set.py | 5 +- pyomo/core/tests/unit/test_set.py | 247 ++++++++++++++++++++++++++++++ 5 files changed, 266 insertions(+), 4 deletions(-) diff --git a/examples/pyomo/tutorials/set.dat b/examples/pyomo/tutorials/set.dat index ab0d00b43cc..d136397f54e 100644 --- a/examples/pyomo/tutorials/set.dat +++ b/examples/pyomo/tutorials/set.dat @@ -14,5 +14,8 @@ set M := 1 3; set S[2] := 1 3; set S[5] := 2 3; +set X[2] := 1; +set X[5] := 2 3; + set T[2] := 1 3; set T[5] := 2 3; diff --git a/examples/pyomo/tutorials/set.out b/examples/pyomo/tutorials/set.out index 818977f6155..dd1ef2d4335 100644 --- a/examples/pyomo/tutorials/set.out +++ b/examples/pyomo/tutorials/set.out @@ -1,4 +1,4 @@ -23 Set Declarations +24 Set Declarations A : Size=1, Index=None, Ordered=Insertion Key : Dimen : Domain : Size : Members None : 1 : Any : 3 : {1, 2, 3} @@ -89,5 +89,9 @@ 2 : 1 : Any : 5 : {1, 3, 5, 7, 9} 3 : 1 : Any : 5 : {1, 4, 7, 10, 13} 4 : 1 : Any : 5 : {1, 5, 9, 13, 17} + X : Size=2, Index=B, Ordered=Insertion + Key : Dimen : Domain : Size : Members + 2 : 1 : S[2] : 1 : {1,} + 5 : 1 : S[5] : 2 : {2, 3} -23 Declarations: A B C D E F G H Hsub I J K K_2 L M N O P R S T U V +24 Declarations: A B C D E F G H Hsub I J K K_2 L M N O P R S X T U V diff --git a/examples/pyomo/tutorials/set.py b/examples/pyomo/tutorials/set.py index a14301484c9..c1ea60b48ad 100644 --- a/examples/pyomo/tutorials/set.py +++ b/examples/pyomo/tutorials/set.py @@ -171,6 +171,13 @@ def P_init(model, i, j): # model.S = Set(model.B, within=model.A) +# +# Validation of a set array can also be linked to another set array. If so, the +# elements under each index must also be found under the corresponding index in +# the validation set array: +# +model.X = Set(model.B, within=model.S) + # # Validation of set arrays can also be performed with the _validate_ option. diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 8b7c2a246d6..3f50d032cad 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1932,7 +1932,8 @@ class Set(IndexedComponent): within : initialiser(set), optional A set that defines the valid values that can be contained - in this set + in this set. If the latter is indexed, the former can be indexed or + non-indexed, in which case it applies to all indices. domain : initializer(set), optional A set that defines the valid values that can be contained in this set @@ -2217,7 +2218,7 @@ def _getitem_when_not_present(self, index): _d = None domain = self._init_domain(_block, index, self) - if domain is not None: + if domain is not None and hasattr(domain, "construct"): domain.construct() if _d is UnknownSetDimen and domain is not None and domain.dimen is not None: _d = domain.dimen diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index f62589a6873..90e629958d8 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -4543,9 +4543,11 @@ def test_construction(self): m.I = Set(initialize=[1, 2, 3]) m.J = Set(initialize=[4, 5, 6]) m.K = Set(initialize=[(1, 4), (2, 6), (3, 5)], within=m.I * m.J) + m.L = Set(initialize=[1, 3], within=m.I) m.II = Set([1, 2, 3], initialize={1: [0], 2: [1, 2], 3: range(3)}) m.JJ = Set([1, 2, 3], initialize={1: [0], 2: [1, 2], 3: range(3)}) m.KK = Set([1, 2], initialize=[], dimen=lambda m, i: i) + m.LL = Set([2, 3], within=m.II, initialize={2: [1, 2], 3: [1]}) output = StringIO() m.I.pprint(ostream=output) @@ -4569,6 +4571,8 @@ def test_construction(self): 'I': [-1, 0], 'II': {1: [10, 11], 3: [30]}, 'K': [-1, 4, -1, 6, 0, 5], + 'L': [-1], + 'LL': {3: [30]}, } } ) @@ -4576,6 +4580,7 @@ def test_construction(self): self.assertEqual(list(i.I), [-1, 0]) self.assertEqual(list(i.J), [4, 5, 6]) self.assertEqual(list(i.K), [(-1, 4), (-1, 6), (0, 5)]) + self.assertEqual(list(i.L), [-1]) self.assertEqual(list(i.II[1]), [10, 11]) self.assertEqual(list(i.II[3]), [30]) self.assertEqual(list(i.JJ[1]), [0]) @@ -4583,9 +4588,11 @@ def test_construction(self): self.assertEqual(list(i.JJ[3]), [0, 1, 2]) self.assertEqual(list(i.KK[1]), []) self.assertEqual(list(i.KK[2]), []) + self.assertEqual(list(i.LL[3]), [30]) # Implicitly-constructed set should fall back on initialize! self.assertEqual(list(i.II[2]), [1, 2]) + self.assertEqual(list(i.LL[2]), [1, 2]) # Additional tests for tuplize: i = m.create_instance(data={None: {'K': [(1, 4), (2, 6)], 'KK': [1, 4, 2, 6]}}) @@ -6388,3 +6395,243 @@ def test_issue_1112(self): self.assertEqual(len(vals), 1) self.assertIsInstance(vals[0], SetProduct_OrderedSet) self.assertIsNot(vals[0], cross) + + def test_issue_3284(self): + # test creating (indexed and non-indexed) sets using the within argument + # using concrete model and initialization + problem = ConcreteModel() + # non-indexed sets not using the within argument + problem.A = Set(initialize=[1,2,3]) + problem.B = Set(dimen=2, initialize=[(1,2),(3,4),(5,6)]) + # non-indexed sets using within argument + problem.subset_A = Set(within=problem.A, initialize=[2,3]) + problem.subset_B = Set(within=problem.B, dimen=2, initialize=[(1,2),(5,6)]) + # indexed sets not using the within argument + problem.C = Set(problem.A, initialize={1:[-1,3], 2:[4,7], 3:[3, 8]}) + problem.D = Set(problem.B, initialize={(1,2): [1,5], (3,4): [3], (5,6): [6,8,9]}) + # indexed sets using an indexed set for the within argument + problem.subset_C = Set(problem.A, within=problem.C, initialize={1:[-1], 2:[4], 3:[3, 8]}) + problem.subset_D = Set(problem.B, within=problem.D, initialize={(1,2): [1,5], (3,4): [], (5,6): [6]}) + # indexed sets using a non-indexed set for the within argument + problem.E = Set([0, 1], within=problem.A, initialize={0:[1, 2], 1:[3]}) + problem.F = Set([(1,2,3), (4,5,6)], within=problem.B, initialize={(1,2,3):[(1,2)], (4,5,6):[(3,4)]}) + # check them + self.assertEqual(list(problem.A), [1,2,3]) + self.assertEqual(list(problem.B), [(1,2),(3,4),(5,6)]) + self.assertEqual(list(problem.subset_A), [2, 3]) + self.assertEqual(list(problem.subset_B), [(1,2),(5,6)]) + self.assertEqual(list(problem.C[1]), [-1, 3]) + self.assertEqual(list(problem.C[2]), [4, 7]) + self.assertEqual(list(problem.C[3]), [3, 8]) + self.assertEqual(list(problem.D[(1,2)]), [1,5]) + self.assertEqual(list(problem.D[(3,4)]), [3]) + self.assertEqual(list(problem.D[(5,6)]), [6,8,9]) + self.assertEqual(list(problem.subset_C[1]), [-1]) + self.assertEqual(list(problem.subset_C[2]), [4]) + self.assertEqual(list(problem.subset_C[3]), [3, 8]) + self.assertEqual(list(problem.subset_D[(1,2)]), [1,5]) + self.assertEqual(list(problem.subset_D[(3,4)]), []) + self.assertEqual(list(problem.subset_D[(5,6)]), [6]) + self.assertEqual(list(problem.E[0]), [1,2]) + self.assertEqual(list(problem.E[1]), [3]) + self.assertEqual(list(problem.F[(1,2,3)]), [(1,2)]) + self.assertEqual(list(problem.F[(4,5,6)]), [(3,4)]) + + # try adding elements to test the domains (1 compatible, 1 incompatible) + # set subset_A + problem.subset_A.add(1) + error_raised = False + try: + problem.subset_A.add(4) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_B + problem.subset_B.add((3,4)) + error_raised = False + try: + problem.subset_B.add((7,8)) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_C + problem.subset_C[2].add(7) + error_raised = False + try: + problem.subset_C[2].add(8) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_D + problem.subset_D[(3,4)].add(3) + error_raised = False + try: + problem.subset_D[(3,4)].add(4) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set E + problem.E[1].add(2) + error_raised = False + try: + problem.E[1].add(4) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set F + problem.F[(1,2,3)].add((3,4)) + error_raised = False + try: + problem.F[(4,5,6)].add((4,3)) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # check them + self.assertEqual(list(problem.A), [1,2,3]) + self.assertEqual(list(problem.B), [(1,2),(3,4),(5,6)]) + self.assertEqual(list(problem.subset_A), [2, 3, 1]) + self.assertEqual(list(problem.subset_B), [(1,2),(5,6),(3,4)]) + self.assertEqual(list(problem.C[1]), [-1, 3]) + self.assertEqual(list(problem.C[2]), [4, 7]) + self.assertEqual(list(problem.C[3]), [3, 8]) + self.assertEqual(list(problem.D[(1,2)]), [1,5]) + self.assertEqual(list(problem.D[(3,4)]), [3]) + self.assertEqual(list(problem.D[(5,6)]), [6,8,9]) + self.assertEqual(list(problem.subset_C[1]), [-1]) + self.assertEqual(list(problem.subset_C[2]), [4, 7]) + self.assertEqual(list(problem.subset_C[3]), [3, 8]) + self.assertEqual(list(problem.subset_D[(1,2)]), [1,5]) + self.assertEqual(list(problem.subset_D[(3,4)]), [3]) + self.assertEqual(list(problem.subset_D[(5,6)]), [6]) + self.assertEqual(list(problem.E[0]), [1,2]) + self.assertEqual(list(problem.E[1]), [3,2]) + self.assertEqual(list(problem.F[(1,2,3)]), [(1,2),(3,4)]) + self.assertEqual(list(problem.F[(4,5,6)]), [(3,4)]) + + + # using abstract model and no initialization + model = AbstractModel() + # non-indexed sets not using the within argument + model.A = Set() + model.B = Set(dimen=2) + # non-indexed sets using within argument + model.subset_A = Set(within=model.A) + model.subset_B = Set(within=model.B, dimen=2) + # indexed sets not using the within argument + model.C = Set(model.A) + model.D = Set(model.B) + # indexed sets using an indexed set for the within argument + model.subset_C = Set(model.A, within=model.C) + model.subset_D = Set(model.B, within=model.D) + # indexed sets using a non-indexed set for the within argument + model.E_index = Set() + model.F_index = Set() + model.E = Set(model.E_index, within=model.A) + model.F = Set(model.F_index, within=model.B) + problem = model.create_instance( + data={ + None: { + 'A': [3, 4, 5], + 'B': [(1,2),(7,8)], + 'subset_A': [3, 4], + 'subset_B': [(1,2)], + 'C': {3: [3], 4: [4,8], 5: [5,6]}, + 'D': {(1,2): [2], (7,8): [0, 1]}, + 'subset_C': {3: [3], 4: [8], 5: []}, + 'subset_D': {(1,2): [], (7,8): [0, 1]}, + 'E_index': [0, 1], + 'F_index': [(1,2,3), (4,5,6)], + 'E': {0:[3, 4], 1:[5]}, + 'F': {(1,2,3):[(1,2)], (4,5,6):[(7,8)]}, + } + } + ) + + # check them + self.assertEqual(list(problem.A), [3, 4, 5]) + self.assertEqual(list(problem.B), [(1,2),(7,8)]) + self.assertEqual(list(problem.subset_A), [3, 4]) + self.assertEqual(list(problem.subset_B), [(1,2)]) + self.assertEqual(list(problem.C[3]), [3]) + self.assertEqual(list(problem.C[4]), [4, 8]) + self.assertEqual(list(problem.C[5]), [5, 6]) + self.assertEqual(list(problem.D[(1,2)]), [2]) + self.assertEqual(list(problem.D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.subset_C[3]), [3]) + self.assertEqual(list(problem.subset_C[4]), [8]) + self.assertEqual(list(problem.subset_C[5]), []) + self.assertEqual(list(problem.subset_D[(1,2)]), []) + self.assertEqual(list(problem.subset_D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3,4]) + self.assertEqual(list(problem.E[1]), [5]) + self.assertEqual(list(problem.F[(1,2,3)]), [(1,2)]) + self.assertEqual(list(problem.F[(4,5,6)]), [(7,8)]) + + # try adding elements to test the domains (1 compatible, 1 incompatible) + # set subset_A + problem.subset_A.add(5) + error_raised = False + try: + problem.subset_A.add(6) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_B + problem.subset_B.add((7,8)) + error_raised = False + try: + problem.subset_B.add((3,4)) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_C + problem.subset_C[4].add(4) + error_raised = False + try: + problem.subset_C[4].add(9) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_D + problem.subset_D[(1,2)].add(2) + error_raised = False + try: + problem.subset_D[(1,2)].add(3) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set E + problem.E[1].add(4) + error_raised = False + try: + problem.E[1].add(1) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set F + problem.F[(1,2,3)].add((7,8)) + error_raised = False + try: + problem.F[(4,5,6)].add((4,3)) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # check them + self.assertEqual(list(problem.A), [3, 4, 5]) + self.assertEqual(list(problem.B), [(1,2),(7,8)]) + self.assertEqual(list(problem.subset_A), [3, 4, 5]) + self.assertEqual(list(problem.subset_B), [(1,2),(7,8)]) + self.assertEqual(list(problem.C[3]), [3]) + self.assertEqual(list(problem.C[4]), [4, 8]) + self.assertEqual(list(problem.C[5]), [5, 6]) + self.assertEqual(list(problem.D[(1,2)]), [2]) + self.assertEqual(list(problem.D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.subset_C[3]), [3]) + self.assertEqual(list(problem.subset_C[4]), [8, 4]) + self.assertEqual(list(problem.subset_C[5]), []) + self.assertEqual(list(problem.subset_D[(1,2)]), [2]) + self.assertEqual(list(problem.subset_D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3,4]) + self.assertEqual(list(problem.E[1]), [5,4]) + self.assertEqual(list(problem.F[(1,2,3)]), [(1,2),(7,8)]) + self.assertEqual(list(problem.F[(4,5,6)]), [(7,8)]) From 2fcfd13c2c0c73b09c0bc5ef3b096ca4d92f6f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Magalh=C3=A3es?= Date: Sat, 15 Jun 2024 20:26:59 +0200 Subject: [PATCH 063/150] Blackened. --- pyomo/core/base/set.py | 2 +- pyomo/core/tests/unit/test_set.py | 171 ++++++++++++++++-------------- 2 files changed, 92 insertions(+), 81 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 3f50d032cad..666332124f1 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1932,7 +1932,7 @@ class Set(IndexedComponent): within : initialiser(set), optional A set that defines the valid values that can be contained - in this set. If the latter is indexed, the former can be indexed or + in this set. If the latter is indexed, the former can be indexed or non-indexed, in which case it applies to all indices. domain : initializer(set), optional A set that defines the valid values that can be contained diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 90e629958d8..a1c502a86d0 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6395,48 +6395,60 @@ def test_issue_1112(self): self.assertEqual(len(vals), 1) self.assertIsInstance(vals[0], SetProduct_OrderedSet) self.assertIsNot(vals[0], cross) - + def test_issue_3284(self): # test creating (indexed and non-indexed) sets using the within argument # using concrete model and initialization problem = ConcreteModel() # non-indexed sets not using the within argument - problem.A = Set(initialize=[1,2,3]) - problem.B = Set(dimen=2, initialize=[(1,2),(3,4),(5,6)]) + problem.A = Set(initialize=[1, 2, 3]) + problem.B = Set(dimen=2, initialize=[(1, 2), (3, 4), (5, 6)]) # non-indexed sets using within argument - problem.subset_A = Set(within=problem.A, initialize=[2,3]) - problem.subset_B = Set(within=problem.B, dimen=2, initialize=[(1,2),(5,6)]) + problem.subset_A = Set(within=problem.A, initialize=[2, 3]) + problem.subset_B = Set(within=problem.B, dimen=2, initialize=[(1, 2), (5, 6)]) # indexed sets not using the within argument - problem.C = Set(problem.A, initialize={1:[-1,3], 2:[4,7], 3:[3, 8]}) - problem.D = Set(problem.B, initialize={(1,2): [1,5], (3,4): [3], (5,6): [6,8,9]}) + problem.C = Set(problem.A, initialize={1: [-1, 3], 2: [4, 7], 3: [3, 8]}) + problem.D = Set( + problem.B, initialize={(1, 2): [1, 5], (3, 4): [3], (5, 6): [6, 8, 9]} + ) # indexed sets using an indexed set for the within argument - problem.subset_C = Set(problem.A, within=problem.C, initialize={1:[-1], 2:[4], 3:[3, 8]}) - problem.subset_D = Set(problem.B, within=problem.D, initialize={(1,2): [1,5], (3,4): [], (5,6): [6]}) + problem.subset_C = Set( + problem.A, within=problem.C, initialize={1: [-1], 2: [4], 3: [3, 8]} + ) + problem.subset_D = Set( + problem.B, + within=problem.D, + initialize={(1, 2): [1, 5], (3, 4): [], (5, 6): [6]}, + ) # indexed sets using a non-indexed set for the within argument - problem.E = Set([0, 1], within=problem.A, initialize={0:[1, 2], 1:[3]}) - problem.F = Set([(1,2,3), (4,5,6)], within=problem.B, initialize={(1,2,3):[(1,2)], (4,5,6):[(3,4)]}) + problem.E = Set([0, 1], within=problem.A, initialize={0: [1, 2], 1: [3]}) + problem.F = Set( + [(1, 2, 3), (4, 5, 6)], + within=problem.B, + initialize={(1, 2, 3): [(1, 2)], (4, 5, 6): [(3, 4)]}, + ) # check them - self.assertEqual(list(problem.A), [1,2,3]) - self.assertEqual(list(problem.B), [(1,2),(3,4),(5,6)]) + self.assertEqual(list(problem.A), [1, 2, 3]) + self.assertEqual(list(problem.B), [(1, 2), (3, 4), (5, 6)]) self.assertEqual(list(problem.subset_A), [2, 3]) - self.assertEqual(list(problem.subset_B), [(1,2),(5,6)]) + self.assertEqual(list(problem.subset_B), [(1, 2), (5, 6)]) self.assertEqual(list(problem.C[1]), [-1, 3]) self.assertEqual(list(problem.C[2]), [4, 7]) self.assertEqual(list(problem.C[3]), [3, 8]) - self.assertEqual(list(problem.D[(1,2)]), [1,5]) - self.assertEqual(list(problem.D[(3,4)]), [3]) - self.assertEqual(list(problem.D[(5,6)]), [6,8,9]) + self.assertEqual(list(problem.D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.D[(3, 4)]), [3]) + self.assertEqual(list(problem.D[(5, 6)]), [6, 8, 9]) self.assertEqual(list(problem.subset_C[1]), [-1]) self.assertEqual(list(problem.subset_C[2]), [4]) self.assertEqual(list(problem.subset_C[3]), [3, 8]) - self.assertEqual(list(problem.subset_D[(1,2)]), [1,5]) - self.assertEqual(list(problem.subset_D[(3,4)]), []) - self.assertEqual(list(problem.subset_D[(5,6)]), [6]) - self.assertEqual(list(problem.E[0]), [1,2]) + self.assertEqual(list(problem.subset_D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.subset_D[(3, 4)]), []) + self.assertEqual(list(problem.subset_D[(5, 6)]), [6]) + self.assertEqual(list(problem.E[0]), [1, 2]) self.assertEqual(list(problem.E[1]), [3]) - self.assertEqual(list(problem.F[(1,2,3)]), [(1,2)]) - self.assertEqual(list(problem.F[(4,5,6)]), [(3,4)]) - + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(3, 4)]) + # try adding elements to test the domains (1 compatible, 1 incompatible) # set subset_A problem.subset_A.add(1) @@ -6447,10 +6459,10 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set subset_B - problem.subset_B.add((3,4)) + problem.subset_B.add((3, 4)) error_raised = False try: - problem.subset_B.add((7,8)) + problem.subset_B.add((7, 8)) except ValueError: error_raised = True self.assertEqual(error_raised, True) @@ -6463,10 +6475,10 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set subset_D - problem.subset_D[(3,4)].add(3) + problem.subset_D[(3, 4)].add(3) error_raised = False try: - problem.subset_D[(3,4)].add(4) + problem.subset_D[(3, 4)].add(4) except ValueError: error_raised = True self.assertEqual(error_raised, True) @@ -6479,36 +6491,35 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set F - problem.F[(1,2,3)].add((3,4)) + problem.F[(1, 2, 3)].add((3, 4)) error_raised = False try: - problem.F[(4,5,6)].add((4,3)) + problem.F[(4, 5, 6)].add((4, 3)) except ValueError: error_raised = True self.assertEqual(error_raised, True) # check them - self.assertEqual(list(problem.A), [1,2,3]) - self.assertEqual(list(problem.B), [(1,2),(3,4),(5,6)]) + self.assertEqual(list(problem.A), [1, 2, 3]) + self.assertEqual(list(problem.B), [(1, 2), (3, 4), (5, 6)]) self.assertEqual(list(problem.subset_A), [2, 3, 1]) - self.assertEqual(list(problem.subset_B), [(1,2),(5,6),(3,4)]) + self.assertEqual(list(problem.subset_B), [(1, 2), (5, 6), (3, 4)]) self.assertEqual(list(problem.C[1]), [-1, 3]) self.assertEqual(list(problem.C[2]), [4, 7]) self.assertEqual(list(problem.C[3]), [3, 8]) - self.assertEqual(list(problem.D[(1,2)]), [1,5]) - self.assertEqual(list(problem.D[(3,4)]), [3]) - self.assertEqual(list(problem.D[(5,6)]), [6,8,9]) + self.assertEqual(list(problem.D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.D[(3, 4)]), [3]) + self.assertEqual(list(problem.D[(5, 6)]), [6, 8, 9]) self.assertEqual(list(problem.subset_C[1]), [-1]) self.assertEqual(list(problem.subset_C[2]), [4, 7]) self.assertEqual(list(problem.subset_C[3]), [3, 8]) - self.assertEqual(list(problem.subset_D[(1,2)]), [1,5]) - self.assertEqual(list(problem.subset_D[(3,4)]), [3]) - self.assertEqual(list(problem.subset_D[(5,6)]), [6]) - self.assertEqual(list(problem.E[0]), [1,2]) - self.assertEqual(list(problem.E[1]), [3,2]) - self.assertEqual(list(problem.F[(1,2,3)]), [(1,2),(3,4)]) - self.assertEqual(list(problem.F[(4,5,6)]), [(3,4)]) - - + self.assertEqual(list(problem.subset_D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.subset_D[(3, 4)]), [3]) + self.assertEqual(list(problem.subset_D[(5, 6)]), [6]) + self.assertEqual(list(problem.E[0]), [1, 2]) + self.assertEqual(list(problem.E[1]), [3, 2]) + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2), (3, 4)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(3, 4)]) + # using abstract model and no initialization model = AbstractModel() # non-indexed sets not using the within argument @@ -6532,41 +6543,41 @@ def test_issue_3284(self): data={ None: { 'A': [3, 4, 5], - 'B': [(1,2),(7,8)], + 'B': [(1, 2), (7, 8)], 'subset_A': [3, 4], - 'subset_B': [(1,2)], - 'C': {3: [3], 4: [4,8], 5: [5,6]}, - 'D': {(1,2): [2], (7,8): [0, 1]}, + 'subset_B': [(1, 2)], + 'C': {3: [3], 4: [4, 8], 5: [5, 6]}, + 'D': {(1, 2): [2], (7, 8): [0, 1]}, 'subset_C': {3: [3], 4: [8], 5: []}, - 'subset_D': {(1,2): [], (7,8): [0, 1]}, + 'subset_D': {(1, 2): [], (7, 8): [0, 1]}, 'E_index': [0, 1], - 'F_index': [(1,2,3), (4,5,6)], - 'E': {0:[3, 4], 1:[5]}, - 'F': {(1,2,3):[(1,2)], (4,5,6):[(7,8)]}, + 'F_index': [(1, 2, 3), (4, 5, 6)], + 'E': {0: [3, 4], 1: [5]}, + 'F': {(1, 2, 3): [(1, 2)], (4, 5, 6): [(7, 8)]}, } } ) - + # check them self.assertEqual(list(problem.A), [3, 4, 5]) - self.assertEqual(list(problem.B), [(1,2),(7,8)]) + self.assertEqual(list(problem.B), [(1, 2), (7, 8)]) self.assertEqual(list(problem.subset_A), [3, 4]) - self.assertEqual(list(problem.subset_B), [(1,2)]) + self.assertEqual(list(problem.subset_B), [(1, 2)]) self.assertEqual(list(problem.C[3]), [3]) self.assertEqual(list(problem.C[4]), [4, 8]) self.assertEqual(list(problem.C[5]), [5, 6]) - self.assertEqual(list(problem.D[(1,2)]), [2]) - self.assertEqual(list(problem.D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.D[(1, 2)]), [2]) + self.assertEqual(list(problem.D[(7, 8)]), [0, 1]) self.assertEqual(list(problem.subset_C[3]), [3]) self.assertEqual(list(problem.subset_C[4]), [8]) self.assertEqual(list(problem.subset_C[5]), []) - self.assertEqual(list(problem.subset_D[(1,2)]), []) - self.assertEqual(list(problem.subset_D[(7,8)]), [0, 1]) - self.assertEqual(list(problem.E[0]), [3,4]) + self.assertEqual(list(problem.subset_D[(1, 2)]), []) + self.assertEqual(list(problem.subset_D[(7, 8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3, 4]) self.assertEqual(list(problem.E[1]), [5]) - self.assertEqual(list(problem.F[(1,2,3)]), [(1,2)]) - self.assertEqual(list(problem.F[(4,5,6)]), [(7,8)]) - + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(7, 8)]) + # try adding elements to test the domains (1 compatible, 1 incompatible) # set subset_A problem.subset_A.add(5) @@ -6577,10 +6588,10 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set subset_B - problem.subset_B.add((7,8)) + problem.subset_B.add((7, 8)) error_raised = False try: - problem.subset_B.add((3,4)) + problem.subset_B.add((3, 4)) except ValueError: error_raised = True self.assertEqual(error_raised, True) @@ -6593,10 +6604,10 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set subset_D - problem.subset_D[(1,2)].add(2) + problem.subset_D[(1, 2)].add(2) error_raised = False try: - problem.subset_D[(1,2)].add(3) + problem.subset_D[(1, 2)].add(3) except ValueError: error_raised = True self.assertEqual(error_raised, True) @@ -6609,29 +6620,29 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set F - problem.F[(1,2,3)].add((7,8)) + problem.F[(1, 2, 3)].add((7, 8)) error_raised = False try: - problem.F[(4,5,6)].add((4,3)) + problem.F[(4, 5, 6)].add((4, 3)) except ValueError: error_raised = True self.assertEqual(error_raised, True) # check them self.assertEqual(list(problem.A), [3, 4, 5]) - self.assertEqual(list(problem.B), [(1,2),(7,8)]) + self.assertEqual(list(problem.B), [(1, 2), (7, 8)]) self.assertEqual(list(problem.subset_A), [3, 4, 5]) - self.assertEqual(list(problem.subset_B), [(1,2),(7,8)]) + self.assertEqual(list(problem.subset_B), [(1, 2), (7, 8)]) self.assertEqual(list(problem.C[3]), [3]) self.assertEqual(list(problem.C[4]), [4, 8]) self.assertEqual(list(problem.C[5]), [5, 6]) - self.assertEqual(list(problem.D[(1,2)]), [2]) - self.assertEqual(list(problem.D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.D[(1, 2)]), [2]) + self.assertEqual(list(problem.D[(7, 8)]), [0, 1]) self.assertEqual(list(problem.subset_C[3]), [3]) self.assertEqual(list(problem.subset_C[4]), [8, 4]) self.assertEqual(list(problem.subset_C[5]), []) - self.assertEqual(list(problem.subset_D[(1,2)]), [2]) - self.assertEqual(list(problem.subset_D[(7,8)]), [0, 1]) - self.assertEqual(list(problem.E[0]), [3,4]) - self.assertEqual(list(problem.E[1]), [5,4]) - self.assertEqual(list(problem.F[(1,2,3)]), [(1,2),(7,8)]) - self.assertEqual(list(problem.F[(4,5,6)]), [(7,8)]) + self.assertEqual(list(problem.subset_D[(1, 2)]), [2]) + self.assertEqual(list(problem.subset_D[(7, 8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3, 4]) + self.assertEqual(list(problem.E[1]), [5, 4]) + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2), (7, 8)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(7, 8)]) From 7aee9e04c907cae3c936954110be1e664a736160 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 16 Jun 2024 14:36:28 -0600 Subject: [PATCH 064/150] NFC: apply black --- pyomo/contrib/fbbt/fbbt.py | 4 ++++ pyomo/contrib/fbbt/interval.py | 6 +++++- pyomo/core/tests/unit/test_con.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/fbbt/fbbt.py b/pyomo/contrib/fbbt/fbbt.py index 62cd90d9c87..593d875ca6f 100644 --- a/pyomo/contrib/fbbt/fbbt.py +++ b/pyomo/contrib/fbbt/fbbt.py @@ -87,18 +87,21 @@ def _prop_bnds_leaf_to_root_equality(visitor, node, arg1, arg2): *bnds_dict[arg1], *bnds_dict[arg2], visitor.feasibility_tol ) + def _prop_bnds_leaf_to_root_inequality(visitor, node, arg1, arg2): bnds_dict = visitor.bnds_dict bnds_dict[node] = interval.ineq( *bnds_dict[arg1], *bnds_dict[arg2], visitor.feasibility_tol ) + def _prop_bnds_leaf_to_root_ranged(visitor, node, arg1, arg2, arg3): bnds_dict = visitor.bnds_dict bnds_dict[node] = interval.ranged( *bnds_dict[arg1], *bnds_dict[arg2], *bnds_dict[arg3], visitor.feasibility_tol ) + def _prop_bnds_leaf_to_root_ProductExpression(visitor, node, arg1, arg2): """ @@ -1030,6 +1033,7 @@ def _prop_bnds_root_to_leaf_NamedExpression(node, bnds_dict, feasibility_tol): _prop_bnds_root_to_leaf_ranged ) + def _check_and_reset_bounds(var, lb, ub): """ This function ensures that lb is not less than var.lb and that ub is not greater than var.ub. diff --git a/pyomo/contrib/fbbt/interval.py b/pyomo/contrib/fbbt/interval.py index b9117961990..4b93d6e3f31 100644 --- a/pyomo/contrib/fbbt/interval.py +++ b/pyomo/contrib/fbbt/interval.py @@ -87,7 +87,11 @@ def eq(xl, xu, yl, yu, feasibility_tol): """ ans = [] - if abs(xl - xu) > feasibility_tol or abs(yl - yu) > feasibility_tol or abs(xl - yl) > feasibility_tol: + if ( + abs(xl - xu) > feasibility_tol + or abs(yl - yu) > feasibility_tol + or abs(xl - yl) > feasibility_tol + ): ans.append(_false) if xl <= yu + feasibility_tol and yl <= xu + feasibility_tol: ans.append(_true) diff --git a/pyomo/core/tests/unit/test_con.py b/pyomo/core/tests/unit/test_con.py index 7274adae397..07c7eb3af8e 100644 --- a/pyomo/core/tests/unit/test_con.py +++ b/pyomo/core/tests/unit/test_con.py @@ -1630,6 +1630,7 @@ def rule1(model): "Constraint 'c' is a Ranged Inequality with a variable lower bound", ): instance.c.upper + # def rule1(model): return (0, model.x, model.z) @@ -1651,7 +1652,6 @@ def rule1(model): ): instance.c.upper - def test_expression_constructor_coverage(self): def rule1(model): expr = model.x From 3d77bc9b400600669d8445f2c1d9ee36779e6fb1 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 16 Jun 2024 23:10:21 -0600 Subject: [PATCH 065/150] Switch from caching .body to caching .expr --- pyomo/contrib/appsi/base.py | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/pyomo/contrib/appsi/base.py b/pyomo/contrib/appsi/base.py index 6d2b5ccfcd4..9c7da1eb60b 100644 --- a/pyomo/contrib/appsi/base.py +++ b/pyomo/contrib/appsi/base.py @@ -1007,7 +1007,7 @@ def add_constraints(self, cons: List[ConstraintData]): raise ValueError( 'constraint {name} has already been added'.format(name=con.name) ) - self._active_constraints[con] = (con.lower, con.body, con.upper) + self._active_constraints[con] = con.expr if self.use_extensions and cmodel_available: tmp = cmodel.prep_for_repn(con.body, self._expr_types) else: @@ -1363,40 +1363,13 @@ def update(self, timer: HierarchicalTimer = None): cons_to_remove_and_add = dict() need_to_set_objective = False if config.update_constraints: - cons_to_update = list() - sos_to_update = list() for c in current_cons_dict.keys(): - if c not in new_cons_set: - cons_to_update.append(c) + if c not in new_cons_set and c.expr is not self._active_constraints[c]: + cons_to_remove_and_add[c] = None + sos_to_update = [] for c in current_sos_dict.keys(): if c not in new_sos_set: sos_to_update.append(c) - for c in cons_to_update: - lower, body, upper = self._active_constraints[c] - new_lower, new_body, new_upper = c.lower, c.body, c.upper - if new_body is not body: - cons_to_remove_and_add[c] = None - continue - if new_lower is not lower: - if ( - type(new_lower) is NumericConstant - and type(lower) is NumericConstant - and new_lower.value == lower.value - ): - pass - else: - cons_to_remove_and_add[c] = None - continue - if new_upper is not upper: - if ( - type(new_upper) is NumericConstant - and type(upper) is NumericConstant - and new_upper.value == upper.value - ): - pass - else: - cons_to_remove_and_add[c] = None - continue self.remove_sos_constraints(sos_to_update) self.add_sos_constraints(sos_to_update) timer.stop('cons') From d8de8100bb08124173e3ab125555a3c8785e7ad2 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 17 Jun 2024 05:15:44 -0600 Subject: [PATCH 066/150] Update APPSI cmodel to call normalize_constraint() --- pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp | 2 +- pyomo/contrib/appsi/cmodel/src/lp_writer.cpp | 2 +- pyomo/contrib/appsi/cmodel/src/nl_writer.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp b/pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp index bd8d7dbf854..708cfd9e073 100644 --- a/pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp +++ b/pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp @@ -205,7 +205,7 @@ void process_fbbt_constraints(FBBTModel *model, PyomoExprTypes &expr_types, py::handle con_body; for (py::handle c : cons) { - lower_body_upper = active_constraints[c]; + lower_body_upper = c.attr("normalize_constraint")(); con_lb = lower_body_upper[0]; con_body = lower_body_upper[1]; con_ub = lower_body_upper[2]; diff --git a/pyomo/contrib/appsi/cmodel/src/lp_writer.cpp b/pyomo/contrib/appsi/cmodel/src/lp_writer.cpp index 68baf2b8ae8..996bb34f564 100644 --- a/pyomo/contrib/appsi/cmodel/src/lp_writer.cpp +++ b/pyomo/contrib/appsi/cmodel/src/lp_writer.cpp @@ -289,7 +289,7 @@ void process_lp_constraints(py::list cons, py::object writer) { py::object nonlinear_expr; PyomoExprTypes expr_types = PyomoExprTypes(); for (py::handle c : cons) { - lower_body_upper = active_constraints[c]; + lower_body_upper = c.attr("normalize_constraint")(); cname = getSymbol(c, labeler); repn = generate_standard_repn( lower_body_upper[1], "compute_values"_a = false, "quadratic"_a = true); diff --git a/pyomo/contrib/appsi/cmodel/src/nl_writer.cpp b/pyomo/contrib/appsi/cmodel/src/nl_writer.cpp index 8de6cc74ab4..477bdd87aee 100644 --- a/pyomo/contrib/appsi/cmodel/src/nl_writer.cpp +++ b/pyomo/contrib/appsi/cmodel/src/nl_writer.cpp @@ -527,7 +527,7 @@ void process_nl_constraints(NLWriter *nl_writer, PyomoExprTypes &expr_types, py::handle repn_nonlinear_expr; for (py::handle c : cons) { - lower_body_upper = active_constraints[c]; + lower_body_upper = c.attr("normalize_constraint")(); repn = generate_standard_repn( lower_body_upper[1], "compute_values"_a = false, "quadratic"_a = false); _const = appsi_expr_from_pyomo_expr(repn.attr("constant"), var_map, From 496a6638f93041055dff951b6a7ec1d94dd44d6a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 17 Jun 2024 08:06:07 -0600 Subject: [PATCH 067/150] Remove repeated function --- pyomo/contrib/fbbt/fbbt.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pyomo/contrib/fbbt/fbbt.py b/pyomo/contrib/fbbt/fbbt.py index 593d875ca6f..39df91675a8 100644 --- a/pyomo/contrib/fbbt/fbbt.py +++ b/pyomo/contrib/fbbt/fbbt.py @@ -437,14 +437,6 @@ def _prop_bnds_root_to_leaf_ranged(node, bnds_dict, feasibility_tol): bnds_dict[arg1] = lb1, ub2 -def _prop_bnds_root_to_leaf_equality(node, bnds_dict, feasibility_tol): - assert bnds_dict[node][1] # This expression is feasible - arg1, arg2 = node.args - lb1, ub1 = bnds_dict[arg1] - lb2, ub2 = bnds_dict[arg2] - bnds_dict[arg1] = bnds_dict[arg2] = max(lb1, lb2), min(ub1, ub2) - - def _prop_bnds_root_to_leaf_ProductExpression(node, bnds_dict, feasibility_tol): """ From 0b0833f1149467ecf2c0c9c2539b9ca08e5fa602 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 17 Jun 2024 08:06:22 -0600 Subject: [PATCH 068/150] Switch logic for mapping lower/body/upper to match previous behavior --- pyomo/core/base/constraint.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pyomo/core/base/constraint.py b/pyomo/core/base/constraint.py index b3846b44a0a..3ee5a82ef58 100644 --- a/pyomo/core/base/constraint.py +++ b/pyomo/core/base/constraint.py @@ -223,7 +223,13 @@ def body(self): _, ans, _ = self._expr.args else: raise - if ans.__class__ in native_numeric_types: + if ans.__class__ in native_types and ans is not None: + # Historically, constraint.lower was guaranteed to return a type + # derived from Pyomo NumericValue (or None). Replicate that. + # + # [JDS 6/2024: it would be nice to remove this behavior, + # although possibly unnecessary, as people should use + # normalize_constraint() instead] return as_numeric(ans) return ans @@ -231,7 +237,7 @@ def body(self): def lower(self): """Access the lower bound of a constraint expression.""" ans = self.normalize_constraint()[0] - if ans.__class__ in native_numeric_types: + if ans.__class__ in native_types and ans is not None: # Historically, constraint.lower was guaranteed to return a type # derived from Pyomo NumericValue (or None). Replicate that # functionality, although clients should in almost all cases @@ -245,8 +251,8 @@ def lower(self): def upper(self): """Access the upper bound of a constraint expression.""" ans = self.normalize_constraint()[2] - if ans.__class__ in native_numeric_types: - # Historically, constraint.lower was guaranteed to return a type + if ans.__class__ in native_types and ans is not None: + # Historically, constraint.upper was guaranteed to return a type # derived from Pyomo NumericValue (or None). Replicate that # functionality, although clients should in almost all cases # move to using ConstraintData.lb instead of accessing From cdfb2b17d5d78147a0a335b996934405cbc5c8c0 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 17 Jun 2024 08:09:08 -0600 Subject: [PATCH 069/150] Update baseline due to changes in Constraint (something caused gdpopt to set variables to int instead of floats) --- doc/OnlineDocs/contributed_packages/gdpopt.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/OnlineDocs/contributed_packages/gdpopt.rst b/doc/OnlineDocs/contributed_packages/gdpopt.rst index d550b0ced76..670d7633f6d 100644 --- a/doc/OnlineDocs/contributed_packages/gdpopt.rst +++ b/doc/OnlineDocs/contributed_packages/gdpopt.rst @@ -93,10 +93,10 @@ An example that includes the modeling approach may be found below. Variables: x : Size=1, Index=None Key : Lower : Value : Upper : Fixed : Stale : Domain - None : -1.2 : 0.0 : 2 : False : False : Reals + None : -1.2 : 0 : 2 : False : False : Reals y : Size=1, Index=None Key : Lower : Value : Upper : Fixed : Stale : Domain - None : -10 : 1.0 : 10 : False : False : Reals + None : -10 : 1 : 10 : False : False : Reals Objectives: objective : Size=1, Index=None, Active=True @@ -106,7 +106,7 @@ An example that includes the modeling approach may be found below. Constraints: c : Size=1 Key : Lower : Body : Upper - None : 1.0 : 1.0 : 1.0 + None : 1.0 : 1 : 1.0 .. note:: From 1e7fabdd345c1f283b53abd3c04c0b49e5f10ce4 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Mon, 17 Jun 2024 10:32:32 -0600 Subject: [PATCH 070/150] Update Pyomo for NumPy 2 --- pyomo/common/dependencies.py | 10 ++++++++-- pyomo/core/kernel/register_numpy_types.py | 10 ++++++---- .../tests/unit/test_kernel_register_numpy_types.py | 6 ++++-- pyomo/core/tests/unit/test_numvalue.py | 8 ++++---- pyomo/core/tests/unit/test_sets.py | 12 ++++++------ 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/pyomo/common/dependencies.py b/pyomo/common/dependencies.py index 4c9e43002ef..cf05b68612d 100644 --- a/pyomo/common/dependencies.py +++ b/pyomo/common/dependencies.py @@ -999,10 +999,13 @@ def _finalize_numpy(np, available): # registration here (to bypass the deprecation warning) until we # finally remove all support for it numeric_types._native_boolean_types.add(t) - _floats = [np.float_, np.float16, np.float32, np.float64] + _floats = [np.float16, np.float32, np.float64] # float96 and float128 may or may not be defined in this particular # numpy build (it depends on platform and version). # Register them only if they are present + if hasattr(np, 'float_'): + # Prepend to preserve previous functionality + _floats = [np.float_] + _floats if hasattr(np, 'float96'): _floats.append(np.float96) if hasattr(np, 'float128'): @@ -1013,10 +1016,13 @@ def _finalize_numpy(np, available): # registration here (to bypass the deprecation warning) until we # finally remove all support for it numeric_types._native_boolean_types.add(t) - _complex = [np.complex_, np.complex64, np.complex128] + _complex = [np.complex64, np.complex128] # complex192 and complex256 may or may not be defined in this # particular numpy build (it depends on platform and version). # Register them only if they are present + if hasattr(np, 'np.complex_'): + # Prepend to preserve functionality + _complex = [np.complex_] + _complex if hasattr(np, 'complex192'): _complex.append(np.complex192) if hasattr(np, 'complex256'): diff --git a/pyomo/core/kernel/register_numpy_types.py b/pyomo/core/kernel/register_numpy_types.py index 86877be2230..b3405645d97 100644 --- a/pyomo/core/kernel/register_numpy_types.py +++ b/pyomo/core/kernel/register_numpy_types.py @@ -45,10 +45,12 @@ # Historically, the lists included several numpy aliases numpy_int_names.extend(('int_', 'intc', 'intp')) numpy_int.extend((numpy.int_, numpy.intc, numpy.intp)) - numpy_float_names.append('float_') - numpy_float.append(numpy.float_) - numpy_complex_names.append('complex_') - numpy_complex.append(numpy.complex_) + if hasattr(numpy, 'float_'): + numpy_float_names.append('float_') + numpy_float.append(numpy.float_) + if hasattr(numpy, 'complex_'): + numpy_complex_names.append('complex_') + numpy_complex.append(numpy.complex_) # Re-build the old numpy_* lists for t in native_boolean_types: diff --git a/pyomo/core/tests/unit/test_kernel_register_numpy_types.py b/pyomo/core/tests/unit/test_kernel_register_numpy_types.py index 91a0f571881..d49b3cd6b4a 100644 --- a/pyomo/core/tests/unit/test_kernel_register_numpy_types.py +++ b/pyomo/core/tests/unit/test_kernel_register_numpy_types.py @@ -34,7 +34,8 @@ # Reals numpy_float_names = [] if numpy_available: - numpy_float_names.append('float_') + if hasattr(numpy, 'float_'): + numpy_float_names.append('float_') numpy_float_names.append('float16') numpy_float_names.append('float32') numpy_float_names.append('float64') @@ -46,7 +47,8 @@ # Complex numpy_complex_names = [] if numpy_available: - numpy_complex_names.append('complex_') + if hasattr(numpy, 'complex_'): + numpy_complex_names.append('complex_') numpy_complex_names.append('complex64') numpy_complex_names.append('complex128') if hasattr(numpy, 'complex192'): diff --git a/pyomo/core/tests/unit/test_numvalue.py b/pyomo/core/tests/unit/test_numvalue.py index 1cccd3863ea..4d39a42ed70 100644 --- a/pyomo/core/tests/unit/test_numvalue.py +++ b/pyomo/core/tests/unit/test_numvalue.py @@ -552,10 +552,10 @@ def test_unknownNumericType(self): @unittest.skipUnless(numpy_available, "This test requires NumPy") def test_numpy_basic_float_registration(self): - self.assertIn(numpy.float_, native_numeric_types) - self.assertNotIn(numpy.float_, native_integer_types) - self.assertIn(numpy.float_, _native_boolean_types) - self.assertIn(numpy.float_, native_types) + self.assertIn(numpy.float64, native_numeric_types) + self.assertNotIn(numpy.float64, native_integer_types) + self.assertIn(numpy.float64, _native_boolean_types) + self.assertIn(numpy.float64, native_types) @unittest.skipUnless(numpy_available, "This test requires NumPy") def test_numpy_basic_int_registration(self): diff --git a/pyomo/core/tests/unit/test_sets.py b/pyomo/core/tests/unit/test_sets.py index 48869397aae..4d305ebab86 100644 --- a/pyomo/core/tests/unit/test_sets.py +++ b/pyomo/core/tests/unit/test_sets.py @@ -1051,7 +1051,7 @@ def setUp(self): self.instance = self.model.create_instance(currdir + "setA.dat") self.e1 = numpy.bool_(1) self.e2 = numpy.int_(2) - self.e3 = numpy.float_(3.0) + self.e3 = numpy.float64(3.0) self.e4 = numpy.int_(4) self.e5 = numpy.int_(5) self.e6 = numpy.int_(6) @@ -1068,7 +1068,7 @@ def test_numpy_int(self): def test_numpy_float(self): model = ConcreteModel() - model.A = Set(initialize=[numpy.float_(1.0), numpy.float_(0.0)]) + model.A = Set(initialize=[numpy.float64(1.0), numpy.float64(0.0)]) self.assertEqual(model.A.bounds(), (0, 1)) @@ -3213,7 +3213,7 @@ def test_numpy_membership(self): self.assertEqual(numpy.int_(1) in Boolean, True) self.assertEqual(numpy.bool_(True) in Boolean, True) self.assertEqual(numpy.bool_(False) in Boolean, True) - self.assertEqual(numpy.float_(1.1) in Boolean, False) + self.assertEqual(numpy.float64(1.1) in Boolean, False) self.assertEqual(numpy.int_(2) in Boolean, False) self.assertEqual(numpy.int_(0) in Integers, True) @@ -3222,7 +3222,7 @@ def test_numpy_membership(self): # identically to 1 self.assertEqual(numpy.bool_(True) in Integers, True) self.assertEqual(numpy.bool_(False) in Integers, True) - self.assertEqual(numpy.float_(1.1) in Integers, False) + self.assertEqual(numpy.float64(1.1) in Integers, False) self.assertEqual(numpy.int_(2) in Integers, True) self.assertEqual(numpy.int_(0) in Reals, True) @@ -3231,14 +3231,14 @@ def test_numpy_membership(self): # identically to 1 self.assertEqual(numpy.bool_(True) in Reals, True) self.assertEqual(numpy.bool_(False) in Reals, True) - self.assertEqual(numpy.float_(1.1) in Reals, True) + self.assertEqual(numpy.float64(1.1) in Reals, True) self.assertEqual(numpy.int_(2) in Reals, True) self.assertEqual(numpy.int_(0) in Any, True) self.assertEqual(numpy.int_(1) in Any, True) self.assertEqual(numpy.bool_(True) in Any, True) self.assertEqual(numpy.bool_(False) in Any, True) - self.assertEqual(numpy.float_(1.1) in Any, True) + self.assertEqual(numpy.float64(1.1) in Any, True) self.assertEqual(numpy.int_(2) in Any, True) def test_setargs1(self): From 64e01bbfcf84d3664925768943ceae166ae61636 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 17 Jun 2024 12:17:11 -0600 Subject: [PATCH 071/150] NFC: improve variable naming --- pyomo/contrib/fbbt/fbbt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/fbbt/fbbt.py b/pyomo/contrib/fbbt/fbbt.py index 39df91675a8..60ac0603388 100644 --- a/pyomo/contrib/fbbt/fbbt.py +++ b/pyomo/contrib/fbbt/fbbt.py @@ -1324,10 +1324,10 @@ def _fbbt_con(con, config): visitorA = _FBBTVisitorLeafToRoot(bnds_dict, feasibility_tol=config.feasibility_tol) visitorA.walk_expression(con.expr) - always_feasible, feasible = bnds_dict[con.expr] + always_feasible, possibly_feasible = bnds_dict[con.expr] # check if the constraint is infeasible - if not feasible: + if not possibly_feasible: raise InfeasibleConstraintException( 'Detected an infeasible constraint during FBBT: {0}'.format(str(con)) ) From cb6723041d999380ef4452158b9dce99629020c0 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Mon, 17 Jun 2024 12:50:10 -0600 Subject: [PATCH 072/150] Change to insert --- pyomo/common/dependencies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/common/dependencies.py b/pyomo/common/dependencies.py index cf05b68612d..bbcea0b85d7 100644 --- a/pyomo/common/dependencies.py +++ b/pyomo/common/dependencies.py @@ -1005,7 +1005,7 @@ def _finalize_numpy(np, available): # Register them only if they are present if hasattr(np, 'float_'): # Prepend to preserve previous functionality - _floats = [np.float_] + _floats + _floats.insert(0, np.float_) if hasattr(np, 'float96'): _floats.append(np.float96) if hasattr(np, 'float128'): @@ -1022,7 +1022,7 @@ def _finalize_numpy(np, available): # Register them only if they are present if hasattr(np, 'np.complex_'): # Prepend to preserve functionality - _complex = [np.complex_] + _complex + _complex.insert(0, np.complex_) if hasattr(np, 'complex192'): _complex.append(np.complex192) if hasattr(np, 'complex256'): From 9c73e8f8d0a7a6acc6c0041220c709bf6dae96c6 Mon Sep 17 00:00:00 2001 From: Atalay Kutlay Date: Sat, 1 Jun 2024 23:52:21 -0400 Subject: [PATCH 073/150] Add test for variable fix and unfix --- .../solvers/tests/test_highs_persistent.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py b/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py index b26f45ff2cc..4d8251e0de9 100644 --- a/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py +++ b/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py @@ -80,6 +80,43 @@ def test_mutable_params_with_remove_vars(self): res = opt.solve(m) self.assertAlmostEqual(res.best_feasible_objective, -9) + def test_fix_and_unfix(self): + # Tests issue https://github.com/Pyomo/pyomo/issues/3127 + + m = pe.ConcreteModel() + m.x = pe.Var(domain=pe.Binary) + m.y = pe.Var(domain=pe.Binary) + m.fx = pe.Var(domain=pe.NonNegativeReals) + m.fy = pe.Var(domain=pe.NonNegativeReals) + m.c1 = pe.Constraint(expr=m.fx <= m.x) + m.c2 = pe.Constraint(expr=m.fy <= m.y) + m.c3 = pe.Constraint(expr=m.x + m.y <= 1) + + m.obj = pe.Objective(expr=m.fx * 0.5 + m.fy * 0.4, sense=pe.maximize) + + opt = Highs() + + # solution 1 has m.x == 1 and m.y == 0 + r = opt.solve(m) + self.assertAlmostEqual(m.fx.value, 1, places=5) + self.assertAlmostEqual(m.fy.value, 0, places=5) + self.assertAlmostEqual(r.best_feasible_objective, 0.5, places=5) + + # solution 2 has m.x == 0 and m.y == 1 + m.y.fix(1) + r = opt.solve(m) + self.assertAlmostEqual(m.fx.value, 0, places=5) + self.assertAlmostEqual(m.fy.value, 1, places=5) + self.assertAlmostEqual(r.best_feasible_objective, 0.4, places=5) + + # solution 3 should be equal solution 1 + m.y.unfix() + m.x.fix(1) + r = opt.solve(m) + self.assertAlmostEqual(m.fx.value, 1, places=5) + self.assertAlmostEqual(m.fy.value, 0, places=5) + self.assertAlmostEqual(r.best_feasible_objective, 0.5, places=5) + def test_capture_highs_output(self): # tests issue #3003 # From f3b65cd50686886fa6b06dbb33c6c77c8d6a226f Mon Sep 17 00:00:00 2001 From: Alma Walmsley Date: Mon, 24 Jun 2024 18:01:21 +1200 Subject: [PATCH 074/150] Ignore errors on ASL solver version check --- pyomo/solvers/plugins/solvers/ASL.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/solvers/plugins/solvers/ASL.py b/pyomo/solvers/plugins/solvers/ASL.py index bb8174a013e..a29c64e017f 100644 --- a/pyomo/solvers/plugins/solvers/ASL.py +++ b/pyomo/solvers/plugins/solvers/ASL.py @@ -102,6 +102,7 @@ def _get_version(self): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, + errors='ignore', ) ver = _extract_version(results.stdout) if ver is None: From 6f86e3231f0c82e4a42693030fb64ba1f58b51dc Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 24 Jun 2024 10:15:53 -0600 Subject: [PATCH 075/150] Switch writers to use str() (instead of repr()) for numeric constants In Python 3.x, str() and repr() are the same for float, and beginning in numpy 2.0 `repr(float64(n))` produces `numpy.float64(n)`, whereas str() produces `n` --- pyomo/repn/plugins/lp_writer.py | 24 ++++++++++++------------ pyomo/repn/plugins/nl_writer.py | 28 ++++++++++++++-------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pyomo/repn/plugins/lp_writer.py b/pyomo/repn/plugins/lp_writer.py index 627a54e3f68..814f79a4eb9 100644 --- a/pyomo/repn/plugins/lp_writer.py +++ b/pyomo/repn/plugins/lp_writer.py @@ -458,13 +458,13 @@ def write(self, model): addSymbol(con, label) ostream.write(f'\n{label}:\n') self.write_expression(ostream, repn, False) - ostream.write(f'>= {(lb - offset)!r}\n') + ostream.write(f'>= {(lb - offset)!s}\n') elif lb == ub: label = f'c_e_{symbol}_' addSymbol(con, label) ostream.write(f'\n{label}:\n') self.write_expression(ostream, repn, False) - ostream.write(f'= {(lb - offset)!r}\n') + ostream.write(f'= {(lb - offset)!s}\n') else: # We will need the constraint body twice. Generate # in a buffer so we only have to do that once. @@ -476,18 +476,18 @@ def write(self, model): addSymbol(con, label) ostream.write(f'\n{label}:\n') ostream.write(buf) - ostream.write(f'>= {(lb - offset)!r}\n') + ostream.write(f'>= {(lb - offset)!s}\n') label = f'r_u_{symbol}_' aliasSymbol(con, label) ostream.write(f'\n{label}:\n') ostream.write(buf) - ostream.write(f'<= {(ub - offset)!r}\n') + ostream.write(f'<= {(ub - offset)!s}\n') elif ub is not None: label = f'c_u_{symbol}_' addSymbol(con, label) ostream.write(f'\n{label}:\n') self.write_expression(ostream, repn, False) - ostream.write(f'<= {(ub - offset)!r}\n') + ostream.write(f'<= {(ub - offset)!s}\n') if with_debug_timing: # report the last constraint @@ -527,8 +527,8 @@ def write(self, model): # Note: Var.bounds guarantees the values are either (finite) # native_numeric_types or None lb, ub = v.bounds - lb = '-inf' if lb is None else repr(lb) - ub = '+inf' if ub is None else repr(ub) + lb = '-inf' if lb is None else str(lb) + ub = '+inf' if ub is None else str(ub) ostream.write(f"\n {lb} <= {v_symbol} <= {ub}") if integer_vars: @@ -565,7 +565,7 @@ def write(self, model): for v, w in getattr(soscon, 'get_items', soscon.items)(): if w.__class__ not in int_float: w = float(f) - ostream.write(f" {getSymbol(v)}:{w!r}\n") + ostream.write(f" {getSymbol(v)}:{w!s}\n") ostream.write("\nend\n") @@ -584,9 +584,9 @@ def write_expression(self, ostream, expr, is_objective): expr.linear.items(), key=lambda x: getVarOrder(x[0]) ): if coef < 0: - ostream.write(f'{coef!r} {getSymbol(getVar(vid))}\n') + ostream.write(f'{coef!s} {getSymbol(getVar(vid))}\n') else: - ostream.write(f'+{coef!r} {getSymbol(getVar(vid))}\n') + ostream.write(f'+{coef!s} {getSymbol(getVar(vid))}\n') quadratic = getattr(expr, 'quadratic', None) if quadratic: @@ -605,9 +605,9 @@ def _normalize_constraint(data): col = c1, c2 sym = f' {getSymbol(getVar(vid1))} * {getSymbol(getVar(vid2))}\n' if coef < 0: - return col, repr(coef) + sym + return col, str(coef) + sym else: - return col, '+' + repr(coef) + sym + return col, f'+{coef!s}{sym}' if is_objective: # diff --git a/pyomo/repn/plugins/nl_writer.py b/pyomo/repn/plugins/nl_writer.py index 43fd2fade68..a8966e44f71 100644 --- a/pyomo/repn/plugins/nl_writer.py +++ b/pyomo/repn/plugins/nl_writer.py @@ -1150,17 +1150,17 @@ def write(self, model): r_lines[idx] = "3" else: # _type = 4 # L == c == U - r_lines[idx] = f"4 {lb - expr_info.const!r}" + r_lines[idx] = f"4 {lb - expr_info.const!s}" n_equality += 1 elif lb is None: # _type = 1 # c <= U - r_lines[idx] = f"1 {ub - expr_info.const!r}" + r_lines[idx] = f"1 {ub - expr_info.const!s}" elif ub is None: # _type = 2 # L <= c - r_lines[idx] = f"2 {lb - expr_info.const!r}" + r_lines[idx] = f"2 {lb - expr_info.const!s}" else: # _type = 0 # L <= c <= U - r_lines[idx] = f"0 {lb - expr_info.const!r} {ub - expr_info.const!r}" + r_lines[idx] = f"0 {lb - expr_info.const!s} {ub - expr_info.const!s}" n_ranges += 1 expr_info.const = 0 # FIXME: this is a HACK to be compatible with the NLv1 @@ -1375,7 +1375,7 @@ def write(self, model): ostream.write(f"S{_field|_float} {len(_vals)} {name}\n") # Note: _SuffixData.compile() guarantees the value is int/float ostream.write( - ''.join(f"{_id} {_vals[_id]!r}\n" for _id in sorted(_vals)) + ''.join(f"{_id} {_vals[_id]!s}\n" for _id in sorted(_vals)) ) # @@ -1485,7 +1485,7 @@ def write(self, model): ostream.write(f"d{len(data.con)}\n") # Note: _SuffixData.compile() guarantees the value is int/float ostream.write( - ''.join(f"{_id} {data.con[_id]!r}\n" for _id in sorted(data.con)) + ''.join(f"{_id} {data.con[_id]!s}\n" for _id in sorted(data.con)) ) # @@ -1507,7 +1507,7 @@ def write(self, model): ) ostream.write( ''.join( - f'{var_idx} {val!r}{col_comments[var_idx]}\n' + f'{var_idx} {val!s}{col_comments[var_idx]}\n' for var_idx, val in _init_lines ) ) @@ -1548,13 +1548,13 @@ def write(self, model): if lb is None: # unbounded ostream.write(f"3{col_comments[var_idx]}\n") else: # == - ostream.write(f"4 {lb!r}{col_comments[var_idx]}\n") + ostream.write(f"4 {lb!s}{col_comments[var_idx]}\n") elif lb is None: # var <= ub - ostream.write(f"1 {ub!r}{col_comments[var_idx]}\n") + ostream.write(f"1 {ub!s}{col_comments[var_idx]}\n") elif ub is None: # lb <= body - ostream.write(f"2 {lb!r}{col_comments[var_idx]}\n") + ostream.write(f"2 {lb!s}{col_comments[var_idx]}\n") else: # lb <= body <= ub - ostream.write(f"0 {lb!r} {ub!r}{col_comments[var_idx]}\n") + ostream.write(f"0 {lb!s} {ub!s}{col_comments[var_idx]}\n") # # "k" lines (column offsets in Jacobian NNZ) @@ -1589,7 +1589,7 @@ def write(self, model): linear[_id] /= scaling_cache[_id] ostream.write(f'J{row_idx} {len(linear)}{row_comments[row_idx]}\n') for _id in sorted(linear, key=column_order.__getitem__): - ostream.write(f'{column_order[_id]} {linear[_id]!r}\n') + ostream.write(f'{column_order[_id]} {linear[_id]!s}\n') # # "G" lines (non-empty terms in the Objective) @@ -1605,7 +1605,7 @@ def write(self, model): linear[_id] /= scaling_cache[_id] ostream.write(f'G{obj_idx} {len(linear)}{row_comments[obj_idx + n_cons]}\n') for _id in sorted(linear, key=column_order.__getitem__): - ostream.write(f'{column_order[_id]} {linear[_id]!r}\n') + ostream.write(f'{column_order[_id]} {linear[_id]!s}\n') # Generate the return information eliminated_vars = [ @@ -2041,7 +2041,7 @@ def _write_v_line(self, expr_id, k): # ostream.write(f'V{self.next_V_line_id} {len(linear)} {k}{lbl}\n') for _id in sorted(linear, key=column_order.__getitem__): - ostream.write(f'{column_order[_id]} {linear[_id]!r}\n') + ostream.write(f'{column_order[_id]} {linear[_id]!s}\n') self._write_nl_expression(info[1], True) self.next_V_line_id += 1 From 2145a1684c726d8170460ef148be673459c3eac5 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 24 Jun 2024 11:06:37 -0600 Subject: [PATCH 076/150] Improve tokenization in baseline comparisons --- pyomo/common/unittest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyomo/common/unittest.py b/pyomo/common/unittest.py index 84d962eb784..a947420a25d 100644 --- a/pyomo/common/unittest.py +++ b/pyomo/common/unittest.py @@ -807,6 +807,15 @@ def filter_file_contents(self, lines, abstol=None): item_list = [] items = line.strip().split() for i in items: + # Split up lists, dicts, and sets + while i and i[0] in '[{': + item_list.append(i[0]) + i = i[1:] + tail = [] + while i and i[-1] in ',:]}': + tail.append(i[-1]) + i = i[:-1] + # A few substitutions to get tests passing on pypy3 if ".inf" in i: i = i.replace(".inf", "inf") @@ -817,6 +826,9 @@ def filter_file_contents(self, lines, abstol=None): item_list.append(float(i)) except: item_list.append(i) + if tail: + tail.reverse() + item_list.extend(tail) # We can get printed results objects where the baseline is # exactly 0 (and omitted) and the test is slightly non-zero. From e5973d4014a63fe7093b3ea70292b14650f9f06f Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 24 Jun 2024 11:07:06 -0600 Subject: [PATCH 077/150] Map new numpy type representations back to pre-numpy2.0 behavior --- pyomo/common/unittest.py | 10 +++++++++- .../tests/unit/test_kernel_register_numpy_types.py | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pyomo/common/unittest.py b/pyomo/common/unittest.py index a947420a25d..5b6d1f003d8 100644 --- a/pyomo/common/unittest.py +++ b/pyomo/common/unittest.py @@ -783,6 +783,7 @@ def filter_fcn(self, line): return False def filter_file_contents(self, lines, abstol=None): + _numpy_scalar_re = re.compile(r'np.(int|float)\d+\(([^\)]+)\)') filtered = [] deprecated = None for line in lines: @@ -823,7 +824,14 @@ def filter_file_contents(self, lines, abstol=None): i = i.replace("null", "None") try: - item_list.append(float(i)) + # Numpy 2.x changed the repr for scalars. Convert + # the new scalar reprs back to the original (which + # were indistinguishable from python floats/ints) + np_match = _numpy_scalar_re.match(i) + if np_match: + item_list.append(float(np_match.group(2))) + else: + item_list.append(float(i)) except: item_list.append(i) if tail: diff --git a/pyomo/core/tests/unit/test_kernel_register_numpy_types.py b/pyomo/core/tests/unit/test_kernel_register_numpy_types.py index d49b3cd6b4a..8186c3d6028 100644 --- a/pyomo/core/tests/unit/test_kernel_register_numpy_types.py +++ b/pyomo/core/tests/unit/test_kernel_register_numpy_types.py @@ -16,7 +16,10 @@ # Boolean numpy_bool_names = [] if numpy_available: - numpy_bool_names.append('bool_') + if numpy.__version__[0] == '2': + numpy_bool_names.append('bool') + else: + numpy_bool_names.append('bool_') # Integers numpy_int_names = [] if numpy_available: From c88e0d5ad9130e24d8019052001196afe0da2222 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 24 Jun 2024 13:03:57 -0600 Subject: [PATCH 078/150] Update near-0 filter based on new tokenizer --- pyomo/common/unittest.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pyomo/common/unittest.py b/pyomo/common/unittest.py index 5b6d1f003d8..c78e003a07d 100644 --- a/pyomo/common/unittest.py +++ b/pyomo/common/unittest.py @@ -844,12 +844,13 @@ def filter_file_contents(self, lines, abstol=None): # results objects and remote them if they are within # tolerance of 0 if ( - len(item_list) == 2 - and item_list[0] == 'Value:' - and type(item_list[1]) is float - and abs(item_list[1]) < (abstol or 0) - and len(filtered[-1]) == 1 - and filtered[-1][0][-1] == ':' + len(item_list) == 3 + and item_list[0] == 'Value' + and item_list[1] == ':' + and type(item_list[2]) is float + and abs(item_list[2]) < (abstol or 0) + and len(filtered[-1]) == 2 + and filtered[-1][1] == ':' ): filtered.pop() else: From 3ea998aa71f297016fdee8541f232ed8746aaeff Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Mon, 24 Jun 2024 14:17:54 -0600 Subject: [PATCH 079/150] Sneak in fix for codecov tokenless upload --- .github/workflows/test_branches.yml | 6 ++++++ .github/workflows/test_pr_and_main.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 8ba04eec466..2689b97746b 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -855,6 +855,9 @@ jobs: token: ${{ secrets.PYOMO_CODECOV_TOKEN }} name: ${{ matrix.TARGET }} flags: ${{ matrix.TARGET }} + # downgrading after v0.7.0 broke tokenless upload + # see codecov/codecov-action#1487 + version: v0.6.0 fail_ci_if_error: true - name: Upload other coverage reports @@ -867,4 +870,7 @@ jobs: token: ${{ secrets.PYOMO_CODECOV_TOKEN }} name: ${{ matrix.TARGET }}/other flags: ${{ matrix.TARGET }},other + # downgrading after v0.7.0 broke tokenless upload + # see codecov/codecov-action#1487 + version: v0.6.0 fail_ci_if_error: true diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index bdf1f7e1aa5..c804372b18a 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -899,6 +899,9 @@ jobs: token: ${{ secrets.PYOMO_CODECOV_TOKEN }} name: ${{ matrix.TARGET }} flags: ${{ matrix.TARGET }} + # downgrading after v0.7.0 broke tokenless upload + # see codecov/codecov-action#1487 + version: v0.6.0 fail_ci_if_error: true - name: Upload other coverage reports @@ -911,4 +914,7 @@ jobs: token: ${{ secrets.PYOMO_CODECOV_TOKEN }} name: ${{ matrix.TARGET }}/other flags: ${{ matrix.TARGET }},other + # downgrading after v0.7.0 broke tokenless upload + # see codecov/codecov-action#1487 + version: v0.6.0 fail_ci_if_error: true From 390463936dbcc2496b694059cad499095a344fea Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 25 Jun 2024 14:21:14 -0600 Subject: [PATCH 080/150] Rework Set initialization to improve performance on large sets --- pyomo/core/base/set.py | 423 +++++++++++++++--------------- pyomo/core/tests/unit/test_set.py | 44 ++-- 2 files changed, 233 insertions(+), 234 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 8b7c2a246d6..fcabf6674a4 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -16,15 +16,18 @@ import math import sys import weakref -from pyomo.common.pyomo_typing import overload -from typing import Union, Type, Any as typingAny + from collections.abc import Iterator +from functools import partial +from typing import Union, Type, Any as typingAny +from pyomo.common.autoslots import AutoSlots from pyomo.common.collections import ComponentSet from pyomo.common.deprecation import deprecated, deprecation_warning, RenamedClass from pyomo.common.errors import DeveloperError, PyomoException from pyomo.common.log import is_debug_set from pyomo.common.modeling import NOTSET +from pyomo.common.pyomo_typing import overload from pyomo.common.sorting import sorted_robust from pyomo.common.timing import ConstructionTimer @@ -478,9 +481,7 @@ def __call__(self, parent, index): if not isinstance(_val, Sequence): _val = tuple(_val) - if len(_val) == 0: - return _val - if isinstance(_val[0], tuple): + if not _val or isinstance(_val[0], tuple): return _val return self._tuplize(_val, parent, index) @@ -501,7 +502,7 @@ def _tuplize(self, _val, parent, index): "length %s is not a multiple of dimen=%s" % (len(_val), d) ) - return list(tuple(_val[d * i : d * (i + 1)]) for i in range(len(_val) // d)) + return (tuple(_val[i : i + d]) for i in range(0, len(_val), d)) class _NotFound(object): @@ -1364,87 +1365,10 @@ def filter(self): return self._filter def add(self, *values): - count = 0 - _block = self.parent_block() - for value in values: - if normalize_index.flatten: - _value = normalize_index(value) - if _value.__class__ is tuple: - _d = len(_value) - else: - _d = 1 - else: - # If we are not normalizing indices, then we cannot reliably - # infer the set dimen - _d = 1 - if isinstance(value, Sequence) and self.dimen != 1: - _d = len(value) - _value = value - if _value not in self._domain: - raise ValueError( - "Cannot add value %s to Set %s.\n" - "\tThe value is not in the domain %s" - % (value, self.name, self._domain) - ) - - # We wrap this check in a try-except because some values - # (like lists) are not hashable and can raise exceptions. - try: - if _value in self: - logger.warning( - "Element %s already exists in Set %s; no action taken" - % (value, self.name) - ) - continue - except: - exc = sys.exc_info() - raise TypeError( - "Unable to insert '%s' into Set %s:\n\t%s: %s" - % (value, self.name, exc[0].__name__, exc[1]) - ) - - if self._filter is not None: - if not self._filter(_block, _value): - continue - - if self._validate is not None: - try: - flag = self._validate(_block, _value) - except: - logger.error( - "Exception raised while validating element '%s' " - "for Set %s" % (value, self.name) - ) - raise - if not flag: - raise ValueError( - "The value=%s violates the validation rule of Set %s" - % (value, self.name) - ) - - # If the Set has a fixed dimension, check that this element is - # compatible. - if self._dimen is not None: - if _d != self._dimen: - if self._dimen is UnknownSetDimen: - # The first thing added to a Set with unknown - # dimension sets its dimension - self._dimen = _d - else: - raise ValueError( - "The value=%s has dimension %s and is not " - "valid for Set %s which has dimen=%s" - % (value, _d, self.name, self._dimen) - ) - - # Add the value to this object (this last redirection allows - # derived classes to implement a different storage mechanism) - self._add_impl(_value) - count += 1 - return count + return self.update(values) - def _add_impl(self, value): - self._values.add(value) + def _update_impl(self, values): + self._values.update(values) def remove(self, val): self._values.remove(val) @@ -1457,17 +1381,144 @@ def clear(self): def set_value(self, val): self.clear() - for x in val: - self.add(x) + self.update(val) def update(self, values): - for v in values: - if v not in self: - self.add(v) + # _values was initialized above... + # + # Special case: set operations that are not first attached + # to the model must be constructed. + if isinstance(values, SetOperator): + values.construct() + try: + val_iter = iter(values) + except TypeError: + logger.error( + "Initializer for Set %s%s returned non-iterable object " + "of type %s." + % ( + self.name, + ("[%s]" % (index,) if self.is_indexed() else ""), + (values if values.__class__ is type else type(values).__name__), + ) + ) + raise + + if self._dimen is not None: + if normalize_index.flatten: + val_iter = self._cb_normalized_dimen_verifier(self._dimen, val_iter) + else: + val_iter = self._cb_raw_dimen_verifier(self._dimen, val_iter) + elif normalize_index.flatten: + val_iter = map(normalize_index, val_iter) + else: + val_iter = self._cb_check_set_end(val_iter) + + if self._domain is not Any: + val_iter = self._cb_domain_verifier(self._domain, val_iter) + + if self._filter is not None: + val_iter = filter(partial(self._filter, self.parent_block()), val_iter) + + if self._validate is not None: + val_iter = self._cb_validate(self._validate, self.parent_block(), val_iter) + + nOrig = len(self) + # We wrap this check in a try-except because some values + # (like lists) are not hashable and can raise exceptions. + try: + self._update_impl(val_iter) + except Set.End: + pass + return len(self) - nOrig def pop(self): return self._values.pop() + def _cb_domain_verifier(self, domain, val_iter): + for value in val_iter: + if value not in domain: + raise ValueError( + "Cannot add value %s to Set %s.\n" + "\tThe value is not in the domain %s" + % (value, self.name, self._domain) + ) + yield value + + def _cb_check_set_end(self, val_iter): + for value in val_iter: + if value is Set.End: + return + yield value + + def _cb_validate(self, validate, block, val_iter): + for value in val_iter: + try: + flag = validate(block, value) + except: + logger.error( + "Exception raised while validating element '%s' " + "for Set %s" % (value, self.name) + ) + raise + if not flag: + raise ValueError( + "The value=%s violates the validation rule of Set %s" + % (value, self.name) + ) + yield value + + def _cb_normalized_dimen_verifier(self, dimen, val_iter): + # It is important that the iterator is an actual iterator + val_iter = iter(val_iter) + for value in val_iter: + if value.__class__ is tuple: + if dimen == len(value): + yield value[0] if dimen == 1 else value + continue + elif dimen == 1 and value.__class__ in native_types: + yield value + continue + + # Note: normalize_index() will never return a 1-tuple + normalized_value = normalize_index(value) + _d = len(normalized_value) if normalized_value.__class__ is tuple else 1 + if _d == dimen: + yield normalized_value + elif dimen is UnknownSetDimen: + # The first thing added to a Set with unknown dimension + # sets its dimension + self._dimen = dimen = _d + yield normalized_value + else: + raise ValueError( + "The value=%s has dimension %s and is not " + "valid for Set %s which has dimen=%s" + % (value, _d, self.name, self._dimen) + ) + + def _cb_raw_dimen_verifier(self, dimen, val_iter): + for value in val_iter: + if isinstance(value, Sequence): + if dimen == len(value): + yield value + continue + elif dimen == 1: + yield value + continue + _d = len(value) if isinstance(value, Sequence) else 1 + if dimen is UnknownSetDimen: + # The first thing added to a Set with unknown dimension + # sets its dimension + self._dimen = dimen = _d + yield value + else: + raise ValueError( + "The value=%s has dimension %s and is not " + "valid for Set %s which has dimen=%s" + % (value, _d, self.name, self._dimen) + ) + class _FiniteSetData(metaclass=RenamedClass): __renamed__new_class__ = FiniteSetData @@ -1655,27 +1706,30 @@ class OrderedSetData(_OrderedSetMixin, FiniteSetData): def __init__(self, component): self._values = {} - self._ordered_values = [] + self._ordered_values = None FiniteSetData.__init__(self, component=component) def _iter_impl(self): """ Return an iterator for the set. """ - return iter(self._ordered_values) + return iter(self._values) def __reversed__(self): - return reversed(self._ordered_values) + return reversed(self._values) - def _add_impl(self, value): - self._values[value] = len(self._values) - self._ordered_values.append(value) + def _update_impl(self, values): + for val in values: + # Note that we reset _ordered_values within the loop because + # of an old example where the initializer rule makes + # reference to values previously inserted into the Set + # (which triggered the creation of the _ordered_values) + self._ordered_values = None + self._values[val] = None def remove(self, val): - idx = self._values.pop(val) - self._ordered_values.pop(idx) - for i in range(idx, len(self._ordered_values)): - self._values[self._ordered_values[i]] -= 1 + self._values.pop(val) + self._ordered_values = None def discard(self, val): try: @@ -1685,7 +1739,7 @@ def discard(self, val): def clear(self): self._values.clear() - self._ordered_values = [] + self._ordered_values = None def pop(self): try: @@ -1704,6 +1758,8 @@ def at(self, index): The public Set API is 1-based, even though the internal _lookup and _values are (pythonically) 0-based. """ + if self._ordered_values is None: + self._rebuild_ordered_values() i = self._to_0_based_index(index) try: return self._ordered_values[i] @@ -1723,6 +1779,8 @@ def ord(self, item): # when they are actually put as Set members. So, we will look # for the exact thing that the user sent us and then fall back # on the scalar. + if self._ordered_values is None: + self._rebuild_ordered_values() try: return self._values[item] + 1 except KeyError: @@ -1733,6 +1791,12 @@ def ord(self, item): except KeyError: raise ValueError("%s.ord(x): x not in %s" % (self.name, self.name)) + def _rebuild_ordered_values(self): + _set = self._values + self._ordered_values = list(_set) + for i, v in enumerate(self._ordered_values): + _set[v] = i + class _OrderedSetData(metaclass=RenamedClass): __renamed__new_class__ = OrderedSetData @@ -1760,7 +1824,8 @@ def set_value(self, val): "This WILL potentially lead to nondeterministic behavior " "in Pyomo" % (type(val).__name__,) ) - super(InsertionOrderSetData, self).set_value(val) + self.clear() + super().update(val) def update(self, values): if type(values) in Set._UnorderedInitializers: @@ -1770,7 +1835,7 @@ def update(self, values): "This WILL potentially lead to nondeterministic behavior " "in Pyomo" % (type(values).__name__,) ) - super(InsertionOrderSetData, self).update(values) + return super().update(values) class _InsertionOrderSetData(metaclass=RenamedClass): @@ -1800,73 +1865,38 @@ class SortedSetData(_SortedSetMixin, OrderedSetData): Public Class Attributes: """ - __slots__ = ('_is_sorted',) - - def __init__(self, component): - # An empty set is sorted... - self._is_sorted = True - OrderedSetData.__init__(self, component=component) + __slots__ = () def _iter_impl(self): """ Return an iterator for the set. """ - if not self._is_sorted: - self._sort() - return super(SortedSetData, self)._iter_impl() + if self._ordered_values is None: + self._rebuild_ordered_values() + return iter(self._ordered_values) def __reversed__(self): - if not self._is_sorted: - self._sort() - return super(SortedSetData, self).__reversed__() + if self._ordered_values is None: + self._rebuild_ordered_values() + return reversed(self._ordered_values) - def _add_impl(self, value): - # Note that the sorted status has no bearing on insertion, - # so there is no reason to check if the data is correctly sorted - self._values[value] = len(self._values) - self._ordered_values.append(value) - self._is_sorted = False + def _update_impl(self, values): + for val in values: + self._values[val] = None + self._ordered_values = None # Note: removing data does not affect the sorted flag # def remove(self, val): # def discard(self, val): - def clear(self): - super(SortedSetData, self).clear() - self._is_sorted = True - - def at(self, index): - """ - Return the specified member of the set. - - The public Set API is 1-based, even though the - internal _lookup and _values are (pythonically) 0-based. - """ - if not self._is_sorted: - self._sort() - return super(SortedSetData, self).at(index) - - def ord(self, item): - """ - Return the position index of the input value. - - Note that Pyomo Set objects have positions starting at 1 (not 0). - - If the search item is not in the Set, then an IndexError is raised. - """ - if not self._is_sorted: - self._sort() - return super(SortedSetData, self).ord(item) - def sorted_data(self): return self.data() - def _sort(self): - self._ordered_values = list( - self.parent_component()._sort_fcn(self._ordered_values) - ) - self._values = {j: i for i, j in enumerate(self._ordered_values)} - self._is_sorted = True + def _rebuild_ordered_values(self): + _set = self._values + self._ordered_values = list(self.parent_component()._sort_fcn(_set)) + for i, v in enumerate(self._ordered_values): + _set[v] = i class _SortedSetData(metaclass=RenamedClass): @@ -1974,10 +2004,11 @@ class Set(IndexedComponent): """ - class End(object): - pass + class SetEndType(type): + def __hash__(self): + raise Set.End() - class Skip(object): + class End(Exception, metaclass=SetEndType): pass class InsertionOrder(object): @@ -1988,6 +2019,7 @@ class SortedOrder(object): _ValidOrderedAuguments = {True, False, InsertionOrder, SortedOrder} _UnorderedInitializers = {set} + _SetEndEncountered = False @overload def __new__(cls: Type[Set], *args, **kwds) -> Union[SetData, IndexedSet]: ... @@ -2222,21 +2254,6 @@ def _getitem_when_not_present(self, index): if _d is UnknownSetDimen and domain is not None and domain.dimen is not None: _d = domain.dimen - if self._init_values is not None: - self._init_values._dimen = _d - try: - _values = self._init_values(_block, index) - except TuplizeError as e: - raise ValueError( - str(e) % (self._name, "[%s]" % index if self.is_indexed() else "") - ) - - if _values is Set.Skip: - return - elif _values is None: - raise ValueError( - "Set rule or initializer returned None instead of Set.Skip" - ) if index is None and not self.is_indexed(): obj = self._data[index] = self else: @@ -2258,55 +2275,35 @@ def _getitem_when_not_present(self, index): obj._validate = self._init_validate if self._init_filter is not None: try: - _filter = Initializer(self._init_filter(_block, index)) - if _filter.constant(): + obj._filter = Initializer(self._init_filter(_block, index)) + if obj._filter.constant(): # _init_filter was the actual filter function; use it. - _filter = self._init_filter + obj._filter = self._init_filter except: # We will assume any exceptions raised when getting the # filter for this index indicate that the function # should have been passed directly to the underlying sets. - _filter = self._init_filter + obj._filter = self._init_filter else: - _filter = None + obj._filter = None if self._init_values is not None: - # _values was initialized above... - if obj.isordered() and type(_values) in Set._UnorderedInitializers: - logger.warning( - "Initializing ordered Set %s with a fundamentally " - "unordered data source (type: %s). This WILL potentially " - "lead to nondeterministic behavior in Pyomo" - % (self.name, type(_values).__name__) - ) - # Special case: set operations that are not first attached - # to the model must be constructed. - if isinstance(_values, SetOperator): - _values.construct() + # record the user-provided dimen in the initializer + self._init_values._dimen = _d try: - val_iter = iter(_values) - except TypeError: - logger.error( - "Initializer for Set %s%s returned non-iterable object " - "of type %s." - % ( - self.name, - ("[%s]" % (index,) if self.is_indexed() else ""), - ( - _values - if _values.__class__ is type - else type(_values).__name__ - ), - ) + _values = self._init_values(_block, index) + except TuplizeError as e: + raise ValueError( + str(e) % (self._name, "[%s]" % index if self.is_indexed() else "") ) - raise - for val in val_iter: - if val is Set.End: - break - if _filter is None or _filter(_block, val): - obj.add(val) - # We defer adding the filter until now so that add() doesn't - # call it a second time. - obj._filter = _filter + if _values is Set.Skip: + del self._data[index] + return + elif _values is None: + raise ValueError( + "Set rule or initializer returned None instead of Set.Skip" + ) + + obj.set_value(_values) return obj @staticmethod diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index f62589a6873..b5e47975c00 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -3763,8 +3763,8 @@ def I_init(m): m = ConcreteModel() m.I = Set(initialize={1, 3, 2, 4}) ref = ( - "Initializing ordered Set I with a " - "fundamentally unordered data source (type: set)." + 'Calling set_value() on an insertion order Set with a fundamentally ' + 'unordered data source (type: set).' ) self.assertIn(ref, output.getvalue()) self.assertEqual(m.I.sorted_data(), (1, 2, 3, 4)) @@ -3877,12 +3877,13 @@ def _verify(_s, _l): m.I.add(4) _verify(m.I, [1, 3, 2, 4]) + N = len(m.I) output = StringIO() with LoggingIntercept(output, 'pyomo.core'): m.I.add(3) - self.assertEqual( - output.getvalue(), "Element 3 already exists in Set I; no action taken\n" - ) + # In Pyomo <= 6.7.3 duplicate values logged a warning. + self.assertEqual(output.getvalue(), "") + self.assertEqual(N, len(m.I)) _verify(m.I, [1, 3, 2, 4]) m.I.remove(3) @@ -3959,12 +3960,13 @@ def _verify(_s, _l): m.I.add(4) _verify(m.I, [1, 2, 3, 4]) + N = len(m.I) output = StringIO() with LoggingIntercept(output, 'pyomo.core'): m.I.add(3) - self.assertEqual( - output.getvalue(), "Element 3 already exists in Set I; no action taken\n" - ) + # In Pyomo <= 6.7.3 duplicate values logged a warning. + self.assertEqual(output.getvalue(), "") + self.assertEqual(N, len(m.I)) _verify(m.I, [1, 2, 3, 4]) m.I.remove(3) @@ -4052,12 +4054,13 @@ def _verify(_s, _l): m.I.add(4) _verify(m.I, [1, 2, 3, 4]) + N = len(m.I) output = StringIO() with LoggingIntercept(output, 'pyomo.core'): m.I.add(3) - self.assertEqual( - output.getvalue(), "Element 3 already exists in Set I; no action taken\n" - ) + # In Pyomo <= 6.7.3 duplicate values logged a warning. + self.assertEqual(output.getvalue(), "") + self.assertEqual(N, len(m.I)) _verify(m.I, [1, 2, 3, 4]) m.I.remove(3) @@ -4248,24 +4251,23 @@ def test_add_filter_validate(self): self.assertIn(1, m.I) self.assertIn(1.0, m.I) + N = len(m.I) output = StringIO() with LoggingIntercept(output, 'pyomo.core'): self.assertFalse(m.I.add(1)) - self.assertEqual( - output.getvalue(), "Element 1 already exists in Set I; no action taken\n" - ) + # In Pyomo <= 6.7.3 duplicate values logged a warning. + self.assertEqual(output.getvalue(), "") + self.assertEqual(N, len(m.I)) output = StringIO() with LoggingIntercept(output, 'pyomo.core'): self.assertFalse(m.I.add((1,))) - self.assertEqual( - output.getvalue(), "Element (1,) already exists in Set I; no action taken\n" - ) + # In Pyomo <= 6.7.3 duplicate values logged a warning. + self.assertEqual(output.getvalue(), "") m.J = Set() # Note that pypy raises a different exception from cpython err = ( - r"Unable to insert '{}' into Set J:\n\tTypeError: " r"((unhashable type: 'dict')|('dict' objects are unhashable))" ) with self.assertRaisesRegex(TypeError, err): @@ -4275,9 +4277,9 @@ def test_add_filter_validate(self): output = StringIO() with LoggingIntercept(output, 'pyomo.core'): self.assertFalse(m.J.add(1)) - self.assertEqual( - output.getvalue(), "Element 1 already exists in Set J; no action taken\n" - ) + # In Pyomo <= 6.7.3 duplicate values logged a warning. + self.assertEqual(output.getvalue(), "") + self.assertEqual(N, len(m.I)) def _l_tri(model, i, j): self.assertIs(model, m) From 5eb9875fe9d0e940fb914d867dc146c2103e8f64 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 25 Jun 2024 14:28:20 -0600 Subject: [PATCH 081/150] implement Set.first() and .last() without forcing _ordered_values construction --- pyomo/core/base/set.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index fcabf6674a4..e6444c48452 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1596,10 +1596,16 @@ def ordered_iter(self): return iter(self) def first(self): - return self.at(1) + try: + return next(iter(self)) + except StopIteration: + raise IndexError(f"{self.name} index out of range") from None def last(self): - return self.at(len(self)) + try: + return next(reversed(self)) + except StopIteration: + raise IndexError(f"{self.name} index out of range") from None def next(self, item, step=1): """ @@ -1745,9 +1751,9 @@ def pop(self): try: ans = self.last() except IndexError: - # Map the index error to a KeyError for consistency with - # set().pop() - raise KeyError('pop from an empty set') + # Map the exception for iterating over an empty dict to a + # KeyError for consistency with set().pop() + raise KeyError('pop from an empty set') from None self.discard(ans) return ans From 4716f9d36bb6c54b3b7b448e7e9976f66f351ec8 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 25 Jun 2024 15:42:19 -0600 Subject: [PATCH 082/150] NFC: apply black --- pyomo/core/tests/unit/test_set.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index b5e47975c00..6fb95a31a59 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -4267,9 +4267,7 @@ def test_add_filter_validate(self): m.J = Set() # Note that pypy raises a different exception from cpython - err = ( - r"((unhashable type: 'dict')|('dict' objects are unhashable))" - ) + err = r"((unhashable type: 'dict')|('dict' objects are unhashable))" with self.assertRaisesRegex(TypeError, err): m.J.add({}) From b6cb09cde03440c599946c376d63abaf4f144b1a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 25 Jun 2024 15:48:26 -0600 Subject: [PATCH 083/150] Restore original ordered set initialization warning --- pyomo/core/base/set.py | 15 ++++++++++++++- pyomo/core/tests/unit/test_set.py | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index e6444c48452..040c66ffd3a 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1383,6 +1383,9 @@ def set_value(self, val): self.clear() self.update(val) + def _initialize(self, val): + self.update(val) + def update(self, values): # _values was initialized above... # @@ -1822,6 +1825,16 @@ class InsertionOrderSetData(OrderedSetData): __slots__ = () + def _initialize(self, val): + if type(val) in Set._UnorderedInitializers: + logger.warning( + "Initializing ordered Set %s with " + "a fundamentally unordered data source (type: %s). " + "This WILL potentially lead to nondeterministic behavior " + "in Pyomo" % (self.name, type(val).__name__) + ) + super().update(val) + def set_value(self, val): if type(val) in Set._UnorderedInitializers: logger.warning( @@ -2309,7 +2322,7 @@ def _getitem_when_not_present(self, index): "Set rule or initializer returned None instead of Set.Skip" ) - obj.set_value(_values) + obj._initialize(_values) return obj @staticmethod diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 6fb95a31a59..ff0aaa5600b 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -3763,7 +3763,7 @@ def I_init(m): m = ConcreteModel() m.I = Set(initialize={1, 3, 2, 4}) ref = ( - 'Calling set_value() on an insertion order Set with a fundamentally ' + 'Initializing ordered Set I with a fundamentally ' 'unordered data source (type: set).' ) self.assertIn(ref, output.getvalue()) From d5e4f05f58d4c25f831130040b4676b6f48555fb Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 25 Jun 2024 16:54:25 -0600 Subject: [PATCH 084/150] Restore previous Set add()/update() return values --- pyomo/core/base/set.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 040c66ffd3a..f183d906607 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1365,7 +1365,9 @@ def filter(self): return self._filter def add(self, *values): - return self.update(values) + N = len(self) + self.update(values) + return len(self) - N def _update_impl(self, values): self._values.update(values) @@ -1426,14 +1428,12 @@ def update(self, values): if self._validate is not None: val_iter = self._cb_validate(self._validate, self.parent_block(), val_iter) - nOrig = len(self) # We wrap this check in a try-except because some values # (like lists) are not hashable and can raise exceptions. try: self._update_impl(val_iter) except Set.End: pass - return len(self) - nOrig def pop(self): return self._values.pop() @@ -1854,7 +1854,7 @@ def update(self, values): "This WILL potentially lead to nondeterministic behavior " "in Pyomo" % (type(values).__name__,) ) - return super().update(values) + super().update(values) class _InsertionOrderSetData(metaclass=RenamedClass): From d0338bc2e960aa12fbd4877cba87d4b7081de1b4 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 25 Jun 2024 17:16:42 -0600 Subject: [PATCH 085/150] Rework Set.End to avoid conflict with dask.multiprocessing (which attempts to hash all known exception types) --- pyomo/core/base/set.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index f183d906607..a691973a1a7 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1432,7 +1432,7 @@ def update(self, values): # (like lists) are not hashable and can raise exceptions. try: self._update_impl(val_iter) - except Set.End: + except Set._SetEndException: pass def pop(self): @@ -2023,11 +2023,14 @@ class Set(IndexedComponent): """ - class SetEndType(type): + class _SetEndException(Exception): + pass + + class _SetEndType(type): def __hash__(self): - raise Set.End() + raise Set._SetEndException() - class End(Exception, metaclass=SetEndType): + class End(metaclass=_SetEndType): pass class InsertionOrder(object): From 1ad9d4eebdf34180e1045bbf1c20d6e097958217 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 25 Jun 2024 17:18:06 -0600 Subject: [PATCH 086/150] Remove unused class attribute --- pyomo/core/base/set.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index a691973a1a7..1ae866e84d7 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -2041,7 +2041,6 @@ class SortedOrder(object): _ValidOrderedAuguments = {True, False, InsertionOrder, SortedOrder} _UnorderedInitializers = {set} - _SetEndEncountered = False @overload def __new__(cls: Type[Set], *args, **kwds) -> Union[SetData, IndexedSet]: ... From 1c0a0307e00848f03f4cf5c46efd8c4a114b1adf Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Wed, 26 Jun 2024 08:07:30 -0400 Subject: [PATCH 087/150] Fix warning and improve session handling --- pyomo/solvers/plugins/solvers/SAS.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index eb5014fbd1a..fbbd6e5c3eb 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -28,7 +28,7 @@ ) from pyomo.common.tempfiles import TempfileManager from pyomo.core.base import Var -from pyomo.core.base.block import _BlockData +from pyomo.core.base.block import BlockData from pyomo.core.kernel.block import IBlock from pyomo.common.log import LogStream from pyomo.common.tee import capture_output, TeeStream @@ -157,7 +157,7 @@ def _presolve(self, *args, **kwds): # Store the model, too bad this is not done in the base class for arg in args: - if isinstance(arg, (_BlockData, IBlock)): + if isinstance(arg, (BlockData, IBlock)): # Store the instance self._instance = arg self._vars = [] @@ -282,16 +282,17 @@ def __init__(self, **kwds): self._python_api_exists = True self._sas.logger.setLevel(logger.level) - # Create the session only as its needed - self._sas_session = None - # Store other options for the SAS session self._session_options = kwds + # Create the session + self._sas_session = self._sas.SASsession(**self._session_options) + def __del__(self): # Close the session, if we created one if self._sas_session: self._sas_session.endsas() + del self._sas_session def _create_statement_str(self, statement): """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" @@ -369,7 +370,10 @@ def _apply_solver(self): sas_options = "option notes nonumber nodate nosource pagesize=max;" # Get the current SAS session, submit the code and return the results - sas = self._sas_session = self._sas.SASsession(**self._session_options) + if not self._sas_session: + sas = self._sas_session = self._sas.SASsession(**self._session_options) + else: + sas = self._sas_session # Find the version of 9.4 we are using self._sasver = sas.sasver @@ -576,14 +580,16 @@ def __init__(self, **kwds): else: self._python_api_exists = True - # Create the session only as its needed - self._sas_session = None self._session_options = kwds + # Create the session + self._sas_session = self._sas.CAS(**self._session_options) + def __del__(self): # Close the session, if we created one if self._sas_session: self._sas_session.close() + del self._sas_session def _uploadMpsFile(self, s, unique): # Declare a unique table name for the mps table From 5d0e19c85ec64c9fe65246f240d1e84c92a64bb7 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Wed, 26 Jun 2024 09:11:51 -0600 Subject: [PATCH 088/150] Update codecov yml to new version --- .codecov.yml => codecov.yml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) rename .codecov.yml => codecov.yml (54%) diff --git a/.codecov.yml b/codecov.yml similarity index 54% rename from .codecov.yml rename to codecov.yml index 6b88f948fe1..1184eaf877d 100644 --- a/.codecov.yml +++ b/codecov.yml @@ -1,19 +1,21 @@ +codecov: + notify: + # GHA: 5, Jenkins: 11 + # Accurate as of Jun 26, 2024 + # Potential to change when Python versions change + after_n_builds: 16 + wait_for_ci: true coverage: - range: "50...100" + range: + - 50.0 + - 100.0 status: + patch: + default: + # Force patches to be covered at the level of the codebase + threshold: 0.0 project: default: # Allow overall coverage to drop to avoid failures due to code # cleanup or CI unavailability/lag - threshold: 5% - patch: - default: - # Force patches to be covered at the level of the codebase - threshold: 0% -# ci: -# - !ci.appveyor.com -codecov: - notify: - # GHA: 4, Jenkins: 8 - after_n_builds: 12 # all - wait_for_ci: yes + threshold: 5.0 From e15ec25c554fae58bf936c65cf01f8bf010b1861 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Wed, 26 Jun 2024 09:13:34 -0600 Subject: [PATCH 089/150] Unpinning 0.6.0 --- .github/workflows/test_branches.yml | 6 ------ .github/workflows/test_pr_and_main.yml | 6 ------ 2 files changed, 12 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 2689b97746b..8ba04eec466 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -855,9 +855,6 @@ jobs: token: ${{ secrets.PYOMO_CODECOV_TOKEN }} name: ${{ matrix.TARGET }} flags: ${{ matrix.TARGET }} - # downgrading after v0.7.0 broke tokenless upload - # see codecov/codecov-action#1487 - version: v0.6.0 fail_ci_if_error: true - name: Upload other coverage reports @@ -870,7 +867,4 @@ jobs: token: ${{ secrets.PYOMO_CODECOV_TOKEN }} name: ${{ matrix.TARGET }}/other flags: ${{ matrix.TARGET }},other - # downgrading after v0.7.0 broke tokenless upload - # see codecov/codecov-action#1487 - version: v0.6.0 fail_ci_if_error: true diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index c804372b18a..bdf1f7e1aa5 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -899,9 +899,6 @@ jobs: token: ${{ secrets.PYOMO_CODECOV_TOKEN }} name: ${{ matrix.TARGET }} flags: ${{ matrix.TARGET }} - # downgrading after v0.7.0 broke tokenless upload - # see codecov/codecov-action#1487 - version: v0.6.0 fail_ci_if_error: true - name: Upload other coverage reports @@ -914,7 +911,4 @@ jobs: token: ${{ secrets.PYOMO_CODECOV_TOKEN }} name: ${{ matrix.TARGET }}/other flags: ${{ matrix.TARGET }},other - # downgrading after v0.7.0 broke tokenless upload - # see codecov/codecov-action#1487 - version: v0.6.0 fail_ci_if_error: true From 498312e587c424fa58081238895966b03b4b255f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Magalh=C3=A3es?= Date: Sat, 29 Jun 2024 00:57:14 +0200 Subject: [PATCH 090/150] Edits after review. --- examples/pyomo/tutorials/set.dat | 6 +-- pyomo/core/base/set.py | 4 +- pyomo/core/tests/unit/test_set.py | 85 ++++++++----------------------- 3 files changed, 26 insertions(+), 69 deletions(-) diff --git a/examples/pyomo/tutorials/set.dat b/examples/pyomo/tutorials/set.dat index d136397f54e..e2ad04122d8 100644 --- a/examples/pyomo/tutorials/set.dat +++ b/examples/pyomo/tutorials/set.dat @@ -14,8 +14,8 @@ set M := 1 3; set S[2] := 1 3; set S[5] := 2 3; -set X[2] := 1; -set X[5] := 2 3; - set T[2] := 1 3; set T[5] := 2 3; + +set X[2] := 1; +set X[5] := 2 3; \ No newline at end of file diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 666332124f1..804c02f7991 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -2218,8 +2218,8 @@ def _getitem_when_not_present(self, index): _d = None domain = self._init_domain(_block, index, self) - if domain is not None and hasattr(domain, "construct"): - domain.construct() + if domain is not None: + domain.parent_component().construct() if _d is UnknownSetDimen and domain is not None and domain.dimen is not None: _d = domain.dimen diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index a1c502a86d0..012669a3484 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6452,52 +6452,33 @@ def test_issue_3284(self): # try adding elements to test the domains (1 compatible, 1 incompatible) # set subset_A problem.subset_A.add(1) - error_raised = False - try: + error_message = ( + "Cannot add value %s to Set %s.\n" + "\tThe value is not in the domain %s" + % (4, 'subset_A', 'A') + ) + with self.assertRaisesRegex(ValueError, error_message): problem.subset_A.add(4) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_B problem.subset_B.add((3, 4)) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_B.add((7, 8)) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_C problem.subset_C[2].add(7) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value 8 to Set"): problem.subset_C[2].add(8) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_D - problem.subset_D[(3, 4)].add(3) - error_raised = False - try: - problem.subset_D[(3, 4)].add(4) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) + problem.subset_D[(5, 6)].add(9) + with self.assertRaisesRegex(ValueError, ".*Cannot add value 2 to Set"): + problem.subset_D[(3, 4)].add(2) # set E problem.E[1].add(2) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value 4 to Set"): problem.E[1].add(4) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set F problem.F[(1, 2, 3)].add((3, 4)) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.F[(4, 5, 6)].add((4, 3)) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # check them self.assertEqual(list(problem.A), [1, 2, 3]) self.assertEqual(list(problem.B), [(1, 2), (3, 4), (5, 6)]) @@ -6513,8 +6494,8 @@ def test_issue_3284(self): self.assertEqual(list(problem.subset_C[2]), [4, 7]) self.assertEqual(list(problem.subset_C[3]), [3, 8]) self.assertEqual(list(problem.subset_D[(1, 2)]), [1, 5]) - self.assertEqual(list(problem.subset_D[(3, 4)]), [3]) - self.assertEqual(list(problem.subset_D[(5, 6)]), [6]) + self.assertEqual(list(problem.subset_D[(3, 4)]), []) + self.assertEqual(list(problem.subset_D[(5, 6)]), [6, 9]) self.assertEqual(list(problem.E[0]), [1, 2]) self.assertEqual(list(problem.E[1]), [3, 2]) self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2), (3, 4)]) @@ -6581,52 +6562,28 @@ def test_issue_3284(self): # try adding elements to test the domains (1 compatible, 1 incompatible) # set subset_A problem.subset_A.add(5) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_A.add(6) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_B problem.subset_B.add((7, 8)) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_B.add((3, 4)) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_C problem.subset_C[4].add(4) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_C[4].add(9) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_D problem.subset_D[(1, 2)].add(2) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_D[(1, 2)].add(3) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set E problem.E[1].add(4) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.E[1].add(1) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set F problem.F[(1, 2, 3)].add((7, 8)) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.F[(4, 5, 6)].add((4, 3)) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # check them self.assertEqual(list(problem.A), [3, 4, 5]) self.assertEqual(list(problem.B), [(1, 2), (7, 8)]) From bbfd6e11bb4504626b0159c10c841c890a1001cd Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Fri, 28 Jun 2024 20:54:12 -0600 Subject: [PATCH 091/150] raise descriptive runtimeerror instead of asserting --- pyomo/contrib/incidence_analysis/scc_solver.py | 9 ++++++++- .../tests/test_scc_solver.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/incidence_analysis/scc_solver.py b/pyomo/contrib/incidence_analysis/scc_solver.py index 378647c190c..ca501d88aa0 100644 --- a/pyomo/contrib/incidence_analysis/scc_solver.py +++ b/pyomo/contrib/incidence_analysis/scc_solver.py @@ -66,7 +66,14 @@ def generate_strongly_connected_components( ) ) - assert len(variables) == len(constraints) + if len(variables) != len(constraints): + nvar = len(variables) + ncon = len(constraints) + raise RuntimeError( + f"generate_strongly_connected_components only supports variables" + f" systems with the same numbers of variables and equality constraints." + f" Got {nvar} variables and {ncon} constraints." + ) if igraph is None: igraph = IncidenceGraphInterface() diff --git a/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py b/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py index b75f93e4a12..293d72ec1f0 100644 --- a/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py +++ b/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py @@ -501,5 +501,23 @@ def test_with_inequalities(self): self.assertEqual(m.x[3].value, 1.0) +@unittest.skipUnless(scipy_available, "SciPy is not available") +@unittest.skipUnless(networkx_available, "NetworkX is not available") +class TestExceptions(unittest.TestCase): + + def test_nonsquare_system(self): + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2], initialize=1) + m.eq = pyo.Constraint(expr=m.x[1] + m.x[2] == 1) + + msg = "Got 2 variables and 1 constraints" + with self.assertRaisesRegex(RuntimeError, msg): + list( + generate_strongly_connected_components( + constraints=[m.eq], variables=[m.x[1], m.x[2]] + ) + ) + + if __name__ == "__main__": unittest.main() From 74c4162a8807c917f1a53dc7d388445f911804f2 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Fri, 28 Jun 2024 20:57:28 -0600 Subject: [PATCH 092/150] add comment about assert --- pyomo/contrib/incidence_analysis/scc_solver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyomo/contrib/incidence_analysis/scc_solver.py b/pyomo/contrib/incidence_analysis/scc_solver.py index ca501d88aa0..508758cce1e 100644 --- a/pyomo/contrib/incidence_analysis/scc_solver.py +++ b/pyomo/contrib/incidence_analysis/scc_solver.py @@ -85,6 +85,8 @@ def generate_strongly_connected_components( subsets, include_fixed=include_fixed ): # TODO: How does len scale for reference-to-list? + # If this assert fails, it may be due to a bug in block_triangularize + # or generate_subsystem_block. assert len(block.vars) == len(block.cons) yield (block, inputs) From 5b594e71ec65646d4ec859c1101b53c3627e45e3 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Fri, 28 Jun 2024 20:58:33 -0600 Subject: [PATCH 093/150] remove whitespace --- pyomo/contrib/incidence_analysis/tests/test_scc_solver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py b/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py index 293d72ec1f0..ef4853d7e9a 100644 --- a/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py +++ b/pyomo/contrib/incidence_analysis/tests/test_scc_solver.py @@ -504,7 +504,6 @@ def test_with_inequalities(self): @unittest.skipUnless(scipy_available, "SciPy is not available") @unittest.skipUnless(networkx_available, "NetworkX is not available") class TestExceptions(unittest.TestCase): - def test_nonsquare_system(self): m = pyo.ConcreteModel() m.x = pyo.Var([1, 2], initialize=1) From 34b131ebd905c62eb69544ad41af0e53030eacda Mon Sep 17 00:00:00 2001 From: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:29:01 -0600 Subject: [PATCH 094/150] Changing date to re-trigger all jobs from scratch --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index 1184eaf877d..5c2e8b4177d 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,7 +1,7 @@ codecov: notify: # GHA: 5, Jenkins: 11 - # Accurate as of Jun 26, 2024 + # Accurate as of July 1, 2024 # Potential to change when Python versions change after_n_builds: 16 wait_for_ci: true From 6cce08fba429f16c6294fd8ff4a463a09d802af6 Mon Sep 17 00:00:00 2001 From: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:44:42 -0600 Subject: [PATCH 095/150] Change `_sas_session` check --- pyomo/solvers/plugins/solvers/SAS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index fbbd6e5c3eb..bc46f1c7465 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -290,7 +290,7 @@ def __init__(self, **kwds): def __del__(self): # Close the session, if we created one - if self._sas_session: + if hasattr(self, '_sas_session'): self._sas_session.endsas() del self._sas_session From ee9d446a882642dd8b7c2e269b9bd6a77a846169 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Tue, 2 Jul 2024 15:44:57 -0600 Subject: [PATCH 096/150] Add a try-except block around session creation --- pyomo/solvers/plugins/solvers/SAS.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index bc46f1c7465..80ec8d2877d 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -286,11 +286,14 @@ def __init__(self, **kwds): self._session_options = kwds # Create the session - self._sas_session = self._sas.SASsession(**self._session_options) + try: + self._sas_session = self._sas.SASsession(**self._session_options) + except: + self._sas_session = None def __del__(self): # Close the session, if we created one - if hasattr(self, '_sas_session'): + if self._sas_session: self._sas_session.endsas() del self._sas_session @@ -583,7 +586,10 @@ def __init__(self, **kwds): self._session_options = kwds # Create the session - self._sas_session = self._sas.CAS(**self._session_options) + try: + self._sas_session = self._sas.CAS(**self._session_options) + except: + self._sas_session = None def __del__(self): # Close the session, if we created one From 0af995b5eeea07e59f014786d8c60e00ed82da8c Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Wed, 3 Jul 2024 07:45:43 -0600 Subject: [PATCH 097/150] Change uuid import to attempt_import --- pyomo/solvers/plugins/solvers/SAS.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 80ec8d2877d..865efa36dc3 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -12,13 +12,13 @@ import logging import sys from os import stat -from uuid import uuid4 from abc import ABC, abstractmethod from io import StringIO from pyomo.opt.base import ProblemFormat, ResultsFormat, OptSolver from pyomo.opt.base.solvers import SolverFactory from pyomo.common.collections import Bunch +from pyomo.common.dependencies import attempt_import from pyomo.opt.results import ( SolverResults, SolverStatus, @@ -34,6 +34,7 @@ from pyomo.common.tee import capture_output, TeeStream +uuid, uuid_available = attempt_import('uuid') logger = logging.getLogger("pyomo.solvers") @@ -338,7 +339,7 @@ def _apply_solver(self): rootnode_str = self._create_statement_str("rootnode") # Get a unique identifier, always use the same with different prefixes - unique = uuid4().hex[:16] + unique = uuid.uuid4().hex[:16] # Create unique filename for output datasets primalout_dataset_name = "pout" + unique From 945082fde32a4e2fedd4f6e5870d85500a7e5ff4 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Wed, 3 Jul 2024 10:41:53 -0600 Subject: [PATCH 098/150] Enable codecovcli uploader in jenkins script --- .jenkins.sh | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index 696847fd92c..60ece9ca63d 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -20,8 +20,11 @@ # # CODECOV_TOKEN: the token to use when uploading results to codecov.io # -# CODECOV_ARGS: additional arguments to pass to the codecov uploader -# (e.g., to support SSL certificates) +# CODECOV_SOURCE_BRANCH: passed to the 'codecov-cli' command; branch of Pyomo +# (e.g., to enable correct codecov uploads) +# +# CODECOV_REPO_OWNER: passed to the 'codecov-cli' command; owner of repo +# (e.g., to enable correct codecov uploads) # # DISABLE_COVERAGE: if nonempty, then coverage analysis is disabled # @@ -208,16 +211,23 @@ if test -z "$MODE" -o "$MODE" == test; then coverage xml else CODECOV_JOB_NAME=`echo ${JOB_NAME} | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/'`.$BUILD_NUMBER.$python + if test -z "$CODECOV_REPO_OWNER"; then + CODECOV_REPO_OWNER="pyomo" + fi + if test -z "CODECOV_SOURCE_BRANCH"; then + CODECOV_SOURCE_BRANCH="main" + fi i=0 while /bin/true; do i=$[$i+1] echo "Uploading coverage to codecov (attempt $i)" - codecov -X gcovcodecov -X gcov -X s3 --no-color \ - -t $CODECOV_TOKEN --root `pwd` -e OS,python \ - --name $CODECOV_JOB_NAME $CODECOV_ARGS \ - | tee .cover.upload - if test $? == 0 -a `grep -i error .cover.upload \ - | grep -v branch= | wc -l` -eq 0; then + codecovcli -v upload-process --sha $PYOMO_SOURCE_SHA \ + --fail-on-error --git-service github --token $CODECOV_TOKEN \ + --slug pyomo/pyomo --file coverage.xml --disable-search \ + --name $CODECOV_JOB_NAME \ + --branch $CODECOV_REPO_OWNER:$CODECOV_SOURCE_BRANCH \ + --env OS,python --network-root-folder `pwd` --plugin noop + if test $? == 0; then break elif test $i -ge 4; then exit 1 From 51568d1a2e92685d640f96e66ddedee3c35727b8 Mon Sep 17 00:00:00 2001 From: Bethany Nicholson Date: Wed, 3 Jul 2024 11:59:00 -0600 Subject: [PATCH 099/150] Fix typos from code review --- pyomo/solvers/plugins/solvers/SAS.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 865efa36dc3..f2cfe279fdc 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -397,12 +397,12 @@ def _apply_solver(self): ) upload_pin = True - # Using a function call to make it easier to moch the version check + # Using a function call to make it easier to mock the version check major_version = self.sas_version()[0] minor_version = self.sas_version().split("M", 1)[1][0] if major_version == "9" and int(minor_version) < 5: raise NotImplementedError( - "Support for SAS 9.4 M4 and earlier is no implemented." + "Support for SAS 9.4 M4 and earlier is not implemented." ) elif major_version == "9" and int(minor_version) == 5: # In 9.4M5 we have to create an MPS data set from an MPS file first @@ -529,7 +529,7 @@ def _apply_solver(self): if proc == "OPTLP": # Convert primal out data set to variable dictionary - # Use panda functions for efficiency + # Use pandas functions for efficiency primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] primal_out = primal_out.set_index("_VAR_", drop=True) primal_out = primal_out.rename( From 079b7bdc83df3b029b4c1d1eb707f9b569e0642d Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Wed, 3 Jul 2024 15:53:38 -0600 Subject: [PATCH 100/150] Changing date to reset test suites --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index 5c2e8b4177d..318a907905f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,7 +1,7 @@ codecov: notify: # GHA: 5, Jenkins: 11 - # Accurate as of July 1, 2024 + # Accurate as of July 3, 2024 # Potential to change when Python versions change after_n_builds: 16 wait_for_ci: true From 4f9acf0427f9715f21e5e678aa498a64d1e595e9 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 7 Jul 2024 05:48:24 -0600 Subject: [PATCH 101/150] Explicitly create coverage.xml file --- .jenkins.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index 60ece9ca63d..51dd92ba123 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -205,11 +205,10 @@ if test -z "$MODE" -o "$MODE" == test; then # Note, that the PWD should still be $WORKSPACE/pyomo # coverage combine || exit 1 - coverage report -i + coverage report -i || exit 1 + coverage xml -i || exit 1 export OS=`uname` - if test -z "$CODECOV_TOKEN"; then - coverage xml - else + if test -n "$CODECOV_TOKEN"; then CODECOV_JOB_NAME=`echo ${JOB_NAME} | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/'`.$BUILD_NUMBER.$python if test -z "$CODECOV_REPO_OWNER"; then CODECOV_REPO_OWNER="pyomo" From 2167033cbea0143aa3c31fce30a6fd731bc9de73 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 7 Jul 2024 05:57:22 -0600 Subject: [PATCH 102/150] Fix spelling caught by new typos release --- .github/workflows/typos.toml | 2 ++ pyomo/common/config.py | 2 +- pyomo/common/tests/test_dependencies.py | 2 +- pyomo/core/base/set.py | 6 +++--- pyomo/core/plugins/transform/radix_linearization.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/typos.toml b/.github/workflows/typos.toml index 7a38164898b..e24957290a5 100644 --- a/.github/workflows/typos.toml +++ b/.github/workflows/typos.toml @@ -67,4 +67,6 @@ RO = "RO" EOF = "EOF" # Ignore lst as shorthand for list lst = "lst" +# Abbreviation of gamma (used in stochpdegas1_automatic.py) +gam = "gam" # AS NEEDED: Add More Words Below diff --git a/pyomo/common/config.py b/pyomo/common/config.py index f9c3a725bb8..ebba2f2732a 100644 --- a/pyomo/common/config.py +++ b/pyomo/common/config.py @@ -996,7 +996,7 @@ class will still create ``c`` instances that only have the single :py:meth:`generate_documentation()`. The simplest is :py:meth:`display()`, which prints out the current values of the configuration object (and if it is a container type, all of it's -children). :py:meth:`generate_yaml_template` is simular to +children). :py:meth:`generate_yaml_template` is similar to :py:meth:`display`, but also includes the description fields as formatted comments. diff --git a/pyomo/common/tests/test_dependencies.py b/pyomo/common/tests/test_dependencies.py index 31f9520b613..6aedc428244 100644 --- a/pyomo/common/tests/test_dependencies.py +++ b/pyomo/common/tests/test_dependencies.py @@ -209,7 +209,7 @@ def test_and_or(self): _and_or = avail0 & avail1 | avail2 self.assertTrue(_and_or) - # Verify operator prescedence + # Verify operator precedence _or_and = avail0 | avail2 & avail2 self.assertTrue(_or_and) _or_and = (avail0 | avail2) & avail2 diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 8b7c2a246d6..c0d427747c5 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1986,7 +1986,7 @@ class InsertionOrder(object): class SortedOrder(object): pass - _ValidOrderedAuguments = {True, False, InsertionOrder, SortedOrder} + _ValidOrderedArguments = {True, False, InsertionOrder, SortedOrder} _UnorderedInitializers = {set} @overload @@ -2015,7 +2015,7 @@ def __new__(cls, *args, **kwds): ordered = kwds.get('ordered', Set.InsertionOrder) if ordered is True: ordered = Set.InsertionOrder - if ordered not in Set._ValidOrderedAuguments: + if ordered not in Set._ValidOrderedArguments: if inspect.isfunction(ordered): ordered = Set.SortedOrder else: @@ -2032,7 +2032,7 @@ def __new__(cls, *args, **kwds): str(_) for _ in sorted_robust( 'Set.' + x.__name__ if isinstance(x, type) else x - for x in Set._ValidOrderedAuguments.union( + for x in Set._ValidOrderedArguments.union( {''} ) ) diff --git a/pyomo/core/plugins/transform/radix_linearization.py b/pyomo/core/plugins/transform/radix_linearization.py index 92270655f31..3cfde28db3c 100644 --- a/pyomo/core/plugins/transform/radix_linearization.py +++ b/pyomo/core/plugins/transform/radix_linearization.py @@ -280,7 +280,7 @@ def _collect_bilinear(self, expr, bilin, quad): if type(expr) is PowExpression and value(expr._args[1]) == 2: # Note: directly testing the value of the exponent above is # safe: we have already verified that this expression is - # polynominal, so the exponent must be constant. + # polynomial, so the exponent must be constant. tmp = ProductExpression() tmp._numerator = [expr._args[0], expr._args[0]] tmp._denominator = [] From fb60c7f0f627bc2c63574d8ea8d6419b127b7605 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 7 Jul 2024 06:10:11 -0600 Subject: [PATCH 103/150] (temporarily) disable checking idaes.org URL --- .github/workflows/test_pr_and_main.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index bdf1f7e1aa5..c3634682462 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -68,9 +68,11 @@ jobs: verbose: true # How many times to retry a failed request (defaults to 1) retry_count: 3 - # Exclude Jenkins because it's behind a firewall; ignore RTD because - # a magically-generated string is triggering a failure - exclude_urls: https://pyomo-jenkins.sandia.gov/,https://pyomo.readthedocs.io/en/%s/errors.html + # Exclude: + # - Jenkins because it's behind a firewall + # - RTD because a magically-generated string triggers failures + # - IDAES.org because the webserver is currently down + exclude_urls: https://pyomo-jenkins.sandia.gov/,https://pyomo.readthedocs.io/en/%s/errors.html,https://idaes.org/ build: From 995209725dc31b462de174421f9c334f6cbe0157 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 7 Jul 2024 06:13:16 -0600 Subject: [PATCH 104/150] (temporarily) disable checking idaes.org URL --- .github/workflows/test_pr_and_main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index c3634682462..0a2a4f51143 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -72,7 +72,7 @@ jobs: # - Jenkins because it's behind a firewall # - RTD because a magically-generated string triggers failures # - IDAES.org because the webserver is currently down - exclude_urls: https://pyomo-jenkins.sandia.gov/,https://pyomo.readthedocs.io/en/%s/errors.html,https://idaes.org/ + exclude_urls: https://pyomo-jenkins.sandia.gov/,https://pyomo.readthedocs.io/en/%s/errors.html,https://idaes.org build: From 9a2704b9df28af91b0d58ecab31125317920b8ba Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 7 Jul 2024 07:45:26 -0600 Subject: [PATCH 105/150] Remove idaes.org skip (site is back up) --- .github/workflows/test_pr_and_main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 0a2a4f51143..0bfd12b998d 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -71,8 +71,7 @@ jobs: # Exclude: # - Jenkins because it's behind a firewall # - RTD because a magically-generated string triggers failures - # - IDAES.org because the webserver is currently down - exclude_urls: https://pyomo-jenkins.sandia.gov/,https://pyomo.readthedocs.io/en/%s/errors.html,https://idaes.org + exclude_urls: https://pyomo-jenkins.sandia.gov/,https://pyomo.readthedocs.io/en/%s/errors.html build: From c787f3707c60160ccac6606a0b8b87f645d277d9 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 8 Jul 2024 06:34:40 -0400 Subject: [PATCH 106/150] fix: Issues in CAS interface --- pyomo/solvers/plugins/solvers/SAS.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index f2cfe279fdc..4e1653ad1b3 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -629,6 +629,7 @@ def _uploadMpsFile(self, s, unique): data=mpscsv_table_name, casOut={"name": mpsdata_table_name, "replace": True}, format="FREE", + maxLength=256 ) # Delete the table we don't need anymore @@ -641,6 +642,7 @@ def _uploadMpsFile(self, s, unique): mpsFileString=mps_file.read(), casout={"name": mpsdata_table_name, "replace": True}, format="FREE", + maxLength=256 ) return mpsdata_table_name @@ -716,7 +718,7 @@ def _apply_solver(self): action = "solveMilp" if self._has_integer_variables() else "solveLp" # Get a unique identifier, always use the same with different prefixes - unique = uuid4().hex[:16] + unique = uuid.uuid4().hex[:16] # Creat the output stream, we want to print to a log string as well as to the console self._log = StringIO() From c2b0be1c56dd5383f1ea4af22c1427ca7091fbb4 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 8 Jul 2024 09:42:01 -0400 Subject: [PATCH 107/150] fix: adjust for black issues --- pyomo/solvers/plugins/solvers/SAS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 4e1653ad1b3..d7b09e29fde 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -629,7 +629,7 @@ def _uploadMpsFile(self, s, unique): data=mpscsv_table_name, casOut={"name": mpsdata_table_name, "replace": True}, format="FREE", - maxLength=256 + maxLength=256, ) # Delete the table we don't need anymore @@ -642,7 +642,7 @@ def _uploadMpsFile(self, s, unique): mpsFileString=mps_file.read(), casout={"name": mpsdata_table_name, "replace": True}, format="FREE", - maxLength=256 + maxLength=256, ) return mpsdata_table_name From 82b89bd6a382083323b07a875347ccc5d3a5ed27 Mon Sep 17 00:00:00 2001 From: Soren Davis Date: Mon, 8 Jul 2024 09:17:01 -0600 Subject: [PATCH 108/150] fix build_linking_constraints error in kernel conic primal_geomean.as_domain() --- pyomo/core/kernel/conic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/core/kernel/conic.py b/pyomo/core/kernel/conic.py index 1bb5f1b6ce8..cfc3e351b4d 100644 --- a/pyomo/core/kernel/conic.py +++ b/pyomo/core/kernel/conic.py @@ -632,7 +632,7 @@ def as_domain(cls, r, x): b = block() b.r = variable_tuple([variable(lb=0) for i in range(len(r))]) b.x = variable() - b.c = _build_linking_constraints(list(r) + [x], list(b.r) + [x]) + b.c = _build_linking_constraints(list(r) + [x], list(b.r) + [b.x]) b.q = cls(r=b.r, x=b.x) return b From 8f311049360a5c4f5f2bba70aafea4794d087642 Mon Sep 17 00:00:00 2001 From: Soren Davis Date: Mon, 8 Jul 2024 10:18:11 -0600 Subject: [PATCH 109/150] fix apparent same issue in dual_geomean --- pyomo/core/kernel/conic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/core/kernel/conic.py b/pyomo/core/kernel/conic.py index cfc3e351b4d..bd78ba310f4 100644 --- a/pyomo/core/kernel/conic.py +++ b/pyomo/core/kernel/conic.py @@ -934,7 +934,7 @@ def as_domain(cls, r, x): b = block() b.r = variable_tuple([variable(lb=0) for i in range(len(r))]) b.x = variable() - b.c = _build_linking_constraints(list(r) + [x], list(b.r) + [x]) + b.c = _build_linking_constraints(list(r) + [x], list(b.r) + [b.x]) b.q = cls(r=b.r, x=b.x) return b From 6c05a17221923a6ef6d9545974e330600c335259 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:17:17 -0600 Subject: [PATCH 110/150] Renaming pseudo-constant to fixed and promoting it to the ExprType enum --- pyomo/repn/parameterized_linear.py | 77 ++++++++++++++---------------- pyomo/repn/util.py | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index ae0856cfe76..16e9ee52a7e 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -10,10 +10,8 @@ # ___________________________________________________________________________ import copy -import enum from pyomo.common.collections import ComponentSet -from pyomo.common.enums import ExtendedEnumType from pyomo.common.numeric_types import native_numeric_types from pyomo.core import Var from pyomo.core.expr.logical_expr import _flattened @@ -40,15 +38,10 @@ import pyomo.repn.linear as linear -class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): - __base_enum__ = ExprType - PSEUDO_CONSTANT = 5 - - -_PSEUDO_CONSTANT = ParameterizedExprType.PSEUDO_CONSTANT -_CONSTANT = ParameterizedExprType.CONSTANT -_LINEAR = ParameterizedExprType.LINEAR -_GENERAL = ParameterizedExprType.GENERAL +_FIXED = ExprType.FIXED +_CONSTANT = ExprType.CONSTANT +_LINEAR = ExprType.LINEAR +_GENERAL = ExprType.GENERAL def _merge_dict(dest_dict, mult, src_dict): @@ -67,7 +60,7 @@ def _merge_dict(dest_dict, mult, src_dict): def to_expression(visitor, arg): - if arg[0] in (_CONSTANT, _PSEUDO_CONSTANT): + if arg[0] in (_CONSTANT, _FIXED): return arg[1] else: return arg[1].to_expression(visitor) @@ -88,7 +81,7 @@ def walker_exitNode(self): elif self.constant.__class__ in native_numeric_types: return _CONSTANT, self.multiplier * self.constant else: - return _PSEUDO_CONSTANT, self.multiplier * self.constant + return _FIXED, self.multiplier * self.constant def to_expression(self, visitor): if self.nonlinear is not None: @@ -131,7 +124,7 @@ def append(self, other): """ _type, other = other - if _type is _CONSTANT or _type is _PSEUDO_CONSTANT: + if _type is _CONSTANT or _type is _FIXED: self.constant += other return @@ -204,7 +197,7 @@ def _before_var(visitor, child): if child in visitor.wrt: # pseudo-constant # We aren't treating this Var as a Var for the purposes of this walker - return False, (_PSEUDO_CONSTANT, child) + return False, (_FIXED, child) # This is a normal situation ParameterizedLinearBeforeChildDispatcher._record_var(visitor, child) ans = visitor.Result() @@ -221,11 +214,11 @@ def _before_var(visitor, child): def _handle_negation_pseudo_constant(visitor, node, arg): - return (_PSEUDO_CONSTANT, -1 * arg[1]) + return (_FIXED, -1 * arg[1]) _exit_node_handlers[NegationExpression].update( - {(_PSEUDO_CONSTANT,): _handle_negation_pseudo_constant} + {(_FIXED,): _handle_negation_pseudo_constant} ) @@ -241,19 +234,19 @@ def _handle_product_constant_constant(visitor, node, arg1, arg2): def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg1[1] * arg2[1] + return _FIXED, arg1[1] * arg2[1] _exit_node_handlers[ProductExpression].update( { (_CONSTANT, _CONSTANT): _handle_product_constant_constant, - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, - (_PSEUDO_CONSTANT, _CONSTANT): _handle_product_pseudo_constant_constant, - (_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, - (_PSEUDO_CONSTANT, _LINEAR): linear._handle_product_constant_ANY, - (_LINEAR, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, - (_PSEUDO_CONSTANT, _GENERAL): linear._handle_product_constant_ANY, - (_GENERAL, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, + (_FIXED, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_product_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _LINEAR): linear._handle_product_constant_ANY, + (_LINEAR, _FIXED): linear._handle_product_ANY_constant, + (_FIXED, _GENERAL): linear._handle_product_constant_ANY, + (_GENERAL, _FIXED): linear._handle_product_ANY_constant, } ) _exit_node_handlers[MonomialTermExpression].update( @@ -266,7 +259,7 @@ def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): def _handle_division_pseudo_constant_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg1[1] / arg2[1] + return _FIXED, arg1[1] / arg2[1] def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): @@ -276,11 +269,11 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): _exit_node_handlers[DivisionExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, - (_PSEUDO_CONSTANT, _CONSTANT): _handle_division_pseudo_constant_constant, - (_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, - (_LINEAR, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, - (_GENERAL, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, + (_FIXED, _FIXED): _handle_division_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_division_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_division_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_division_ANY_pseudo_constant, + (_GENERAL, _FIXED): _handle_division_ANY_pseudo_constant, } ) @@ -290,7 +283,7 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): def _handle_pow_pseudo_constant_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, to_expression(visitor, arg1) ** to_expression( + return _FIXED, to_expression(visitor, arg1) ** to_expression( visitor, arg2 ) @@ -305,13 +298,13 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): _exit_node_handlers[PowExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, - (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_constant, - (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, - (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_nonlinear, - (_PSEUDO_CONSTANT, _LINEAR): _handle_pow_nonlinear, - (_GENERAL, _PSEUDO_CONSTANT): _handle_pow_nonlinear, - (_PSEUDO_CONSTANT, _GENERAL): _handle_pow_nonlinear, + (_FIXED, _FIXED): _handle_pow_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_pow_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_pow_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_pow_nonlinear, + (_FIXED, _LINEAR): _handle_pow_nonlinear, + (_GENERAL, _FIXED): _handle_pow_nonlinear, + (_FIXED, _GENERAL): _handle_pow_nonlinear, } ) @@ -322,13 +315,13 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): def _handle_unary_pseudo_constant(visitor, node, arg): # We override this because we can't blindly use apply_node_operation in this case - return _PSEUDO_CONSTANT, node.create_node_with_local_data( + return _FIXED, node.create_node_with_local_data( (to_expression(visitor, arg),) ) _exit_node_handlers[UnaryFunctionExpression].update( - {(_PSEUDO_CONSTANT,): _handle_unary_pseudo_constant} + {(_FIXED,): _handle_unary_pseudo_constant} ) _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] @@ -397,6 +390,6 @@ def finalizeResult(self, result): return ans ans = self.Result() - assert result[0] in (_CONSTANT, _PSEUDO_CONSTANT) + assert result[0] in (_CONSTANT, _FIXED) ans.constant = result[1] return ans diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index 8d902d0f99a..c8fc6212d08 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -67,6 +67,7 @@ class ExprType(enum.IntEnum): CONSTANT = 0 + FIXED = 5 MONOMIAL = 10 LINEAR = 20 QUADRATIC = 30 From dd1d3b2d66a9eeeb6ee743e90298e639e9d2dd61 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:17:48 -0600 Subject: [PATCH 111/150] Black has an opinion --- pyomo/repn/parameterized_linear.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 16e9ee52a7e..45ff2943b11 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -283,9 +283,7 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): def _handle_pow_pseudo_constant_constant(visitor, node, arg1, arg2): - return _FIXED, to_expression(visitor, arg1) ** to_expression( - visitor, arg2 - ) + return _FIXED, to_expression(visitor, arg1) ** to_expression(visitor, arg2) def _handle_pow_nonlinear(visitor, node, arg1, arg2): @@ -315,9 +313,7 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): def _handle_unary_pseudo_constant(visitor, node, arg): # We override this because we can't blindly use apply_node_operation in this case - return _FIXED, node.create_node_with_local_data( - (to_expression(visitor, arg),) - ) + return _FIXED, node.create_node_with_local_data((to_expression(visitor, arg),)) _exit_node_handlers[UnaryFunctionExpression].update( From a49f8187cfabe463e8a9f0029460088b320caa8f Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 8 Jul 2024 14:36:05 -0600 Subject: [PATCH 112/150] Move exit node handler registrations into a helper function --- pyomo/repn/linear.py | 122 +++++++++++++++++------------------- pyomo/repn/quadratic.py | 133 +++++++++++++++++++--------------------- pyomo/repn/util.py | 10 ++- 3 files changed, 124 insertions(+), 141 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 6d084067511..029fe892b62 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -183,8 +183,6 @@ def to_expression(visitor, arg): return arg[1].to_expression(visitor) -_exit_node_handlers = {} - # # NEGATION handlers # @@ -199,11 +197,6 @@ def _handle_negation_ANY(visitor, node, arg): return arg -_exit_node_handlers[NegationExpression] = { - None: _handle_negation_ANY, - (_CONSTANT,): _handle_negation_constant, -} - # # PRODUCT handlers # @@ -272,16 +265,6 @@ def _handle_product_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[ProductExpression] = { - None: _handle_product_nonlinear, - (_CONSTANT, _CONSTANT): _handle_product_constant_constant, - (_CONSTANT, _LINEAR): _handle_product_constant_ANY, - (_CONSTANT, _GENERAL): _handle_product_constant_ANY, - (_LINEAR, _CONSTANT): _handle_product_ANY_constant, - (_GENERAL, _CONSTANT): _handle_product_ANY_constant, -} -_exit_node_handlers[MonomialTermExpression] = _exit_node_handlers[ProductExpression] - # # DIVISION handlers # @@ -302,13 +285,6 @@ def _handle_division_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[DivisionExpression] = { - None: _handle_division_nonlinear, - (_CONSTANT, _CONSTANT): _handle_division_constant_constant, - (_LINEAR, _CONSTANT): _handle_division_ANY_constant, - (_GENERAL, _CONSTANT): _handle_division_ANY_constant, -} - # # EXPONENTIATION handlers # @@ -345,13 +321,6 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[PowExpression] = { - None: _handle_pow_nonlinear, - (_CONSTANT, _CONSTANT): _handle_pow_constant_constant, - (_LINEAR, _CONSTANT): _handle_pow_ANY_constant, - (_GENERAL, _CONSTANT): _handle_pow_ANY_constant, -} - # # ABS and UNARY handlers # @@ -371,12 +340,6 @@ def _handle_unary_nonlinear(visitor, node, arg): return _GENERAL, ans -_exit_node_handlers[UnaryFunctionExpression] = { - None: _handle_unary_nonlinear, - (_CONSTANT,): _handle_unary_constant, -} -_exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] - # # NAMED EXPRESSION handlers # @@ -395,11 +358,6 @@ def _handle_named_ANY(visitor, node, arg1): return _type, arg1.duplicate() -_exit_node_handlers[Expression] = { - None: _handle_named_ANY, - (_CONSTANT,): _handle_named_constant, -} - # # EXPR_IF handlers # @@ -430,11 +388,6 @@ def _handle_expr_if_nonlinear(visitor, node, arg1, arg2, arg3): return _GENERAL, ans -_exit_node_handlers[Expr_ifExpression] = {None: _handle_expr_if_nonlinear} -for j in (_CONSTANT, _LINEAR, _GENERAL): - for k in (_CONSTANT, _LINEAR, _GENERAL): - _exit_node_handlers[Expr_ifExpression][_CONSTANT, j, k] = _handle_expr_if_const - # # Relational expression handlers # @@ -462,12 +415,6 @@ def _handle_equality_general(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[EqualityExpression] = { - None: _handle_equality_general, - (_CONSTANT, _CONSTANT): _handle_equality_const, -} - - def _handle_inequality_const(visitor, node, arg1, arg2): # It is exceptionally likely that if we get here, one of the # arguments is an InvalidNumber @@ -490,12 +437,6 @@ def _handle_inequality_general(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[InequalityExpression] = { - None: _handle_inequality_general, - (_CONSTANT, _CONSTANT): _handle_inequality_const, -} - - def _handle_ranged_const(visitor, node, arg1, arg2, arg3): # It is exceptionally likely that if we get here, one of the # arguments is an InvalidNumber @@ -523,10 +464,62 @@ def _handle_ranged_general(visitor, node, arg1, arg2, arg3): return _GENERAL, ans -_exit_node_handlers[RangedExpression] = { - None: _handle_ranged_general, - (_CONSTANT, _CONSTANT, _CONSTANT): _handle_ranged_const, -} +def define_exit_node_handlers(_exit_node_handlers=None): + if _exit_node_handlers is None: + _exit_node_handlers = {} + _exit_node_handlers[NegationExpression] = { + None: _handle_negation_ANY, + (_CONSTANT,): _handle_negation_constant, + } + _exit_node_handlers[ProductExpression] = { + None: _handle_product_nonlinear, + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, + (_CONSTANT, _LINEAR): _handle_product_constant_ANY, + (_CONSTANT, _GENERAL): _handle_product_constant_ANY, + (_LINEAR, _CONSTANT): _handle_product_ANY_constant, + (_GENERAL, _CONSTANT): _handle_product_ANY_constant, + } + _exit_node_handlers[MonomialTermExpression] = _exit_node_handlers[ProductExpression] + _exit_node_handlers[DivisionExpression] = { + None: _handle_division_nonlinear, + (_CONSTANT, _CONSTANT): _handle_division_constant_constant, + (_LINEAR, _CONSTANT): _handle_division_ANY_constant, + (_GENERAL, _CONSTANT): _handle_division_ANY_constant, + } + _exit_node_handlers[PowExpression] = { + None: _handle_pow_nonlinear, + (_CONSTANT, _CONSTANT): _handle_pow_constant_constant, + (_LINEAR, _CONSTANT): _handle_pow_ANY_constant, + (_GENERAL, _CONSTANT): _handle_pow_ANY_constant, + } + _exit_node_handlers[UnaryFunctionExpression] = { + None: _handle_unary_nonlinear, + (_CONSTANT,): _handle_unary_constant, + } + _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] + _exit_node_handlers[Expression] = { + None: _handle_named_ANY, + (_CONSTANT,): _handle_named_constant, + } + _exit_node_handlers[Expr_ifExpression] = {None: _handle_expr_if_nonlinear} + for j in (_CONSTANT, _LINEAR, _GENERAL): + for k in (_CONSTANT, _LINEAR, _GENERAL): + _exit_node_handlers[Expr_ifExpression][ + _CONSTANT, j, k + ] = _handle_expr_if_const + _exit_node_handlers[EqualityExpression] = { + None: _handle_equality_general, + (_CONSTANT, _CONSTANT): _handle_equality_const, + } + _exit_node_handlers[InequalityExpression] = { + None: _handle_inequality_general, + (_CONSTANT, _CONSTANT): _handle_inequality_const, + } + _exit_node_handlers[RangedExpression] = { + None: _handle_ranged_general, + (_CONSTANT, _CONSTANT, _CONSTANT): _handle_ranged_const, + } + return _exit_node_handlers class LinearBeforeChildDispatcher(BeforeChildDispatcher): @@ -728,9 +721,8 @@ def _initialize_exit_node_dispatcher(exit_handlers): class LinearRepnVisitor(StreamBasedExpressionVisitor): Result = LinearRepn - exit_node_handlers = _exit_node_handlers exit_node_dispatcher = ExitNodeDispatcher( - _initialize_exit_node_dispatcher(_exit_node_handlers) + _initialize_exit_node_dispatcher(define_exit_node_handlers()) ) expand_nonlinear_products = False max_exponential_expansion = 1 diff --git a/pyomo/repn/quadratic.py b/pyomo/repn/quadratic.py index f6e0a43623d..a9c8b7bf2b5 100644 --- a/pyomo/repn/quadratic.py +++ b/pyomo/repn/quadratic.py @@ -157,17 +157,6 @@ def append(self, other): self.nonlinear += nl -_exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) - -# -# NEGATION -# -_exit_node_handlers[NegationExpression][(_QUADRATIC,)] = linear._handle_negation_ANY - - -# -# PRODUCT -# def _mul_linear_linear(varOrder, linear1, linear2): quadratic = {} for vid1, coef1 in linear1.items(): @@ -275,69 +264,73 @@ def _handle_product_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[ProductExpression].update( - { - None: _handle_product_nonlinear, - (_CONSTANT, _QUADRATIC): linear._handle_product_constant_ANY, - (_QUADRATIC, _CONSTANT): linear._handle_product_ANY_constant, - # Replace handler from the linear walker - (_LINEAR, _LINEAR): _handle_product_linear_linear, - } -) - -# -# DIVISION -# -_exit_node_handlers[DivisionExpression].update( - {(_QUADRATIC, _CONSTANT): linear._handle_division_ANY_constant} -) - - -# -# EXPONENTIATION -# -_exit_node_handlers[PowExpression].update( - {(_QUADRATIC, _CONSTANT): linear._handle_pow_ANY_constant} -) - -# -# ABS and UNARY handlers -# -# (no changes needed) - -# -# NAMED EXPRESSION handlers -# -# (no changes needed) - -# -# EXPR_IF handlers -# -# Note: it is easier to just recreate the entire data structure, rather -# than update it -_exit_node_handlers[Expr_ifExpression].update( - { - (_CONSTANT, i, _QUADRATIC): linear._handle_expr_if_const - for i in (_CONSTANT, _LINEAR, _QUADRATIC, _GENERAL) - } -) -_exit_node_handlers[Expr_ifExpression].update( - { - (_CONSTANT, _QUADRATIC, i): linear._handle_expr_if_const - for i in (_CONSTANT, _LINEAR, _GENERAL) - } -) - -# -# RELATIONAL handlers -# -# (no changes needed) +def define_exit_node_handlers(_exit_node_handlers=None): + if _exit_node_handlers is None: + _exit_node_handlers = {} + linear.define_exit_node_handlers(_exit_node_handlers) + # + # NEGATION + # + _exit_node_handlers[NegationExpression][(_QUADRATIC,)] = linear._handle_negation_ANY + # + # PRODUCT + # + _exit_node_handlers[ProductExpression].update( + { + None: _handle_product_nonlinear, + (_CONSTANT, _QUADRATIC): linear._handle_product_constant_ANY, + (_QUADRATIC, _CONSTANT): linear._handle_product_ANY_constant, + # Replace handler from the linear walker + (_LINEAR, _LINEAR): _handle_product_linear_linear, + } + ) + # + # DIVISION + # + _exit_node_handlers[DivisionExpression].update( + {(_QUADRATIC, _CONSTANT): linear._handle_division_ANY_constant} + ) + # + # EXPONENTIATION + # + _exit_node_handlers[PowExpression].update( + {(_QUADRATIC, _CONSTANT): linear._handle_pow_ANY_constant} + ) + # + # ABS and UNARY handlers + # + # (no changes needed) + # + # NAMED EXPRESSION handlers + # + # (no changes needed) + # + # EXPR_IF handlers + # + # Note: it is easier to just recreate the entire data structure, rather + # than update it + _exit_node_handlers[Expr_ifExpression].update( + { + (_CONSTANT, i, _QUADRATIC): linear._handle_expr_if_const + for i in (_CONSTANT, _LINEAR, _QUADRATIC, _GENERAL) + } + ) + _exit_node_handlers[Expr_ifExpression].update( + { + (_CONSTANT, _QUADRATIC, i): linear._handle_expr_if_const + for i in (_CONSTANT, _LINEAR, _GENERAL) + } + ) + # + # RELATIONAL handlers + # + # (no changes needed) + return _exit_node_handlers class QuadraticRepnVisitor(linear.LinearRepnVisitor): Result = QuadraticRepn - exit_node_handlers = _exit_node_handlers exit_node_dispatcher = linear.ExitNodeDispatcher( - linear._initialize_exit_node_dispatcher(_exit_node_handlers) + linear._initialize_exit_node_dispatcher(define_exit_node_handlers()) ) max_exponential_expansion = 2 diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index 8d902d0f99a..1c58821ba6c 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -378,18 +378,16 @@ class ExitNodeDispatcher(collections.defaultdict): `exitNode` callback This dispatcher implements a specialization of :py:`defaultdict` - that supports automatic type registration. Any missing types will - return the :py:meth:`register_dispatcher` method, which (when called - as a callback) will interrogate the type, identify the appropriate - callback, add the callback to the dict, and return the result of - calling the callback. As the callback is added to the dict, no type - will incur the overhead of `register_dispatcher` more than once. + that supports automatic type registration. As the identified + callback is added to the dict, no type will incur the overhead of + `register_dispatcher` more than once. Note that in this case, the client is expected to register all non-NPV expression types. The auto-registration is designed to only handle two cases: - Auto-detection of user-defined Named Expression types - Automatic mappimg of NPV expressions to their equivalent non-NPV handlers + - Automatic registration of derived expression types """ From b8ac19a6f19fc7cf27154072665c27ff3f568690 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:37:10 -0600 Subject: [PATCH 113/150] Defining define exit node handlers function --- pyomo/repn/parameterized_linear.py | 104 +++++++++++++++-------------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 45ff2943b11..2f83c450ea5 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -208,20 +208,14 @@ def _before_var(visitor, child): _before_child_dispatcher = ParameterizedLinearBeforeChildDispatcher() _exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) + # # NEGATION handlers # - def _handle_negation_pseudo_constant(visitor, node, arg): return (_FIXED, -1 * arg[1]) - -_exit_node_handlers[NegationExpression].update( - {(_FIXED,): _handle_negation_pseudo_constant} -) - - # # PRODUCT handlers # @@ -237,22 +231,6 @@ def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): return _FIXED, arg1[1] * arg2[1] -_exit_node_handlers[ProductExpression].update( - { - (_CONSTANT, _CONSTANT): _handle_product_constant_constant, - (_FIXED, _FIXED): _handle_product_pseudo_constant_constant, - (_FIXED, _CONSTANT): _handle_product_pseudo_constant_constant, - (_CONSTANT, _FIXED): _handle_product_pseudo_constant_constant, - (_FIXED, _LINEAR): linear._handle_product_constant_ANY, - (_LINEAR, _FIXED): linear._handle_product_ANY_constant, - (_FIXED, _GENERAL): linear._handle_product_constant_ANY, - (_GENERAL, _FIXED): linear._handle_product_ANY_constant, - } -) -_exit_node_handlers[MonomialTermExpression].update( - _exit_node_handlers[ProductExpression] -) - # # DIVISION handlers # @@ -267,15 +245,6 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): return arg1 -_exit_node_handlers[DivisionExpression].update( - { - (_FIXED, _FIXED): _handle_division_pseudo_constant_constant, - (_FIXED, _CONSTANT): _handle_division_pseudo_constant_constant, - (_CONSTANT, _FIXED): _handle_division_pseudo_constant_constant, - (_LINEAR, _FIXED): _handle_division_ANY_pseudo_constant, - (_GENERAL, _FIXED): _handle_division_ANY_pseudo_constant, - } -) # # EXPONENTIATION handlers @@ -294,17 +263,6 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[PowExpression].update( - { - (_FIXED, _FIXED): _handle_pow_pseudo_constant_constant, - (_FIXED, _CONSTANT): _handle_pow_pseudo_constant_constant, - (_CONSTANT, _FIXED): _handle_pow_pseudo_constant_constant, - (_LINEAR, _FIXED): _handle_pow_nonlinear, - (_FIXED, _LINEAR): _handle_pow_nonlinear, - (_GENERAL, _FIXED): _handle_pow_nonlinear, - (_FIXED, _GENERAL): _handle_pow_nonlinear, - } -) # # ABS and UNARY handlers @@ -316,17 +274,65 @@ def _handle_unary_pseudo_constant(visitor, node, arg): return _FIXED, node.create_node_with_local_data((to_expression(visitor, arg),)) -_exit_node_handlers[UnaryFunctionExpression].update( - {(_FIXED,): _handle_unary_pseudo_constant} -) -_exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] +def define_exit_node_handlers(_exit_node_handlers=None): + if _exit_node_handlers is None: + _exit_node_handlers = {} + + _exit_node_handlers[NegationExpression].update( + {(_FIXED,): _handle_negation_pseudo_constant} + ) + + _exit_node_handlers[ProductExpression].update( + { + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, + (_FIXED, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_product_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _LINEAR): linear._handle_product_constant_ANY, + (_LINEAR, _FIXED): linear._handle_product_ANY_constant, + (_FIXED, _GENERAL): linear._handle_product_constant_ANY, + (_GENERAL, _FIXED): linear._handle_product_ANY_constant, + } + ) + + _exit_node_handlers[MonomialTermExpression].update( + _exit_node_handlers[ProductExpression] + ) + + _exit_node_handlers[DivisionExpression].update( + { + (_FIXED, _FIXED): _handle_division_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_division_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_division_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_division_ANY_pseudo_constant, + (_GENERAL, _FIXED): _handle_division_ANY_pseudo_constant, + } + ) + + _exit_node_handlers[PowExpression].update( + { + (_FIXED, _FIXED): _handle_pow_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_pow_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_pow_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_pow_nonlinear, + (_FIXED, _LINEAR): _handle_pow_nonlinear, + (_GENERAL, _FIXED): _handle_pow_nonlinear, + (_FIXED, _GENERAL): _handle_pow_nonlinear, + } + ) + _exit_node_handlers[UnaryFunctionExpression].update( + {(_FIXED,): _handle_unary_pseudo_constant} + ) + _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] + + return _exit_node_handlers class ParameterizedLinearRepnVisitor(LinearRepnVisitor): Result = ParameterizedLinearRepn - exit_node_handlers = _exit_node_handlers + exit_node_handlers = define_exit_node_handlers(_exit_node_handlers) exit_node_dispatcher = ExitNodeDispatcher( - _initialize_exit_node_dispatcher(_exit_node_handlers) + _initialize_exit_node_dispatcher(exit_node_handlers) ) def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): From 0f7390da8319c2e81b0d8c7e556567529032d32f Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:46:45 -0600 Subject: [PATCH 114/150] Calling linear define exit node handler --- pyomo/repn/parameterized_linear.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 2f83c450ea5..47f9445bd7f 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -206,8 +206,6 @@ def _before_var(visitor, child): _before_child_dispatcher = ParameterizedLinearBeforeChildDispatcher() -_exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) - # # NEGATION handlers @@ -274,15 +272,16 @@ def _handle_unary_pseudo_constant(visitor, node, arg): return _FIXED, node.create_node_with_local_data((to_expression(visitor, arg),)) -def define_exit_node_handlers(_exit_node_handlers=None): - if _exit_node_handlers is None: - _exit_node_handlers = {} +def define_exit_node_handlers(exit_node_handlers=None): + if exit_node_handlers is None: + exit_node_handlers = {} + linear.define_exit_node_handlers(exit_node_handlers) - _exit_node_handlers[NegationExpression].update( + exit_node_handlers[NegationExpression].update( {(_FIXED,): _handle_negation_pseudo_constant} ) - _exit_node_handlers[ProductExpression].update( + exit_node_handlers[ProductExpression].update( { (_CONSTANT, _CONSTANT): _handle_product_constant_constant, (_FIXED, _FIXED): _handle_product_pseudo_constant_constant, @@ -295,11 +294,11 @@ def define_exit_node_handlers(_exit_node_handlers=None): } ) - _exit_node_handlers[MonomialTermExpression].update( - _exit_node_handlers[ProductExpression] + exit_node_handlers[MonomialTermExpression].update( + exit_node_handlers[ProductExpression] ) - _exit_node_handlers[DivisionExpression].update( + exit_node_handlers[DivisionExpression].update( { (_FIXED, _FIXED): _handle_division_pseudo_constant_constant, (_FIXED, _CONSTANT): _handle_division_pseudo_constant_constant, @@ -309,7 +308,7 @@ def define_exit_node_handlers(_exit_node_handlers=None): } ) - _exit_node_handlers[PowExpression].update( + exit_node_handlers[PowExpression].update( { (_FIXED, _FIXED): _handle_pow_pseudo_constant_constant, (_FIXED, _CONSTANT): _handle_pow_pseudo_constant_constant, @@ -320,19 +319,18 @@ def define_exit_node_handlers(_exit_node_handlers=None): (_FIXED, _GENERAL): _handle_pow_nonlinear, } ) - _exit_node_handlers[UnaryFunctionExpression].update( + exit_node_handlers[UnaryFunctionExpression].update( {(_FIXED,): _handle_unary_pseudo_constant} ) - _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] + exit_node_handlers[AbsExpression] = exit_node_handlers[UnaryFunctionExpression] - return _exit_node_handlers + return exit_node_handlers class ParameterizedLinearRepnVisitor(LinearRepnVisitor): Result = ParameterizedLinearRepn - exit_node_handlers = define_exit_node_handlers(_exit_node_handlers) exit_node_dispatcher = ExitNodeDispatcher( - _initialize_exit_node_dispatcher(exit_node_handlers) + _initialize_exit_node_dispatcher(define_exit_node_handlers()) ) def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): From 3235f928cd73724815236b28bec93ba0b2cde25b Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:47:26 -0600 Subject: [PATCH 115/150] Black: breaking even on whitespace, for once --- pyomo/repn/parameterized_linear.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 47f9445bd7f..d1295b73e14 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -211,9 +211,11 @@ def _before_var(visitor, child): # NEGATION handlers # + def _handle_negation_pseudo_constant(visitor, node, arg): return (_FIXED, -1 * arg[1]) + # # PRODUCT handlers # @@ -243,7 +245,6 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): return arg1 - # # EXPONENTIATION handlers # @@ -261,7 +262,6 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans - # # ABS and UNARY handlers # @@ -280,7 +280,7 @@ def define_exit_node_handlers(exit_node_handlers=None): exit_node_handlers[NegationExpression].update( {(_FIXED,): _handle_negation_pseudo_constant} ) - + exit_node_handlers[ProductExpression].update( { (_CONSTANT, _CONSTANT): _handle_product_constant_constant, From 205eb3d7caa2507875df784e0d51d970efa3bdf7 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 8 Jul 2024 15:30:31 -0600 Subject: [PATCH 116/150] Set default git SHA for codecov when building main branch --- .jenkins.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.jenkins.sh b/.jenkins.sh index 51dd92ba123..f64c413690c 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -208,7 +208,10 @@ if test -z "$MODE" -o "$MODE" == test; then coverage report -i || exit 1 coverage xml -i || exit 1 export OS=`uname` - if test -n "$CODECOV_TOKEN"; then + if test -z "$PYOMO_SOURCE_SHA"; then + PYOMO_SOURCE_SHA=$GIT_COMMIT + fi + if test -n "$CODECOV_TOKEN" -a -n "$PYOMO_SOURCE_SHA"; then CODECOV_JOB_NAME=`echo ${JOB_NAME} | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/'`.$BUILD_NUMBER.$python if test -z "$CODECOV_REPO_OWNER"; then CODECOV_REPO_OWNER="pyomo" From 7197f966f4060a047094f5a56c96f46df22b76ef Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 8 Jul 2024 16:57:11 -0600 Subject: [PATCH 117/150] Jenkins: include calculation of CODECOV_REPO_OWNER and CODECOV_SOURCE_BRANCH --- .jenkins.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index f64c413690c..defb9879b90 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -212,12 +212,24 @@ if test -z "$MODE" -o "$MODE" == test; then PYOMO_SOURCE_SHA=$GIT_COMMIT fi if test -n "$CODECOV_TOKEN" -a -n "$PYOMO_SOURCE_SHA"; then - CODECOV_JOB_NAME=`echo ${JOB_NAME} | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/'`.$BUILD_NUMBER.$python + CODECOV_JOB_NAME=$(echo ${JOB_NAME} \ + | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/').$BUILD_NUMBER.$python if test -z "$CODECOV_REPO_OWNER"; then - CODECOV_REPO_OWNER="pyomo" + if test -n "$PYOMO_SOURCE_REPO" + CODECOV_REPO_OWNER=$(echo $PYOMO_SOURCE_REPO | cut -d '/' -f 4) + elif test -n "$GIT_URL"; then + CODECOV_REPO_OWNER=$(echo $GIT_URL | cut -d '/' -f 4) + else + CODECOV_REPO_OWNER="" + fi fi - if test -z "CODECOV_SOURCE_BRANCH"; then - CODECOV_SOURCE_BRANCH="main" + if test -z "$CODECOV_SOURCE_BRANCH"; then + CODECOV_SOURCE_BRANCH=$(git branch -av --contains "$PYOMO_SOURCE_SHA" \ + | grep "${PYOMO_SOURCE_SHA:0:7}" | grep "/origin/" \ + | cut -d '/' -f 3 | cut -d' ' -f 1) + if test -z "$CODECOV_SOURCE_BRANCH"; then + CODECOV_SOURCE_BRANCH=main + fi fi i=0 while /bin/true; do From 6edb167805eefc16e7ea45b474483fb416e56bd1 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 9 Jul 2024 09:57:21 -0600 Subject: [PATCH 118/150] Fix typo in Jenkins driver --- .jenkins.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index defb9879b90..8771427805d 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -215,10 +215,10 @@ if test -z "$MODE" -o "$MODE" == test; then CODECOV_JOB_NAME=$(echo ${JOB_NAME} \ | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/').$BUILD_NUMBER.$python if test -z "$CODECOV_REPO_OWNER"; then - if test -n "$PYOMO_SOURCE_REPO" - CODECOV_REPO_OWNER=$(echo $PYOMO_SOURCE_REPO | cut -d '/' -f 4) + if test -n "$PYOMO_SOURCE_REPO"; then + CODECOV_REPO_OWNER=$(echo "$PYOMO_SOURCE_REPO" | cut -d '/' -f 4) elif test -n "$GIT_URL"; then - CODECOV_REPO_OWNER=$(echo $GIT_URL | cut -d '/' -f 4) + CODECOV_REPO_OWNER=$(echo "$GIT_URL" | cut -d '/' -f 4) else CODECOV_REPO_OWNER="" fi From c8bf1312c4c08fb7a35a484fda0d9e651c0822c6 Mon Sep 17 00:00:00 2001 From: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> Date: Tue, 9 Jul 2024 10:08:26 -0600 Subject: [PATCH 119/150] Change universal_newlines -> text --- pyomo/solvers/plugins/solvers/ASL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/ASL.py b/pyomo/solvers/plugins/solvers/ASL.py index a29c64e017f..7acd59936b1 100644 --- a/pyomo/solvers/plugins/solvers/ASL.py +++ b/pyomo/solvers/plugins/solvers/ASL.py @@ -101,7 +101,7 @@ def _get_version(self): timeout=5, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True, + text=True, errors='ignore', ) ver = _extract_version(results.stdout) From f658981cee62da884f60347ab1341d709d387d57 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 9 Jul 2024 12:54:52 -0600 Subject: [PATCH 120/150] NFC: update spaces in message --- pyomo/core/base/constraint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/core/base/constraint.py b/pyomo/core/base/constraint.py index 3ee5a82ef58..defaea99dff 100644 --- a/pyomo/core/base/constraint.py +++ b/pyomo/core/base/constraint.py @@ -347,8 +347,8 @@ def set_value(self, expr): if getattr(expr, 'strict', False) in _strict_relational_exprs: raise ValueError( "Constraint '%s' encountered a strict " - "inequality expression ('>' or '< '). All" - " constraints must be formulated using " + "inequality expression ('>' or '<'). All " + "constraints must be formulated using " "using '<=', '>=', or '=='." % (self.name,) ) self._expr = expr From 3bb2100b5bc8eb1972f1e62a78c4314ae494fd4d Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 9 Jul 2024 15:37:07 -0600 Subject: [PATCH 121/150] Update test_set.py exception reference strings --- pyomo/core/tests/unit/test_set.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 012669a3484..4a49f6a2fae 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6453,15 +6453,14 @@ def test_issue_3284(self): # set subset_A problem.subset_A.add(1) error_message = ( - "Cannot add value %s to Set %s.\n" - "\tThe value is not in the domain %s" - % (4, 'subset_A', 'A') + "Cannot add value 4 to Set subset_A.\n" + "\tThe value is not in the domain A" ) with self.assertRaisesRegex(ValueError, error_message): problem.subset_A.add(4) # set subset_B problem.subset_B.add((3, 4)) - with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + with self.assertRaisesRegex(ValueError, ".*Cannot add value \(7, 8\)"): problem.subset_B.add((7, 8)) # set subset_C problem.subset_C[2].add(7) @@ -6477,7 +6476,7 @@ def test_issue_3284(self): problem.E[1].add(4) # set F problem.F[(1, 2, 3)].add((3, 4)) - with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + with self.assertRaisesRegex(ValueError, ".*Cannot add value \(4, 3\)"): problem.F[(4, 5, 6)].add((4, 3)) # check them self.assertEqual(list(problem.A), [1, 2, 3]) From ef55217c0229fe67fcf6e07951a9152009e8119e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 9 Jul 2024 15:38:59 -0600 Subject: [PATCH 122/150] NFC: apply black --- pyomo/core/tests/unit/test_set.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 4a49f6a2fae..1ef1b5d6867 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6453,9 +6453,8 @@ def test_issue_3284(self): # set subset_A problem.subset_A.add(1) error_message = ( - "Cannot add value 4 to Set subset_A.\n" - "\tThe value is not in the domain A" - ) + "Cannot add value 4 to Set subset_A.\n\tThe value is not in the domain A" + ) with self.assertRaisesRegex(ValueError, error_message): problem.subset_A.add(4) # set subset_B From 643c0e55deb54b823ecb0d47da2c0d95501c1bbf Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 9 Jul 2024 16:13:17 -0600 Subject: [PATCH 123/150] Bugfix: regex should be a raw string --- pyomo/core/tests/unit/test_set.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 1ef1b5d6867..2b0da8b861d 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6459,7 +6459,7 @@ def test_issue_3284(self): problem.subset_A.add(4) # set subset_B problem.subset_B.add((3, 4)) - with self.assertRaisesRegex(ValueError, ".*Cannot add value \(7, 8\)"): + with self.assertRaisesRegex(ValueError, r".*Cannot add value \(7, 8\)"): problem.subset_B.add((7, 8)) # set subset_C problem.subset_C[2].add(7) @@ -6475,7 +6475,7 @@ def test_issue_3284(self): problem.E[1].add(4) # set F problem.F[(1, 2, 3)].add((3, 4)) - with self.assertRaisesRegex(ValueError, ".*Cannot add value \(4, 3\)"): + with self.assertRaisesRegex(ValueError, r".*Cannot add value \(4, 3\)"): problem.F[(4, 5, 6)].add((4, 3)) # check them self.assertEqual(list(problem.A), [1, 2, 3]) From 533f6165bd7b9245ae3f69e1368dd6e54e101d1a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 10 Jul 2024 10:59:00 -0600 Subject: [PATCH 124/150] Fix (and test) FBBT error with RangedExpressions --- pyomo/contrib/fbbt/fbbt.py | 2 +- pyomo/contrib/fbbt/tests/test_fbbt.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/fbbt/fbbt.py b/pyomo/contrib/fbbt/fbbt.py index 60ac0603388..1a0b2992b07 100644 --- a/pyomo/contrib/fbbt/fbbt.py +++ b/pyomo/contrib/fbbt/fbbt.py @@ -1389,7 +1389,7 @@ def _fbbt_block(m, config): for c in m.component_data_objects( ctype=Constraint, active=True, descend_into=config.descend_into, sort=True ): - for v in identify_variables(c.body): + for v in identify_variables(c.expr): if v not in var_to_con_map: var_to_con_map[v] = list() if v.lb is None: diff --git a/pyomo/contrib/fbbt/tests/test_fbbt.py b/pyomo/contrib/fbbt/tests/test_fbbt.py index f7d08d11215..033d3ac7a78 100644 --- a/pyomo/contrib/fbbt/tests/test_fbbt.py +++ b/pyomo/contrib/fbbt/tests/test_fbbt.py @@ -1331,6 +1331,29 @@ def test_named_expr(self): self.assertAlmostEqual(m.x.lb, 2) self.assertAlmostEqual(m.x.ub, 3) + def test_ranged_expression(self): + m = pyo.ConcreteModel() + m.l = pyo.Var(bounds=(2, None)) + m.x = pyo.Var() + m.u = pyo.Var(bounds=(None, 8)) + m.c = pyo.Constraint(expr=pyo.inequality(m.l, m.x, m.u)) + self.tightener(m) + self.tightener(m) + self.assertEqual(m.l.bounds, (2, 8)) + self.assertEqual(m.x.bounds, (2, 8)) + self.assertEqual(m.u.bounds, (2, 8)) + + m = pyo.ConcreteModel() + m.l = pyo.Var(bounds=(2, None)) + m.x = pyo.Var(bounds=(3, 7)) + m.u = pyo.Var(bounds=(None, 8)) + m.c = pyo.Constraint(expr=pyo.inequality(m.l, m.x, m.u)) + self.tightener(m) + self.tightener(m) + self.assertEqual(m.l.bounds, (2, 7)) + self.assertEqual(m.x.bounds, (3, 7)) + self.assertEqual(m.u.bounds, (3, 8)) + class TestFBBT(FbbtTestBase, unittest.TestCase): def setUp(self) -> None: From 399c7311afaebcfbf1fb8753a477ee6f82717f9e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 10 Jul 2024 12:11:33 -0600 Subject: [PATCH 125/150] Only test variable RangedExpressions in Python FBBT --- pyomo/contrib/fbbt/tests/test_fbbt.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/fbbt/tests/test_fbbt.py b/pyomo/contrib/fbbt/tests/test_fbbt.py index 033d3ac7a78..ff1cc8a5cfb 100644 --- a/pyomo/contrib/fbbt/tests/test_fbbt.py +++ b/pyomo/contrib/fbbt/tests/test_fbbt.py @@ -1331,7 +1331,17 @@ def test_named_expr(self): self.assertAlmostEqual(m.x.lb, 2) self.assertAlmostEqual(m.x.ub, 3) + +class TestFBBT(FbbtTestBase, unittest.TestCase): + def setUp(self) -> None: + self.tightener = fbbt + def test_ranged_expression(self): + # The python version of FBBT is slightly more flexible than + # APPSI's cmodel (it allows - and correctly handles - + # RangedExpressions with variable lower / upper bounds. If we + # ever port that functionality into APPSI, then this test can be + # moved into the base class. m = pyo.ConcreteModel() m.l = pyo.Var(bounds=(2, None)) m.x = pyo.Var() @@ -1353,8 +1363,3 @@ def test_ranged_expression(self): self.assertEqual(m.l.bounds, (2, 7)) self.assertEqual(m.x.bounds, (3, 7)) self.assertEqual(m.u.bounds, (3, 8)) - - -class TestFBBT(FbbtTestBase, unittest.TestCase): - def setUp(self) -> None: - self.tightener = fbbt From 7f84d3e9b5e23a0273dc8c55527c74c538a9f436 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 10 Jul 2024 12:12:28 -0600 Subject: [PATCH 126/150] Move to using expr (not body) when gathering variabels from constraints --- pyomo/contrib/community_detection/community_graph.py | 2 +- pyomo/contrib/fbbt/fbbt.py | 6 +++--- pyomo/util/infeasible.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/community_detection/community_graph.py b/pyomo/contrib/community_detection/community_graph.py index 889940b5996..c67a8cd6690 100644 --- a/pyomo/contrib/community_detection/community_graph.py +++ b/pyomo/contrib/community_detection/community_graph.py @@ -123,7 +123,7 @@ def generate_model_graph( # Create a list of the variable numbers that occur in the given constraint equation numbered_variables_in_constraint_equation = [ component_number_map[constraint_variable] - for constraint_variable in identify_variables(model_constraint.body) + for constraint_variable in identify_variables(model_constraint.expr) ] # Update constraint_variable_map diff --git a/pyomo/contrib/fbbt/fbbt.py b/pyomo/contrib/fbbt/fbbt.py index 1a0b2992b07..4bd0e4552a1 100644 --- a/pyomo/contrib/fbbt/fbbt.py +++ b/pyomo/contrib/fbbt/fbbt.py @@ -1576,14 +1576,14 @@ def __init__(self, comp): if comp.ctype == Constraint: if comp.is_indexed(): for c in comp.values(): - self._vars.update(identify_variables(c.body)) + self._vars.update(identify_variables(c.expr)) else: - self._vars.update(identify_variables(comp.body)) + self._vars.update(identify_variables(comp.expr)) else: for c in comp.component_data_objects( Constraint, descend_into=True, active=True, sort=True ): - self._vars.update(identify_variables(c.body)) + self._vars.update(identify_variables(c.expr)) def save_bounds(self): bnds = ComponentMap() diff --git a/pyomo/util/infeasible.py b/pyomo/util/infeasible.py index 961d5b35036..6a90a4c3773 100644 --- a/pyomo/util/infeasible.py +++ b/pyomo/util/infeasible.py @@ -159,7 +159,7 @@ def log_infeasible_constraints( if log_variables: line += ''.join( f"\n - VAR {v.name}: {v.value}" - for v in identify_variables(constr.body, include_fixed=True) + for v in identify_variables(constr.expr, include_fixed=True) ) logger.info(line) From 663b64982bc9012ec11da627e4aad1b58aca3540 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 10 Jul 2024 13:00:13 -0600 Subject: [PATCH 127/150] Fixing a bug in multiple bigm where we were not updating the list of active Disjuncts after we encountered an infeasible one and deactivated it. Nothing was wrong here, but we were doing extra work (and hitting a bug in the baron writer) --- pyomo/gdp/plugins/multiple_bigm.py | 25 +++++++++++++++++++------ pyomo/gdp/tests/test_mbigm.py | 14 ++++++-------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/pyomo/gdp/plugins/multiple_bigm.py b/pyomo/gdp/plugins/multiple_bigm.py index 4dffd4e9f9a..6914d6937ef 100644 --- a/pyomo/gdp/plugins/multiple_bigm.py +++ b/pyomo/gdp/plugins/multiple_bigm.py @@ -12,7 +12,7 @@ import itertools import logging -from pyomo.common.collections import ComponentMap +from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.config import ConfigDict, ConfigValue from pyomo.common.gc_manager import PauseGC from pyomo.common.modeling import unique_component_name @@ -310,9 +310,12 @@ def _transform_disjunctionData(self, obj, index, parent_disjunct, root_disjunct) arg_Ms = self._config.bigM if self._config.bigM is not None else {} + # ESJ: I am relying on the fact that the ComponentSet is going to be + # ordered here, but using a set because I will remove infeasible + # Disjuncts from it if I encounter them calculating M's. + active_disjuncts = ComponentSet(disj for disj in obj.disjuncts if disj.active) # First handle the bound constraints if we are dealing with them # separately - active_disjuncts = [disj for disj in obj.disjuncts if disj.active] transformed_constraints = set() if self._config.reduce_bound_constraints: transformed_constraints = self._transform_bound_constraints( @@ -585,7 +588,7 @@ def _calculate_missing_M_values( ): if disjunct is other_disjunct: continue - if id(other_disjunct) in scratch_blocks: + elif id(other_disjunct) in scratch_blocks: scratch = scratch_blocks[id(other_disjunct)] else: scratch = scratch_blocks[id(other_disjunct)] = Block() @@ -631,7 +634,7 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.lower scratch.obj.sense = minimize lower_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg + other_disjunct, scratch, unsuccessful_solve_msg, active_disjuncts ) if constraint.upper is not None and upper_M is None: # last resort: calculate @@ -639,7 +642,7 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.upper scratch.obj.sense = maximize upper_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg + other_disjunct, scratch, unsuccessful_solve_msg, active_disjuncts ) arg_Ms[constraint, other_disjunct] = (lower_M, upper_M) transBlock._mbm_values[constraint, other_disjunct] = (lower_M, upper_M) @@ -651,9 +654,18 @@ def _calculate_missing_M_values( return arg_Ms def _solve_disjunct_for_M( - self, other_disjunct, scratch_block, unsuccessful_solve_msg + self, other_disjunct, scratch_block, unsuccessful_solve_msg, active_disjuncts ): + if not other_disjunct.active: + # If a Disjunct is infeasible, we will discover that and deactivate + # it when we are calculating the M values. We remove that disjunct + # from active_disjuncts inside of the loop in + # _calculate_missing_M_values. So that means that we might have + # deactivated Disjuncts here that we should skip over. + return 0 + solver = self._config.solver + results = solver.solve(other_disjunct, load_solutions=False) if results.solver.termination_condition is TerminationCondition.infeasible: # [2/18/24]: TODO: After the solver rewrite is complete, we will not @@ -669,6 +681,7 @@ def _solve_disjunct_for_M( "Disjunct '%s' is infeasible, deactivating." % other_disjunct.name ) other_disjunct.deactivate() + active_disjuncts.remove(other_disjunct) M = 0 else: # This is a solver that might report diff --git a/pyomo/gdp/tests/test_mbigm.py b/pyomo/gdp/tests/test_mbigm.py index 9e82b1010f9..1516681f5a4 100644 --- a/pyomo/gdp/tests/test_mbigm.py +++ b/pyomo/gdp/tests/test_mbigm.py @@ -1019,11 +1019,14 @@ def test_calculate_Ms_infeasible_Disjunct(self): out.getvalue().strip(), ) - # We just fixed the infeasible by to False + # We just fixed the infeasible disjunct to False self.assertFalse(m.disjunction.disjuncts[0].active) self.assertTrue(m.disjunction.disjuncts[0].indicator_var.fixed) self.assertFalse(value(m.disjunction.disjuncts[0].indicator_var)) + # We didn't actually transform the infeasible disjunct + self.assertIsNone(m.disjunction.disjuncts[0].transformation_block) + # the remaining constraints are transformed correctly. cons = mbm.get_transformed_constraints(m.disjunction.disjuncts[1].constraint[1]) self.assertEqual(len(cons), 1) @@ -1031,18 +1034,14 @@ def test_calculate_Ms_infeasible_Disjunct(self): self, cons[0].expr, 21 + m.x - m.y - <= 0 * m.disjunction.disjuncts[0].binary_indicator_var - + 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, + <= 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, ) cons = mbm.get_transformed_constraints(m.disjunction.disjuncts[2].constraint[1]) self.assertEqual(len(cons), 2) - print(cons[0].expr) - print(cons[1].expr) assertExpressionsEqual( self, cons[0].expr, - 0.0 * m.disjunction_disjuncts[0].binary_indicator_var - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var <= m.x - (m.y - 9), ) @@ -1050,8 +1049,7 @@ def test_calculate_Ms_infeasible_Disjunct(self): self, cons[1].expr, m.x - (m.y - 9) - <= 0.0 * m.disjunction_disjuncts[0].binary_indicator_var - - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var, + <= - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var, ) @unittest.skipUnless( From 6f0d95461f2805f8cba0ff1f38dbc3190eb4ac5b Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 10 Jul 2024 13:00:53 -0600 Subject: [PATCH 128/150] Blackify --- pyomo/gdp/plugins/multiple_bigm.py | 10 ++++++++-- pyomo/gdp/tests/test_mbigm.py | 9 +++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pyomo/gdp/plugins/multiple_bigm.py b/pyomo/gdp/plugins/multiple_bigm.py index 6914d6937ef..3362276246b 100644 --- a/pyomo/gdp/plugins/multiple_bigm.py +++ b/pyomo/gdp/plugins/multiple_bigm.py @@ -634,7 +634,10 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.lower scratch.obj.sense = minimize lower_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg, active_disjuncts + other_disjunct, + scratch, + unsuccessful_solve_msg, + active_disjuncts, ) if constraint.upper is not None and upper_M is None: # last resort: calculate @@ -642,7 +645,10 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.upper scratch.obj.sense = maximize upper_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg, active_disjuncts + other_disjunct, + scratch, + unsuccessful_solve_msg, + active_disjuncts, ) arg_Ms[constraint, other_disjunct] = (lower_M, upper_M) transBlock._mbm_values[constraint, other_disjunct] = (lower_M, upper_M) diff --git a/pyomo/gdp/tests/test_mbigm.py b/pyomo/gdp/tests/test_mbigm.py index 1516681f5a4..14a23160574 100644 --- a/pyomo/gdp/tests/test_mbigm.py +++ b/pyomo/gdp/tests/test_mbigm.py @@ -1033,8 +1033,7 @@ def test_calculate_Ms_infeasible_Disjunct(self): assertExpressionsEqual( self, cons[0].expr, - 21 + m.x - m.y - <= 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, + 21 + m.x - m.y <= 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, ) cons = mbm.get_transformed_constraints(m.disjunction.disjuncts[2].constraint[1]) @@ -1042,14 +1041,12 @@ def test_calculate_Ms_infeasible_Disjunct(self): assertExpressionsEqual( self, cons[0].expr, - - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var - <= m.x - (m.y - 9), + -12.0 * m.disjunction_disjuncts[1].binary_indicator_var <= m.x - (m.y - 9), ) assertExpressionsEqual( self, cons[1].expr, - m.x - (m.y - 9) - <= - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var, + m.x - (m.y - 9) <= -12.0 * m.disjunction_disjuncts[1].binary_indicator_var, ) @unittest.skipUnless( From fe7941414d74b3991284355cc97fb48fc86179e2 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 10 Jul 2024 15:31:30 -0600 Subject: [PATCH 129/150] fix typo --- pyomo/contrib/incidence_analysis/scc_solver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/scc_solver.py b/pyomo/contrib/incidence_analysis/scc_solver.py index 508758cce1e..db201dccb0a 100644 --- a/pyomo/contrib/incidence_analysis/scc_solver.py +++ b/pyomo/contrib/incidence_analysis/scc_solver.py @@ -70,9 +70,9 @@ def generate_strongly_connected_components( nvar = len(variables) ncon = len(constraints) raise RuntimeError( - f"generate_strongly_connected_components only supports variables" - f" systems with the same numbers of variables and equality constraints." - f" Got {nvar} variables and {ncon} constraints." + "generate_strongly_connected_components only supports systems with the" + f" same numbers of variables and equality constraints. Got {nvar}" + f" variables and {ncon} constraints." ) if igraph is None: igraph = IncidenceGraphInterface() From a41efb574b9f50ffc3498f15d18d45ddbe0e1d94 Mon Sep 17 00:00:00 2001 From: Soren Davis Date: Fri, 12 Jul 2024 11:46:27 -0600 Subject: [PATCH 130/150] test for proper structure of changed as_domain() methods --- pyomo/core/tests/unit/kernel/test_conic.py | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/pyomo/core/tests/unit/kernel/test_conic.py b/pyomo/core/tests/unit/kernel/test_conic.py index ccfbcca7e1f..fc70b421060 100644 --- a/pyomo/core/tests/unit/kernel/test_conic.py +++ b/pyomo/core/tests/unit/kernel/test_conic.py @@ -35,6 +35,8 @@ primal_power, dual_exponential, dual_power, + primal_geomean, + dual_geomean, ) @@ -784,6 +786,40 @@ def test_as_domain(self): x[1].value = None +# these mosek 10 constraints are really anemic and can't be evaluated, pprinted, +# checked for convexity, pickled, etc. +class Test_primal_geomean(unittest.TestCase): + def test_as_domain(self): + b = primal_geomean.as_domain(r=[2, 3], x=6) + self.assertIs(type(b), block) + self.assertIs(type(b.q), primal_geomean) + self.assertIs(type(b.r), variable_tuple) + self.assertIs(type(b.x), variable) + self.assertIs(type(b.c), constraint_tuple) + self.assertExpressionsEqual(b.c[0].body, b.r[0]) + self.assertExpressionsEqual(b.c[0].rhs, 2) + self.assertExpressionsEqual(b.c[1].body, b.r[1]) + self.assertExpressionsEqual(b.c[1].rhs, 3) + self.assertExpressionsEqual(b.c[2].body, b.x) + self.assertExpressionsEqual(b.c[2].rhs, 6) + + +class Test_dual_geomean(unittest.TestCase): + def test_as_domain(self): + b = dual_geomean.as_domain(r=[2, 3], x=6) + self.assertIs(type(b), block) + self.assertIs(type(b.q), dual_geomean) + self.assertIs(type(b.r), variable_tuple) + self.assertIs(type(b.x), variable) + self.assertIs(type(b.c), constraint_tuple) + self.assertExpressionsEqual(b.c[0].body, b.r[0]) + self.assertExpressionsEqual(b.c[0].rhs, 2) + self.assertExpressionsEqual(b.c[1].body, b.r[1]) + self.assertExpressionsEqual(b.c[1].rhs, 3) + self.assertExpressionsEqual(b.c[2].body, b.x) + self.assertExpressionsEqual(b.c[2].rhs, 6) + + class TestMisc(unittest.TestCase): def test_build_linking_constraints(self): c = _build_linking_constraints([], []) From 402784bb186ddcb41875ae875bc735d49ccf6159 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Fri, 12 Jul 2024 14:30:40 -0600 Subject: [PATCH 131/150] fixing bug in _load_slacks --- pyomo/solvers/plugins/solvers/xpress_direct.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/xpress_direct.py b/pyomo/solvers/plugins/solvers/xpress_direct.py index c62f76d85ce..33a3c8d0282 100644 --- a/pyomo/solvers/plugins/solvers/xpress_direct.py +++ b/pyomo/solvers/plugins/solvers/xpress_direct.py @@ -1036,10 +1036,8 @@ def _load_slacks(self, cons_to_load=None): if xpress_con in self._range_constraints: ## for xpress, the slack on a range constraint ## is based on the upper bound - ## FIXME: This looks like a bug - there is no variable named - ## `con` - there is, however, `xpress_con` and `pyomo_con` - lb = con.lb - ub = con.ub + lb = xpress_con.lb + ub = xpress_con.ub ub_s = val expr_val = ub - ub_s lb_s = lb - expr_val From e720fd837f4cae5aa1c17587da9503b3cf17890e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 14 Jul 2024 15:50:51 -0600 Subject: [PATCH 132/150] NLv2: support models with expressions with nested external functions --- pyomo/repn/plugins/nl_writer.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pyomo/repn/plugins/nl_writer.py b/pyomo/repn/plugins/nl_writer.py index a8966e44f71..8fc82d21d30 100644 --- a/pyomo/repn/plugins/nl_writer.py +++ b/pyomo/repn/plugins/nl_writer.py @@ -1988,6 +1988,18 @@ def _record_named_expression_usage(self, named_exprs, src, comp_type): elif info[comp_type] != src: info[comp_type] = 0 + def _resolve_subexpression_args(self, nl, args): + final_args = [] + for arg in args: + if arg in self.var_id_to_nl_map: + final_args.append(self.var_id_to_nl_map[arg]) + else: + _nl, _ids, _ = self.subexpression_cache[arg][1].compile_repn( + self.visitor + ) + final_args.append(self._resolve_subexpression_args(_nl, _ids)) + return nl % tuple(final_args) + def _write_nl_expression(self, repn, include_const): # Note that repn.mult should always be 1 (the AMPLRepn was # compiled before this point). Omitting the assertion for @@ -2007,18 +2019,7 @@ def _write_nl_expression(self, repn, include_const): nl % tuple(map(self.var_id_to_nl_map.__getitem__, args)) ) except KeyError: - final_args = [] - for arg in args: - if arg in self.var_id_to_nl_map: - final_args.append(self.var_id_to_nl_map[arg]) - else: - _nl, _ids, _ = self.subexpression_cache[arg][1].compile_repn( - self.visitor - ) - final_args.append( - _nl % tuple(map(self.var_id_to_nl_map.__getitem__, _ids)) - ) - self.ostream.write(nl % tuple(final_args)) + self.ostream.write(self._resolve_subexpression_args(nl, args)) elif include_const: self.ostream.write(self.template.const % repn.const) From 4464b99c969660be7309097f59a05c6aee8b60e9 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 14 Jul 2024 15:51:32 -0600 Subject: [PATCH 133/150] Add test --- pyomo/repn/tests/ampl/test_nlv2.py | 69 ++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/pyomo/repn/tests/ampl/test_nlv2.py b/pyomo/repn/tests/ampl/test_nlv2.py index b6bb5f6c074..4d7b5d9ab6c 100644 --- a/pyomo/repn/tests/ampl/test_nlv2.py +++ b/pyomo/repn/tests/ampl/test_nlv2.py @@ -2703,3 +2703,72 @@ def test_presolve_check_invalid_monomial_constraints(self): r"\(fixed body value 5.0 outside bounds \[10, None\]\)\.", ): nl_writer.NLWriter().write(m, OUT, linear_presolve=True) + + def test_nested_external_expressions(self): + # This tests nested external functions in a single expression + DLL = find_GSL() + if not DLL: + self.skipTest("Could not find the amplgsl.dll library") + + m = ConcreteModel() + m.hypot = ExternalFunction(library=DLL, function="gsl_hypot") + m.p = Param(initialize=1, mutable=True) + m.x = Var(bounds=(None, 3)) + m.y = Var(bounds=(3, None)) + m.z = Var(initialize=1) + m.o = Objective(expr=m.z**2 * m.hypot(m.z, m.hypot(m.x, m.y)) ** 2) + m.c = Constraint(expr=m.x == m.y) + + OUT = io.StringIO() + nl_writer.NLWriter().write( + m, OUT, symbolic_solver_labels=True, linear_presolve=False + ) + self.assertEqual( + *nl_diff( + """g3 1 1 0 #problem unknown + 3 1 1 0 1 #vars, constraints, objectives, ranges, eqns + 0 1 0 0 0 0 #nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb + 0 0 #network constraints: nonlinear, linear + 0 3 0 #nonlinear vars in constraints, objectives, both + 0 1 0 1 #linear network variables; functions; arith, flags + 0 0 0 0 0 #discrete variables: binary, integer, nonlinear (b,c,o) + 2 3 #nonzeros in Jacobian, obj. gradient + 1 1 #max name lengths: constraints, variables + 0 0 0 0 0 #common exprs: b,c,o,c1,o1 +F0 1 -1 gsl_hypot +C0 #c +n0 +O0 0 #o +o2 #* +o5 #^ +v0 #z +n2 +o5 #^ +f0 2 #hypot +v0 #z +f0 2 #hypot +v1 #x +v2 #y +n2 +x1 #initial guess +0 1 #z +r #1 ranges (rhs's) +4 0 #c +b #3 bounds (on variables) +3 #z +1 3 #x +2 3 #y +k2 #intermediate Jacobian column lengths +0 +1 +J0 2 #c +1 1 +2 -1 +G0 3 #o +0 0 +1 0 +2 0 +""", + OUT.getvalue(), + ) + ) From 6d25c370393d34454d79711b4290c64aca6472ed Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 16 Jul 2024 08:01:29 -0600 Subject: [PATCH 134/150] Disable interface/testing for NEOS/octeract --- pyomo/neos/__init__.py | 2 +- pyomo/neos/tests/test_neos.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyomo/neos/__init__.py b/pyomo/neos/__init__.py index 7d18535e753..9f910f4a302 100644 --- a/pyomo/neos/__init__.py +++ b/pyomo/neos/__init__.py @@ -30,7 +30,7 @@ 'minos': 'SLC NLP solver', 'minto': 'MILP solver', 'mosek': 'Interior point NLP solver', - 'octeract': 'Deterministic global MINLP solver', + #'octeract': 'Deterministic global MINLP solver', 'ooqp': 'Convex QP solver', 'path': 'Nonlinear MCP solver', 'snopt': 'SQP NLP solver', diff --git a/pyomo/neos/tests/test_neos.py b/pyomo/neos/tests/test_neos.py index a4c4e9e6367..01b19a76b15 100644 --- a/pyomo/neos/tests/test_neos.py +++ b/pyomo/neos/tests/test_neos.py @@ -149,8 +149,11 @@ def test_minto(self): def test_mosek(self): self._run('mosek') - def test_octeract(self): - self._run('octeract') + # [16 Jul 24] Octeract is erroring. We will disable the interface + # (and testing) until we have time to resolve #3321 + # + # def test_octeract(self): + # self._run('octeract') def test_ooqp(self): if self.sense == pyo.maximize: From 5a8ea16c9965d6dbaca5a93fbea81ed2d10335b5 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 16 Jul 2024 08:31:01 -0600 Subject: [PATCH 135/150] Remove octeract as an expected solver interface --- pyomo/neos/tests/test_neos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyomo/neos/tests/test_neos.py b/pyomo/neos/tests/test_neos.py index 01b19a76b15..363368cd616 100644 --- a/pyomo/neos/tests/test_neos.py +++ b/pyomo/neos/tests/test_neos.py @@ -79,6 +79,9 @@ def test_doc(self): doc = pyomo.neos.doc dockeys = set(doc.keys()) + # Octeract interface is disabled, see #3321 + amplsolvers.pop('octeract') + self.assertEqual(amplsolvers, dockeys) # gamssolvers = set(v[0].lower() for v in tmp if v[1]=='GAMS') From e7a3711475bc905f6a4a608b4cf8cbcfb178b59e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 16 Jul 2024 09:05:52 -0600 Subject: [PATCH 136/150] bugfix: use correct set api --- pyomo/neos/tests/test_neos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/neos/tests/test_neos.py b/pyomo/neos/tests/test_neos.py index 363368cd616..681856781be 100644 --- a/pyomo/neos/tests/test_neos.py +++ b/pyomo/neos/tests/test_neos.py @@ -80,7 +80,7 @@ def test_doc(self): dockeys = set(doc.keys()) # Octeract interface is disabled, see #3321 - amplsolvers.pop('octeract') + amplsolvers.remove('octeract') self.assertEqual(amplsolvers, dockeys) From a8a78a78e9f5e68f811ee416aeb6154beade4122 Mon Sep 17 00:00:00 2001 From: Soren Davis Date: Wed, 17 Jul 2024 10:38:46 -0600 Subject: [PATCH 137/150] Be more direct with my comments --- pyomo/core/tests/unit/kernel/test_conic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/core/tests/unit/kernel/test_conic.py b/pyomo/core/tests/unit/kernel/test_conic.py index fc70b421060..bd97c13fc2e 100644 --- a/pyomo/core/tests/unit/kernel/test_conic.py +++ b/pyomo/core/tests/unit/kernel/test_conic.py @@ -786,8 +786,8 @@ def test_as_domain(self): x[1].value = None -# these mosek 10 constraints are really anemic and can't be evaluated, pprinted, -# checked for convexity, pickled, etc. +# These mosek 10 constraints can't be evaluated, pprinted, checked for convexity, +# pickled, etc., so I won't use the _conic_tester_base for them class Test_primal_geomean(unittest.TestCase): def test_as_domain(self): b = primal_geomean.as_domain(r=[2, 3], x=6) From 74ef05b0f38ece19b27b6acb18c1a2374e7fe7d8 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Tue, 23 Jul 2024 11:14:23 -0600 Subject: [PATCH 138/150] Change BARON download URL --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 8ba04eec466..03894a1cb20 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -519,7 +519,7 @@ jobs: $BARON_DIR = "${env:TPL_DIR}/baron" echo "$BARON_DIR" | ` Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - $URL = "https://www.minlp.com/downloads/xecs/baron/current/" + $URL = "https://minlp.com/downloads/xecs/baron/current/" if ( "${{matrix.TARGET}}" -eq "win" ) { $INSTALLER = "${env:DOWNLOAD_DIR}/baron_install.exe" $URL += "baron-win64.exe" diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 0bfd12b998d..cc9760cbe5d 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -562,7 +562,7 @@ jobs: $BARON_DIR = "${env:TPL_DIR}/baron" echo "$BARON_DIR" | ` Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - $URL = "https://www.minlp.com/downloads/xecs/baron/current/" + $URL = "https://minlp.com/downloads/xecs/baron/current/" if ( "${{matrix.TARGET}}" -eq "win" ) { $INSTALLER = "${env:DOWNLOAD_DIR}/baron_install.exe" $URL += "baron-win64.exe" From 08ccf404447c59d532fca8150f5b9233e0652abd Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 24 Jul 2024 10:52:34 -0600 Subject: [PATCH 139/150] Rename normalize_constraint -> to_bounded_expression --- pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp | 2 +- pyomo/contrib/appsi/cmodel/src/lp_writer.cpp | 2 +- pyomo/contrib/appsi/cmodel/src/nl_writer.cpp | 2 +- pyomo/core/base/constraint.py | 45 +++++++++++++++---- pyomo/core/kernel/constraint.py | 2 +- pyomo/gdp/plugins/bilinear.py | 2 +- pyomo/gdp/plugins/cuttingplane.py | 4 +- .../plugins/solvers/persistent_solver.py | 2 +- 8 files changed, 44 insertions(+), 17 deletions(-) diff --git a/pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp b/pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp index 708cfd9e073..ca865d429e2 100644 --- a/pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp +++ b/pyomo/contrib/appsi/cmodel/src/fbbt_model.cpp @@ -205,7 +205,7 @@ void process_fbbt_constraints(FBBTModel *model, PyomoExprTypes &expr_types, py::handle con_body; for (py::handle c : cons) { - lower_body_upper = c.attr("normalize_constraint")(); + lower_body_upper = c.attr("to_bounded_expression")(); con_lb = lower_body_upper[0]; con_body = lower_body_upper[1]; con_ub = lower_body_upper[2]; diff --git a/pyomo/contrib/appsi/cmodel/src/lp_writer.cpp b/pyomo/contrib/appsi/cmodel/src/lp_writer.cpp index 996bb34f564..f33060ee523 100644 --- a/pyomo/contrib/appsi/cmodel/src/lp_writer.cpp +++ b/pyomo/contrib/appsi/cmodel/src/lp_writer.cpp @@ -289,7 +289,7 @@ void process_lp_constraints(py::list cons, py::object writer) { py::object nonlinear_expr; PyomoExprTypes expr_types = PyomoExprTypes(); for (py::handle c : cons) { - lower_body_upper = c.attr("normalize_constraint")(); + lower_body_upper = c.attr("to_bounded_expression")(); cname = getSymbol(c, labeler); repn = generate_standard_repn( lower_body_upper[1], "compute_values"_a = false, "quadratic"_a = true); diff --git a/pyomo/contrib/appsi/cmodel/src/nl_writer.cpp b/pyomo/contrib/appsi/cmodel/src/nl_writer.cpp index 477bdd87aee..854262496ea 100644 --- a/pyomo/contrib/appsi/cmodel/src/nl_writer.cpp +++ b/pyomo/contrib/appsi/cmodel/src/nl_writer.cpp @@ -527,7 +527,7 @@ void process_nl_constraints(NLWriter *nl_writer, PyomoExprTypes &expr_types, py::handle repn_nonlinear_expr; for (py::handle c : cons) { - lower_body_upper = c.attr("normalize_constraint")(); + lower_body_upper = c.attr("to_bounded_expression")(); repn = generate_standard_repn( lower_body_upper[1], "compute_values"_a = false, "quadratic"_a = false); _const = appsi_expr_from_pyomo_expr(repn.attr("constant"), var_map, diff --git a/pyomo/core/base/constraint.py b/pyomo/core/base/constraint.py index defaea99dff..5a9d1da5af1 100644 --- a/pyomo/core/base/constraint.py +++ b/pyomo/core/base/constraint.py @@ -174,12 +174,35 @@ def __init__(self, expr=None, component=None): def __call__(self, exception=True): """Compute the value of the body of this constraint.""" - body = self.normalize_constraint()[1] + body = self.to_bounded_expression()[1] if body.__class__ not in native_numeric_types: body = value(self.body, exception=exception) return body - def normalize_constraint(self): + def to_bounded_expression(self): + """Convert this constraint to a tuple of 3 expressions (lb, body, ub) + + This method "standardizes" the expression into a 3-tuple of + expressions: (`lower_bound`, `body`, `upper_bound`). Upon + conversion, `lower_bound` and `upper_bound` are guaranteed to be + `None`, numeric constants, or fixed (not necessarily constant) + expressions. + + Note + ---- + As this method operates on the *current state* of the + expression, the any required expression manipulations (and by + extension, the result) can change after fixing / unfixing + :py:class:`Var` objects. + + Raises + ------ + + ValueError: Raised if the expression cannot be mapped to this + form (i.e., :py:class:`RangedExpression` constraints with + variable lower of upper bounds. + + """ expr = self._expr if expr.__class__ is RangedExpression: lb, body, ub = ans = expr.args @@ -217,8 +240,12 @@ def normalize_constraint(self): def body(self): """Access the body of a constraint expression.""" try: - ans = self.normalize_constraint()[1] + ans = self.to_bounded_expression()[1] except ValueError: + # It is possible that the expression is not currently valid + # (i.e., a ranged expression with a non-fixed bound). We + # will catch that exception here and - if this actually *is* + # a RangedExpression - return the body. if self._expr.__class__ is RangedExpression: _, ans, _ = self._expr.args else: @@ -229,14 +256,14 @@ def body(self): # # [JDS 6/2024: it would be nice to remove this behavior, # although possibly unnecessary, as people should use - # normalize_constraint() instead] + # to_bounded_expression() instead] return as_numeric(ans) return ans @property def lower(self): """Access the lower bound of a constraint expression.""" - ans = self.normalize_constraint()[0] + ans = self.to_bounded_expression()[0] if ans.__class__ in native_types and ans is not None: # Historically, constraint.lower was guaranteed to return a type # derived from Pyomo NumericValue (or None). Replicate that @@ -250,7 +277,7 @@ def lower(self): @property def upper(self): """Access the upper bound of a constraint expression.""" - ans = self.normalize_constraint()[2] + ans = self.to_bounded_expression()[2] if ans.__class__ in native_types and ans is not None: # Historically, constraint.upper was guaranteed to return a type # derived from Pyomo NumericValue (or None). Replicate that @@ -264,7 +291,7 @@ def upper(self): @property def lb(self): """Access the value of the lower bound of a constraint expression.""" - bound = self.normalize_constraint()[0] + bound = self.to_bounded_expression()[0] if bound is None: return None if bound.__class__ not in native_numeric_types: @@ -282,7 +309,7 @@ def lb(self): @property def ub(self): """Access the value of the upper bound of a constraint expression.""" - bound = self.normalize_constraint()[2] + bound = self.to_bounded_expression()[2] if bound is None: return None if bound.__class__ not in native_numeric_types: @@ -824,7 +851,7 @@ class SimpleConstraint(metaclass=RenamedClass): { 'add', 'set_value', - 'normalize_constraint', + 'to_bounded_expression', 'body', 'lower', 'upper', diff --git a/pyomo/core/kernel/constraint.py b/pyomo/core/kernel/constraint.py index 6b8c4c619f5..fe8eb8b2c1f 100644 --- a/pyomo/core/kernel/constraint.py +++ b/pyomo/core/kernel/constraint.py @@ -177,7 +177,7 @@ class _MutableBoundsConstraintMixin(object): # Define some of the IConstraint abstract methods # - def normalize_constraint(self): + def to_bounded_expression(self): return self.lower, self.body, self.upper @property diff --git a/pyomo/gdp/plugins/bilinear.py b/pyomo/gdp/plugins/bilinear.py index 70b6e83b52f..bc91836ea9c 100644 --- a/pyomo/gdp/plugins/bilinear.py +++ b/pyomo/gdp/plugins/bilinear.py @@ -77,7 +77,7 @@ def _transformBlock(self, block, instance): for component in block.component_data_objects( Constraint, active=True, descend_into=False ): - lb, body, ub = component.normalize_constraint() + lb, body, ub = component.to_bounded_expression() expr = self._transformExpression(body, instance) instance.bilinear_data_.c_body[id(component)] = body component.set_value((lb, expr, ub)) diff --git a/pyomo/gdp/plugins/cuttingplane.py b/pyomo/gdp/plugins/cuttingplane.py index a757f23c826..4cef098eba9 100644 --- a/pyomo/gdp/plugins/cuttingplane.py +++ b/pyomo/gdp/plugins/cuttingplane.py @@ -400,7 +400,7 @@ def back_off_constraint_with_calculated_cut_violation( val = value(transBlock_rHull.infeasibility_objective) - TOL if val <= 0: logger.info("\tBacking off cut by %s" % val) - lb, body, ub = cut.normalize_constraint() + lb, body, ub = cut.to_bounded_expression() cut.set_value((lb, body + abs(val), ub)) # else there is nothing to do: restore the objective transBlock_rHull.del_component(transBlock_rHull.infeasibility_objective) @@ -425,7 +425,7 @@ def back_off_constraint_by_fixed_tolerance( this callback TOL: An absolute tolerance to be added to make cut more conservative. """ - lb, body, ub = cut.normalize_constraint() + lb, body, ub = cut.to_bounded_expression() cut.set_value((lb, body + TOL, ub)) diff --git a/pyomo/solvers/plugins/solvers/persistent_solver.py b/pyomo/solvers/plugins/solvers/persistent_solver.py index ef96bfa339f..ef883fe5496 100644 --- a/pyomo/solvers/plugins/solvers/persistent_solver.py +++ b/pyomo/solvers/plugins/solvers/persistent_solver.py @@ -262,7 +262,7 @@ def _add_and_collect_column_data(self, var, obj_coef, constraints, coefficients) coeff_list = list() constr_list = list() for val, c in zip(coefficients, constraints): - lb, body, ub = c.normalize_constraint() + lb, body, ub = c.to_bounded_expression() body += val * var c.set_value((lb, body, ub)) self._vars_referenced_by_con[c].add(var) From 2529557649b39abd0e963632f0cb608a2a2a6627 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 24 Jul 2024 11:23:51 -0600 Subject: [PATCH 140/150] NFC: fix comment typo --- pyomo/contrib/fbbt/tests/test_fbbt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/fbbt/tests/test_fbbt.py b/pyomo/contrib/fbbt/tests/test_fbbt.py index ff1cc8a5cfb..83e69233bb5 100644 --- a/pyomo/contrib/fbbt/tests/test_fbbt.py +++ b/pyomo/contrib/fbbt/tests/test_fbbt.py @@ -1339,7 +1339,7 @@ def setUp(self) -> None: def test_ranged_expression(self): # The python version of FBBT is slightly more flexible than # APPSI's cmodel (it allows - and correctly handles - - # RangedExpressions with variable lower / upper bounds. If we + # RangedExpressions with variable lower / upper bounds). If we # ever port that functionality into APPSI, then this test can be # moved into the base class. m = pyo.ConcreteModel() From caa8cb58bb9040d1da347539e5a8e45df295a31b Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 25 Jul 2024 08:16:31 -0600 Subject: [PATCH 141/150] Allow maingo_solvermodel to be imported without maingopy --- pyomo/contrib/appsi/solvers/maingo.py | 40 +++++-------------- .../appsi/solvers/maingo_solvermodel.py | 12 +----- 2 files changed, 13 insertions(+), 39 deletions(-) diff --git a/pyomo/contrib/appsi/solvers/maingo.py b/pyomo/contrib/appsi/solvers/maingo.py index e52130061f7..c5860b42ce7 100644 --- a/pyomo/contrib/appsi/solvers/maingo.py +++ b/pyomo/contrib/appsi/solvers/maingo.py @@ -57,33 +57,13 @@ from pyomo.repn.util import valid_expr_ctypes_minlp -def _import_SolverModel(): - try: - from . import maingo_solvermodel - except ImportError: - raise - return maingo_solvermodel - - -maingo_solvermodel, solvermodel_available = attempt_import( - "maingo_solvermodel", importer=_import_SolverModel -) - -MaingoVar = namedtuple("MaingoVar", "type name lb ub init") - logger = logging.getLogger(__name__) - - -def _import_maingopy(): - try: - import maingopy - except ImportError: - MAiNGO._available = MAiNGO.Availability.NotFound - raise - return maingopy - - -maingopy, maingopy_available = attempt_import("maingopy", importer=_import_maingopy) +MaingoVar = namedtuple("MaingoVar", "type name lb ub init") +maingopy, maingopy_available = attempt_import("maingopy") +# Note that importing maingo_solvermodel will trigger the import of +# maingopy, so we defer that import using attempt_import (which will +# always succeed, even if maingopy is not available) +maingo_solvermodel = attempt_import("pyomo.contrib.appsi.solvers.maingo_solvermodel")[0] class MAiNGOConfig(MIPSolverConfig): @@ -185,9 +165,11 @@ def __init__(self, only_child_vars=False): self._last_results_object: Optional[MAiNGOResults] = None def available(self): - if not maingopy_available: - return self.Availability.NotFound - self._available = True + if self._available is None: + if maingopy_available: + MAiNGO._available = True + else: + MAiNGO._available = MAiNGO.Availability.NotFound return self._available def version(self): diff --git a/pyomo/contrib/appsi/solvers/maingo_solvermodel.py b/pyomo/contrib/appsi/solvers/maingo_solvermodel.py index ca746c4a9b7..b12a386284c 100644 --- a/pyomo/contrib/appsi/solvers/maingo_solvermodel.py +++ b/pyomo/contrib/appsi/solvers/maingo_solvermodel.py @@ -28,15 +28,7 @@ from pyomo.repn.util import valid_expr_ctypes_minlp -def _import_maingopy(): - try: - import maingopy - except ImportError: - raise - return maingopy - - -maingopy, maingopy_available = attempt_import("maingopy", importer=_import_maingopy) +maingopy, maingopy_available = attempt_import("maingopy") _plusMinusOne = {1, -1} @@ -219,7 +211,7 @@ def _linear_to_maingo(self, node): return sum(values) -class SolverModel(maingopy.MAiNGOmodel): +class SolverModel(maingopy.MAiNGOmodel if maingopy_available else object): def __init__(self, var_list, objective, con_list, idmap, logger): maingopy.MAiNGOmodel.__init__(self) self._var_list = var_list From ac479032cb52dc14708bcd8d499642b874e76332 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Jul 2024 05:36:58 -0600 Subject: [PATCH 142/150] Rework _initialize and avoid redundant call to iter() --- pyomo/core/base/set.py | 36 ++++++++++++++----------------- pyomo/core/tests/unit/test_set.py | 9 ++++++++ 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 87e9549ec8d..e8ac978c355 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1386,29 +1386,27 @@ def set_value(self, val): self.update(val) def _initialize(self, val): - self.update(val) + try: + self.update(val) + except TypeError as e: + if 'not iterable' in str(e): + logger.error( + "Initializer for Set %s returned non-iterable object " + "of type %s." + % ( + self.name, + (val if val.__class__ is type else type(val).__name__), + ) + ) + raise def update(self, values): - # _values was initialized above... - # # Special case: set operations that are not first attached # to the model must be constructed. if isinstance(values, SetOperator): values.construct() - try: - val_iter = iter(values) - except TypeError: - logger.error( - "Initializer for Set %s%s returned non-iterable object " - "of type %s." - % ( - self.name, - ("[%s]" % (index,) if self.is_indexed() else ""), - (values if values.__class__ is type else type(values).__name__), - ) - ) - raise - + # It is important that val_iter is an actual iterator + val_iter = iter(values) if self._dimen is not None: if normalize_index.flatten: val_iter = self._cb_normalized_dimen_verifier(self._dimen, val_iter) @@ -1472,8 +1470,6 @@ def _cb_validate(self, validate, block, val_iter): yield value def _cb_normalized_dimen_verifier(self, dimen, val_iter): - # It is important that the iterator is an actual iterator - val_iter = iter(val_iter) for value in val_iter: if value.__class__ is tuple: if dimen == len(value): @@ -1833,7 +1829,7 @@ def _initialize(self, val): "This WILL potentially lead to nondeterministic behavior " "in Pyomo" % (self.name, type(val).__name__) ) - super().update(val) + super()._initialize(val) def set_value(self, val): if type(val) in Set._UnorderedInitializers: diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index ff0aaa5600b..7fe7d1fd7ae 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -3811,6 +3811,7 @@ def I_init(m): self.assertEqual(m.I.data(), (4, 3, 2, 1)) self.assertEqual(m.I.dimen, 1) + def test_initialize_with_noniterable(self): output = StringIO() with LoggingIntercept(output, 'pyomo.core'): with self.assertRaisesRegex(TypeError, "'int' object is not iterable"): @@ -3819,6 +3820,14 @@ def I_init(m): ref = "Initializer for Set I returned non-iterable object of type int." self.assertIn(ref, output.getvalue()) + output = StringIO() + with LoggingIntercept(output, 'pyomo.core'): + with self.assertRaisesRegex(TypeError, "'int' object is not iterable"): + m = ConcreteModel() + m.I = Set([1,2], initialize=5) + ref = "Initializer for Set I[1] returned non-iterable object of type int." + self.assertIn(ref, output.getvalue()) + def test_scalar_indexed_api(self): m = ConcreteModel() m.I = Set(initialize=range(3)) From 9aab8c2e487cefd8f0ed05ef19351cf19ed6e88b Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Jul 2024 05:37:35 -0600 Subject: [PATCH 143/150] Duplicate _update_impl workaround from ordered sets in sorted sets --- pyomo/core/base/set.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index e8ac978c355..b62f61f8557 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1897,8 +1897,12 @@ def __reversed__(self): def _update_impl(self, values): for val in values: + # Note that we reset _ordered_values within the loop because + # of an old example where the initializer rule makes + # reference to values previously inserted into the Set + # (which triggered the creation of the _ordered_values) + self._ordered_values = None self._values[val] = None - self._ordered_values = None # Note: removing data does not affect the sorted flag # def remove(self, val): From 26488f38c49961d5b8c6ac5b847096c720971edb Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Jul 2024 07:26:58 -0600 Subject: [PATCH 144/150] Catch (and test) an edge case when flattening sets --- pyomo/core/base/set.py | 19 +++++++++++-------- pyomo/core/tests/unit/test_set.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index b62f61f8557..045fc134f74 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1471,16 +1471,19 @@ def _cb_validate(self, validate, block, val_iter): def _cb_normalized_dimen_verifier(self, dimen, val_iter): for value in val_iter: - if value.__class__ is tuple: - if dimen == len(value): - yield value[0] if dimen == 1 else value + if value.__class__ in native_types: + if dimen == 1: + yield value continue - elif dimen == 1 and value.__class__ in native_types: - yield value - continue + normalized_value = value + else: + normalized_value = normalize_index(value) + # Note: normalize_index() will never return a 1-tuple + if normalized_value.__class__ is tuple: + if dimen == len(normalized_value): + yield normalized_value[0] if dimen == 1 else normalized_value + continue - # Note: normalize_index() will never return a 1-tuple - normalized_value = normalize_index(value) _d = len(normalized_value) if normalized_value.__class__ is tuple else 1 if _d == dimen: yield normalized_value diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 7fe7d1fd7ae..2a2651ca554 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -5256,6 +5256,21 @@ def Bindex(m): self.assertIs(m.K.index_set()._domain, Integers) self.assertEqual(m.K.index_set(), [0, 1, 2, 3, 4]) + def test_normalize_index(self): + try: + _oldFlatten = normalize_index.flatten + normalize_index.flatten = True + + m = ConcreteModel() + with self.assertRaisesRegex( + ValueError, + r"The value=\(\(2, 3\),\) has dimension 2 and is not " + "valid for Set I which has dimen=1", + ): + m.I = Set(initialize=[1, ((2, 3),)]) + finally: + normalize_index.flatten = _oldFlatten + def test_no_normalize_index(self): try: _oldFlatten = normalize_index.flatten From 0fc78ad21925e647da1f58483e2ab37be48f716a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Jul 2024 07:27:25 -0600 Subject: [PATCH 145/150] NFC: apply black --- pyomo/core/tests/unit/test_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 2a2651ca554..8c0360d618b 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -3824,7 +3824,7 @@ def test_initialize_with_noniterable(self): with LoggingIntercept(output, 'pyomo.core'): with self.assertRaisesRegex(TypeError, "'int' object is not iterable"): m = ConcreteModel() - m.I = Set([1,2], initialize=5) + m.I = Set([1, 2], initialize=5) ref = "Initializer for Set I[1] returned non-iterable object of type int." self.assertIn(ref, output.getvalue()) From 49a53d89cef82445091afb352a59129914b73f0e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Jul 2024 07:46:49 -0600 Subject: [PATCH 146/150] NFC: fix doc typo Co-authored-by: Bethany Nicholson --- pyomo/core/base/constraint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/core/base/constraint.py b/pyomo/core/base/constraint.py index 5a9d1da5af1..b79bc178e80 100644 --- a/pyomo/core/base/constraint.py +++ b/pyomo/core/base/constraint.py @@ -191,7 +191,7 @@ def to_bounded_expression(self): Note ---- As this method operates on the *current state* of the - expression, the any required expression manipulations (and by + expression, any required expression manipulations (and by extension, the result) can change after fixing / unfixing :py:class:`Var` objects. From 21502959923070e54076d943647fcaa723a224cc Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Jul 2024 07:47:04 -0600 Subject: [PATCH 147/150] NFC: fix doc typo Co-authored-by: Bethany Nicholson --- pyomo/core/base/constraint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/core/base/constraint.py b/pyomo/core/base/constraint.py index b79bc178e80..bc9a32f5404 100644 --- a/pyomo/core/base/constraint.py +++ b/pyomo/core/base/constraint.py @@ -200,7 +200,7 @@ def to_bounded_expression(self): ValueError: Raised if the expression cannot be mapped to this form (i.e., :py:class:`RangedExpression` constraints with - variable lower of upper bounds. + variable lower or upper bounds. """ expr = self._expr From d8900b296b698b29d5b285dd115cd899014994c0 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Jul 2024 08:10:54 -0600 Subject: [PATCH 148/150] Remove redundant header, simplify imports --- pyomo/contrib/sensitivity_toolbox/sens.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/pyomo/contrib/sensitivity_toolbox/sens.py b/pyomo/contrib/sensitivity_toolbox/sens.py index a3d69b2c7b1..818f13cb789 100644 --- a/pyomo/contrib/sensitivity_toolbox/sens.py +++ b/pyomo/contrib/sensitivity_toolbox/sens.py @@ -9,16 +9,6 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -# ______________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License -# ______________________________________________________________________________ from pyomo.environ import ( Param, Var, @@ -36,6 +26,7 @@ from pyomo.core.expr import ExpressionReplacementVisitor from pyomo.common.modeling import unique_component_name +from pyomo.common.dependencies import numpy as np, scipy from pyomo.common.deprecation import deprecated from pyomo.common.tempfiles import TempfileManager from pyomo.opt import SolverFactory, SolverStatus @@ -44,8 +35,6 @@ import os import io import shutil -from pyomo.common.dependencies import numpy as np, numpy_available -from pyomo.common.dependencies import scipy, scipy_available logger = logging.getLogger('pyomo.contrib.sensitivity_toolbox') From bb17e9c9bc115238c34ff67fd9e3354eff63c782 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Jul 2024 08:11:41 -0600 Subject: [PATCH 149/150] Update constraint processing to leverage new Constraint internal storage --- pyomo/contrib/sensitivity_toolbox/sens.py | 35 +++++++++++++---------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/pyomo/contrib/sensitivity_toolbox/sens.py b/pyomo/contrib/sensitivity_toolbox/sens.py index 818f13cb789..34fbb92327a 100644 --- a/pyomo/contrib/sensitivity_toolbox/sens.py +++ b/pyomo/contrib/sensitivity_toolbox/sens.py @@ -24,6 +24,7 @@ from pyomo.common.sorting import sorted_robust from pyomo.core.expr import ExpressionReplacementVisitor +from pyomo.core.expr.numvalue import is_potentially_variable from pyomo.common.modeling import unique_component_name from pyomo.common.dependencies import numpy as np, scipy @@ -673,25 +674,29 @@ def _replace_parameters_in_constraints(self, variableSubMap): ) last_idx = 0 for con in old_con_list: - if con.equality or con.lower is None or con.upper is None: - new_expr = param_replacer.walk_expression(con.expr) - block.constList.add(expr=new_expr) + new_expr = param_replacer.walk_expression(con.expr) + # TODO: We could only create new constraints for expressions + # where substitution actually happened, but that breaks some + # current tests: + # + # if new_expr is con.expr: + # # No params were substituted. We can ignore this constraint + # continue + if new_expr.nargs() == 3 and ( + is_potentially_variable(new_expr.arg(0)) + or is_potentially_variable(new_expr.arg(2)) + ): + # This is a potentially "invalid" range constraint: it + # may now have variables in the bounds. For safety, we + # will split it into two simple inequalities. + block.constList.add(expr=(new_expr.arg(0) <= new_expr.arg(1))) last_idx += 1 new_old_comp_map[block.constList[last_idx]] = con - else: - # Constraint must be a ranged inequality, break into - # separate constraints - new_body = param_replacer.walk_expression(con.body) - new_lower = param_replacer.walk_expression(con.lower) - new_upper = param_replacer.walk_expression(con.upper) - - # Add constraint for lower bound - block.constList.add(expr=(new_lower <= new_body)) + block.constList.add(expr=(new_expr.arg(1) <= new_expr.arg(2))) last_idx += 1 new_old_comp_map[block.constList[last_idx]] = con - - # Add constraint for upper bound - block.constList.add(expr=(new_body <= new_upper)) + else: + block.constList.add(expr=new_expr) last_idx += 1 new_old_comp_map[block.constList[last_idx]] = con con.deactivate() From 7df3801d9679d88e4f195fd38bc6f0a6750a08c2 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Jul 2024 08:28:11 -0600 Subject: [PATCH 150/150] Avoid duplicate logging of unordered Set data warning --- pyomo/core/base/set.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index fadd0c3e7d0..e4a6d13e96e 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1387,7 +1387,10 @@ def set_value(self, val): def _initialize(self, val): try: - self.update(val) + # We want to explicitly call the update() on *this class* to + # bypass potential double logging of the use of unordered + # data with ordered Sets + FiniteSetData.update(self, val) except TypeError as e: if 'not iterable' in str(e): logger.error(