diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..cbc3411 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black \ No newline at end of file diff --git a/engforge.egg-info/PKG-INFO b/engforge.egg-info/PKG-INFO index 694e267..2ad2f47 100644 --- a/engforge.egg-info/PKG-INFO +++ b/engforge.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: engforge -Version: 0.0.9 +Version: 0.1.0 Summary: The Engineer's Framework Home-page: https://github.com/SoundsSerious/engforge Author: Kevin russell diff --git a/engforge/_testing_components.py b/engforge/_testing_components.py index 48512d7..cf43fa8 100644 --- a/engforge/_testing_components.py +++ b/engforge/_testing_components.py @@ -852,7 +852,9 @@ def calculate_production(self, parent, term): @forge class FanSystem(System, CostModel): - base = Slot.define(Component) + base = Slot.define( + Component + ) # FIXME: not showing in "econ" (due to base default cost?) fan = Slot.define(Fan) motor = Slot.define(Motor) diff --git a/engforge/attributes.py b/engforge/attributes.py index 697a470..1853384 100644 --- a/engforge/attributes.py +++ b/engforge/attributes.py @@ -186,11 +186,14 @@ def define(cls, **kwargs): def _setup_cls(cls, name, new_dict, **kwargs): # randomize name for specifics reasons uid = str(uuid.uuid4()) + # base info name = name + "_" + uid.replace("-", "")[0:16] new_dict["uuid"] = uid new_dict["default_options"] = cls.default_options.copy() new_dict["template_class"] = False new_dict["name"] = name + + # create a new type of attribute new_slot = type(name, (cls,), new_dict) new_slot.default_options["default"] = new_slot.make_factory() new_slot.default_options["validator"] = new_slot.configure_instance diff --git a/engforge/configuration.py b/engforge/configuration.py index ec93767..add80b4 100644 --- a/engforge/configuration.py +++ b/engforge/configuration.py @@ -170,12 +170,12 @@ def property_changed(instance, variable, value): # elif session: # notify session that this variable has changed # log.info(f'property changed {variable.name} {value}') + # TODO: notify change input system here, perhaps with & without a session # session.change_sys_var(variable,value,doset=False) - attrs = attr.fields(instance.__class__) # check identity of variable cur = getattr(instance, variable.name) - is_different = value != cur + is_different = value != cur if isinstance(value, (int, float, str)) else True is_var = variable in attrs chgnw = instance._anything_changed @@ -183,6 +183,9 @@ def property_changed(instance, variable, value): log.debug( f"checking property changed {instance}{variable.name} {value}|invar: {is_var}| nteqval: {is_different}" ) + print( + f"checking property changed {instance}{variable.name} {value}|invar: {is_var}| nteqval: {is_different}" + ) # Check if should be updated if not chgnw and is_var and is_different: @@ -317,6 +320,7 @@ def signals_slots_handler( cls_properties = cls.system_properties_classdef(True) else: cls_properties = {} + cls_dict = cls.__dict__.copy() cls.__anony_store = {} # print(f'tab found!! {cls_properties.keys()}') @@ -460,6 +464,7 @@ def copy_config_at_state( new_sys.system_references(recache=True) # update the parents + # TODO: use pyee to broadcast change if hasattr(self, "parent"): if self.parent in changed: new_sys.parent = changed[self.parent] @@ -481,6 +486,8 @@ def go_through_configurations( :return: level,config""" from engforge.configuration import Configuration + # TODO: instead of a recursive loop a global map per problem context should be used, with a static map of slots, updating with every change per note in system_references. This function could be a part of that but each system shouldn't be responsible for it. + should_yield_level = lambda level: all( [ level >= parent_level, @@ -546,9 +553,9 @@ def __attrs_post_init__(self): # TODO: allow multiple parents if (hasattr(comp, "parent")) and (comp.parent is not None): self.warning( - f"Component {compnm} already has a parent {comp.parent} copying, and assigning to {self}" + f"Component {compnm} already has a parent {comp.parent} #copying, and assigning to {self}" ) - setattr(self, compnm, attrs.evolve(comp, parent=self)) + # setattr(self, compnm, attrs.evolve(comp, parent=self)) else: comp.parent = self diff --git a/engforge/dataframe.py b/engforge/dataframe.py index a72bb28..d70735c 100644 --- a/engforge/dataframe.py +++ b/engforge/dataframe.py @@ -32,9 +32,11 @@ class DataFrameLog(LoggingMixin): # Dataframe interrogation functions def is_uniform(s: pandas.Series): a = s.to_numpy() # s.values (pandas<0.24) - if (a[0] == a).all(): + + if numpy.all(a == a[0]): return True try: + # if not numpy.isfinite(a).any(): return True except: @@ -107,8 +109,8 @@ def split_dataframe(df: pandas.DataFrame) -> tuple: class DataframeMixin: dataframe: pandas.DataFrame - _split_dataframe_func = split_dataframe - _determine_split_func = determine_split + _split_dataframe_func = staticmethod(split_dataframe) + _determine_split_func = staticmethod(determine_split) def smart_split_dataframe(self, df=None, split_groups=0, key_f=key_func): """splits dataframe between constant values and variants""" @@ -166,9 +168,8 @@ def dataframe_variants(self): return o def format_columns(self, dataframe: pandas.DataFrame): - dataframe.rename( - lambda x: x.replace(".", "_").lower(), axis="columns", inplace=True - ) + # replace(".", "_") + dataframe.rename(lambda x: x.lower(), axis="columns", inplace=True) # Plotting Interface @property diff --git a/engforge/datastores/data.py b/engforge/datastores/data.py index 07a69fc..a7f798e 100644 --- a/engforge/datastores/data.py +++ b/engforge/datastores/data.py @@ -195,10 +195,16 @@ class DiskCacheStore(LoggingMixin, metaclass=SingletonMeta): _current_keys = None expire_threshold = 60.0 - retries = 3 + retries = 1 sleep_time = 0.1 - def __init__(self, **kwargs): + proj_dir: str = None # avoid using implicit determinination + cache_path: str = None # override for implicit path, a recommended practice + + def __init__(self, root_path=None, **kwargs): + if root_path is not None: + self.cache_path = root_path + if kwargs: self.cache_init_kwargs = kwargs else: @@ -206,13 +212,23 @@ def __init__(self, **kwargs): self.info(f"Created DiskCacheStore In {self.cache_root}") self.cache + @property + def proj_root(self): + if self.proj_dir is not None: + return self.proj_dir + return client_path(skip_wsl=False) + @property def cache_root(self): # TODO: CHECK CACHE IS NOT SYNCED TO DROPBOX + if self.cache_path is not None: + return self.cache_path + if self.alt_path is not None: - return os.path.join(client_path(skip_wsl=False), "cache", self.alt_path) + return os.path.join(self.proj_root, "cache", self.alt_path) + return os.path.join( - client_path(skip_wsl=False), + self.proj_root, "cache", "{}".format(type(self).__name__).lower(), ) @@ -247,7 +263,7 @@ def set(self, key=None, data=None, retry=True, ttl=None, **kwargs): time.sleep(self.sleep_time * (self.retries - ttl)) return self.set(key=key, data=data, retry=True, ttl=ttl) else: - self.error(e, "Issue Getting Item From Cache") + self.error(e, "Issue Setting Item In Cache") # @ray_cache def get(self, key=None, on_missing=None, retry=True, ttl=None): diff --git a/engforge/dynamics.py b/engforge/dynamics.py index be56a4c..2b94abd 100644 --- a/engforge/dynamics.py +++ b/engforge/dynamics.py @@ -79,7 +79,7 @@ def remap_indexes_to(self, new_index, *args, invert=False, old_data=None): old_data = self.data opt1 = {arg: self.indify(old_data, arg)[0] for arg in args} opt2 = { - arg: self.indify(old_data, val)[0] if (not isinstance(val, str)) else val + arg: (self.indify(old_data, val)[0] if (not isinstance(val, str)) else val) for arg, val in opt1.items() } oop1 = {arg: self.indify(new_index, val)[0] for arg, val in opt2.items()} diff --git a/engforge/eng/costs.py b/engforge/eng/costs.py index d116222..c913d1c 100644 --- a/engforge/eng/costs.py +++ b/engforge/eng/costs.py @@ -4,16 +4,22 @@ CostModel's can have cost_property's which detail how and when a cost should be applied & grouped. By default each CostModel has a `cost_per_item` which is reflected in `item_cost` cost_property set on the `initial` term as a `unit` category. Multiple categories of cost are also able to be set on cost_properties as follows +Cost Models can represent multiple instances of a component, and can be set to have a `num_items` multiplier to account for multiple instances of a component. CostModels can have a `term_length` which will apply costs over the term, using the `cost_property.mode` to determine at which terms a cost should be applied. + ``` @forge class Widget(Component,CostModel): + + #use num_items as a multiplier for costs, `cost_properties` can have their own custom num_items value. + num_items:float = 100 + + @cost_property(mode='initial',category='capex,manufacturing',num_items=1) + def cost_of_XYZ(self) -> float: + return cost - @cost_property(mode='initial',category='capex,manufacturing') - def cost_of_XYZ(self): - return ... ``` -Economics models sum CostModel.cost_properties recursively on the parent they are defined. Economics computes the grouped category costs for each item recursively as well as summary properties like annualized values and levalized cost. Economic output is determined by a `fixed_output` or overriding `calculate_production(self,parent)` to dynamically calculate changing economics based on factors in the parent. +Economics models sum CostModel.cost_properties recursively on the parent they are defined. Economics computes the grouped category costs for each item recursively as well as summary properties like annualized values and levelized cost. Economic output is determined by a `fixed_output` or overriding `calculate_production(self,parent)` to dynamically calculate changing economics based on factors in the parent. Default costs can be set on any CostModel.Slot attribute, by using default_cost(,) on the class, this will provide a default cost for the slot if no cost is set on the instance. Custom costs can be set on the instance with custom_cost(,). If cost is a CostModel, it will be assigned to the slot if it is not already assigned. @@ -47,6 +53,8 @@ class Parent(System,CostModel) import numpy import collections import pandas +import collections +import re class CostLog(LoggingMixin): @@ -58,15 +66,26 @@ class CostLog(LoggingMixin): # Cost Term Modes are a quick lookup for cost term support global COST_TERM_MODES, COST_CATEGORIES COST_TERM_MODES = { - "initial": lambda inst, term: True if term < 1 else False, - "maintenance": lambda inst, term: True if term >= 1 else False, - "always": lambda inst, term: True, + "initial": lambda inst, term, econ: True if term < 1 else False, + "maintenance": lambda inst, term, econ: True if term >= 1 else False, + "always": lambda inst, term, econ: True, + "end": lambda inst, term, econ: ( + True if hasattr(econ, "term_length") and term == econ.term_length - 1 else False + ), } category_type = typing.Union[str, list] COST_CATEGORIES = set(("misc",)) +def get_num_from_cost_prop(ref): + """analyzes the reference and returns the number of items""" + if isinstance(ref, (float, int)): + return ref + co = getattr(ref.comp.__class__, ref.key, None) + return co.get_num_items(ref.comp) + + class cost_property(system_property): """A thin wrapper over `system_property` that will be accounted by `Economics` Components and apply term & categorization @@ -81,8 +100,10 @@ class cost_property(system_property): """ + valild_types = (int, float) cost_categories: list = None term_mode: str = None + num_items: int = None _all_modes: dict = COST_TERM_MODES _all_categories: set = COST_CATEGORIES @@ -98,11 +119,14 @@ def __init__( stochastic=False, mode: str = "initial", category: category_type = None, + num_items: int = None, ): """extends system_property interface with mode & category keywords - :param mode: can be one of `initial`,`maintenance`,`always` or a function with signature f(inst,term) as an integer and returning a boolean True if it is to be applied durring that term. + :param mode: can be one of `initial`,`maintenance`,`always` or a function with signature f(inst,term,econ) as an integer and returning a boolean True if it is to be applied durring that term. """ super().__init__(fget, fset, fdel, doc, desc, label, stochastic) + + self.valild_types = (int, str) # only numerics if isinstance(mode, str): mode = mode.lower() assert ( @@ -116,6 +140,7 @@ def __init__( else: raise ValueError(f"mode: {mode} must be cost term str or callable") + # cost categories if category is not None: if isinstance(category, str): self.cost_categories = category.split(",") @@ -128,10 +153,14 @@ def __init__( else: self.cost_categories = ["misc"] - def apply_at_term(self, inst, term): + # number of items override + if num_items is not None: + self.num_items = num_items + + def apply_at_term(self, inst, term, econ=None): if term < 0: raise ValueError(f"negative term!") - if self.__class__._all_modes[self.term_mode](inst, term): + if self.__class__._all_modes[self.term_mode](inst, term, econ): return True return False @@ -146,6 +175,24 @@ def get_func_return(self, func): else: self.return_type = float + def get_num_items(self, obj): + """applies the num_items override or the costmodel default if not set""" + if self.num_items is not None: + k = self.num_items + else: + k = obj.num_items if isinstance(obj, CostModel) else 1 + return k + + def __get__(self, obj, objtype=None): + if obj is None: + return self # class support + if self.fget is None: + raise AttributeError("unreadable attribute") + + # apply the costmodel with the item multiplier + k = self.get_num_items(obj) + return k * self.fget(obj) + @forge class CostModel(Configuration, TabulationMixin): @@ -156,9 +203,13 @@ class CostModel(Configuration, TabulationMixin): `sub_items_cost` system_property summarizes the costs of any component in a Slot that has a `CostModel` or for SlotS which CostModel.declare_cost(`slot`,default=numeric|CostModelInst|dict[str,float]) """ + # TODO: remove "default costs" concept and just use cost_properties since thats declarative and doesn't create a "phantom" representation to maintain + # TODO: it might be a good idea to add a "castable" namespace for components so they can all reference a common dictionary. Maybe all problem variables are merged into a single namespace to solve issues of "duplication" _slot_costs: dict # TODO: insantiate per class + # basic attribute interface returns the item cost as `N x cost_per_item`` cost_per_item: float = attrs.field(default=numpy.nan) + num_items: int = attrs.field(default=1) # set to 0 to disable the item cost def __on_init__(self): self.set_default_costs() @@ -227,8 +278,8 @@ def default_cost( cost, CostModel ), "only numeric types or CostModel instances supported" - atrb = cls.slots_attributes()[slot_name] - atypes = atrb.type.accepted + atrb = cls.slots_attributes(attr_type=True)[slot_name] + atypes = atrb.accepted if warn_on_non_costmodel and not any( [issubclass(at, CostModel) for at in atypes] ): @@ -253,8 +304,8 @@ def custom_cost( cost, CostModel ), "only numeric types or CostModel instances supported" - atrb = self.__class__.slots_attributes()[slot_name] - atypes = atrb.type.accepted + atrb = self.__class__.slots_attributes(attr_type=True)[slot_name] + atypes = atrb.accepted if warn_on_non_costmodel and not any( [issubclass(at, CostModel) for at in atypes] ): @@ -276,7 +327,7 @@ def custom_cost( def calculate_item_cost(self) -> float: """override this with a parametric model related to this systems attributes and properties""" - return self.cost_per_item + return self.num_items * self.cost_per_item @system_property def sub_items_cost(self) -> float: @@ -290,6 +341,7 @@ def item_cost(self) -> float: @system_property def combine_cost(self) -> float: + """the sum of all cost properties""" return self.sum_costs() @system_property @@ -304,29 +356,40 @@ def future_costs(self) -> float: initial_costs = self.costs_at_term(0, False) return numpy.nansum(list(initial_costs.values())) - def sum_costs(self, saved: set = None, categories: tuple = None, term=0): + def sum_costs(self, saved: set = None, categories: tuple = None, term=0, econ=None): """sums costs of cost_property's in this item that are present at term=0, and by category if define as input""" if saved is None: saved = set((self,)) # item cost included! elif self not in saved: saved.add(self) - itemcst = list(self.dict_itemized_costs(saved, categories, term).values()) + itemcst = list( + self.dict_itemized_costs(saved, categories, term, econ=econ).values() + ) csts = [self.sub_costs(saved, categories, term), numpy.nansum(itemcst)] return numpy.nansum(csts) def dict_itemized_costs( - self, saved: set = None, categories: tuple = None, term=0, test_val=True + self, + saved: set = None, + categories: tuple = None, + term=0, + test_val=True, + econ=None, ) -> dict: ccp = self.class_cost_properties() costs = { - k: obj.__get__(self) if obj.apply_at_term(self, term) == test_val else 0 + k: ( + obj.__get__(self) + if obj.apply_at_term(self, term, econ) == test_val + else 0 + ) for k, obj in ccp.items() if categories is None or any([cc in categories for cc in obj.cost_categories]) } return costs - def sub_costs(self, saved: set = None, categories: tuple = None, term=0): + def sub_costs(self, saved: set = None, categories: tuple = None, term=0, econ=None): """gets items from CostModel's defined in a Slot attribute or in a slot default, tolerrant to nan's in cost definitions""" if saved is None: saved = set() @@ -344,7 +407,7 @@ def sub_costs(self, saved: set = None, categories: tuple = None, term=0): saved.add(comp) if isinstance(comp, CostModel): - sub = comp.sum_costs(saved, categories, term) + sub = comp.sum_costs(saved, categories, term, econ=econ) log.debug( f"{self.identity} adding: {comp.identity if comp else comp}: {sub}+{sub_tot}" ) @@ -368,7 +431,7 @@ def sub_costs(self, saved: set = None, categories: tuple = None, term=0): # add base class slot values when comp was nonee if comp is None: # print(f'skipping {slot}:{comp}') - comp_cls = self.slots_attributes()[slot].type.accepted + comp_cls = self.slots_attributes(attr_type=True)[slot].accepted for cc in comp_cls: if issubclass(cc, CostModel): if cc._slot_costs: @@ -384,12 +447,19 @@ def sub_costs(self, saved: set = None, categories: tuple = None, term=0): return sub_tot # Cost Term & Category Reporting - def costs_at_term(self, term: int, test_val=True) -> dict: + def costs_at_term(self, term: int, test_val=True, econ=None) -> dict: """returns a dictionary of all costs at term i, with zero if the mode - function returns False at that term""" + function returns False at that term + + :param econ: the economics component to apply "end" term mode + """ ccp = self.class_cost_properties() return { - k: obj.__get__(self) if obj.apply_at_term(self, term) == test_val else 0 + k: ( + obj.__get__(self) + if obj.apply_at_term(self, term, econ) == test_val + else 0 + ) for k, obj in ccp.items() } @@ -398,7 +468,7 @@ def class_cost_properties(cls) -> dict: """returns cost_property objects from this class & subclasses""" return { k: v - for k, v in cls.system_properties_classdef().items() + for k, v in cls.system_properties_classdef(True).items() if isinstance(v, cost_property) } @@ -417,10 +487,10 @@ def cost_categories(self): base[cc] += obj.__get__(self) return base - def cost_categories_at_term(self, term: int): + def cost_categories_at_term(self, term: int, econ=None): base = {cc: 0 for cc in self.all_categories()} for k, obj in self.class_cost_properties().items(): - if obj.apply_at_term(self, term): + if obj.apply_at_term(self, term, econ): for cc in obj.cost_categories: base[cc] += obj.__get__(self) return base @@ -465,7 +535,41 @@ def gend(deect: dict): # TODO: automatically apply economics at problem level if cost_model present, no need for parent econ lookups @forge class Economics(Component): - """Economics is a component that summarizes costs and reports the economics of a system and its components in a recursive format""" + """ + Economics is a component that summarizes costs and reports the economics of a system and its components in a recursive format. + Attributes: + term_length (int): The length of the term for economic calculations. Default is 0. + discount_rate (float): The discount rate applied to economic calculations. Default is 0.0. + fixed_output (float): The fixed output value for the economic model. Default is numpy.nan. + output_type (str): The type of output for the economic model. Default is "generic". + terms_per_year (int): The number of terms per year for economic calculations. Default is 1. + _calc_output (float): Internal variable to store calculated output. + _costs (float): Internal variable to store calculated costs. + _cost_references (dict): Internal dictionary to store cost references. + _cost_categories (dict): Internal dictionary to store cost categories. + _comp_categories (dict): Internal dictionary to store component categories. + _comp_costs (dict): Internal dictionary to store component costs. + parent (parent_types): The parent component or system. + Methods: + __on_init__(): Initializes internal dictionaries for cost and component categories. + update(parent: parent_types): Updates the economic model with the given parent component or system. + calculate_production(parent, term) -> float: Calculates the production output for the given parent and term. Must be overridden. + calculate_costs(parent) -> float: Recursively calculates the costs for the given parent component or system. + sum_cost_references(): Sums the cost references stored in the internal dictionary. + sum_references(refs): Sums the values of the given references. + get_prop(ref): Retrieves the property associated with the given reference. + term_fgen(comp, prop): Generates a function to calculate the term value for the given component and property. + sum_term_fgen(ref_group): Sums the term functions for the given reference group. + internal_references(recache=True, numeric_only=False): Gathers and sets internal references for the economic model. + lifecycle_output() -> dict: Returns lifecycle calculations for the levelized cost of energy (LCOE). + lifecycle_dataframe() -> pandas.DataFrame: Simulates the economics lifecycle and stores the results in a term-based dataframe. + _create_term_eval_functions(): Creates evaluation functions for term-based calculations grouped by categories and components. + _gather_cost_references(parent: "System"): Gathers cost references from the parent component or system. + _extract_cost_references(conf: "CostModel", bse: str): Extracts cost references from the given cost model configuration. + cost_references(): Returns the internal dictionary of cost references. + combine_cost() -> float: Returns the combined cost of the economic model. + output() -> float: Returns the calculated output of the economic model. + """ term_length: int = attrs.field(default=0) discount_rate: float = attrs.field(default=0.0) @@ -502,6 +606,37 @@ def update(self, parent: parent_types): if self._costs is None: self.warning(f"no economic costs!") + self._anything_changed = True + + # TODO: expand this... + @solver_cached + def econ_output(self): + return self.lifecycle_output + + @system_property(label="cost/output") + def levelized_cost(self) -> float: + """Price per kwh""" + eco = self.econ_output + return eco["summary.levelized_cost"] + + @system_property + def total_cost(self) -> float: + eco = self.econ_output + return eco["summary.total_cost"] + + @system_property + def levelized_output(self) -> float: + """ouput per dollar (KW/$)""" + eco = self.econ_output + return eco["summary.levelized_output"] + + @system_property + def term_years(self) -> float: + """ouput per dollar (KW/$)""" + eco = self.econ_output + return eco["summary.years"] + + # Calculate Output def calculate_production(self, parent, term) -> float: """must override this function and set economic_output""" return numpy.nansum([0, self.fixed_output]) @@ -511,6 +646,7 @@ def calculate_costs(self, parent) -> float: return self.sum_cost_references() # Reference Utilitly Functions + # Set Costs over time "flat" through ref trickery... def sum_cost_references(self): cst = 0 for k, v in self._cost_references.items(): @@ -540,7 +676,9 @@ def get_prop(self, ref): def term_fgen(self, comp, prop): if isinstance(comp, dict): return lambda term: comp[prop] if term == 0 else 0 - return lambda term: prop.__get__(comp) if prop.apply_at_term(comp, term) else 0 + return lambda term: ( + prop.__get__(comp) if prop.apply_at_term(comp, term, self) else 0 + ) def sum_term_fgen(self, ref_group): term_funs = [self.term_fgen(ref.comp, self.get_prop(ref)) for ref in ref_group] @@ -550,6 +688,9 @@ def sum_term_fgen(self, ref_group): # TODO: update internal_references callback to problem def internal_references(self, recache=True, numeric_only=False): """standard component references are""" + + recache = True # override + d = self._gather_references() self._create_term_eval_functions() # Gather all internal economic variables and report costs @@ -561,6 +702,7 @@ def internal_references(self, recache=True, numeric_only=False): if self._cost_references: props.update(**self._cost_references) + # lookup ref from the cost categories dictionary, recreate every time if self._cost_categories: for key, refs in self._cost_categories.items(): props[key] = Ref( @@ -586,6 +728,128 @@ def internal_references(self, recache=True, numeric_only=False): return d + def cost_summary(self, annualized=False, do_print=True, ignore_zero=True): + """ + Generate a summary of costs, optionally annualized, and optionally print the summary. + :param annualized: If True, include only annualized costs in the summary. Default is False. + :type annualized: bool + :param do_print: If True, print the summary to the console. Default is True. + :type do_print: bool + :param ignore_zero: If True, ignore costs with a value of zero. Default is True. + :type ignore_zero: bool + :return: A dictionary containing component costs, skipped costs, and summary. + :rtype: dict + """ + + dct = self.lifecycle_output + cols = list(dct.keys()) + + skippd = set() + comp_costs = collections.defaultdict(dict) + comp_nums = collections.defaultdict(dict) + summary = {} + costs = { + "comps": comp_costs, + "skip": skippd, + "summary": summary, + } + + def abriv(val): + evall = abs(val) + if evall > 1e6: + return f"{val/1E6:>12.4f} M" + elif evall > 1e3: + return f"{val/1E3:>12.2f} k" + return f"{val:>12.2f}" + + for col in cols: + is_ann = ".annualized." in col + val = dct[col] + + # handle no value case + if val == 0 and ignore_zero: + continue + + if ".cost." in col and is_ann == annualized: + base, cst = col.split(".cost.") + if base == "lifecycle": + ckey = f"cost.{cst}" # you're at the top bby + else: + ckey = f"{base.replace('lifecycle.','')}.cost.{cst}" + # print(ckey,str(self._comp_costs.keys())) + comp_costs[base][cst] = val + comp_nums[base][cst] = get_num_from_cost_prop(self._comp_costs[ckey]) + elif col.startswith("summary."): + summary[col.replace("summary.", "")] = val + else: + self.msg(f"skipping: {col}") + + total_cost = sum([sum(list(cc.values())) for cc in comp_costs.values()]) + + # provide consistent format + hdr = "{key:<32}|\t{value:12.10f}" + fmt = "{key:<32}|\t{fmt:<24} | {total:^12} | {pct:3.3f}%" + title = f"COST SUMMARY: {self.parent.identity}" + if do_print: + self.info("#" * 80) + self.info(f"{title:^80}") + self.info("_" * 80) + # possible core values + if NI := getattr(self, "num_items", None): # num items + self.info(hdr.format(key="num_items", value=NI)) + if TI := getattr(self, "term_length", None): + self.info(hdr.format(key="term_length", value=TI)) + if DR := getattr(self, "discount_rate", None): + self.info(hdr.format(key="discount_rate", value=DR)) + if DR := getattr(self, "output", None): + self.info(hdr.format(key="output", value=DR)) + # summary items + for key, val in summary.items(): + self.info(hdr.format(key=key, value=val)) + itcst = "{val:>24}".format(val="TOTAL----->") + self.info("=" * 80) + self.info( + fmt.format(key="COMBINED", fmt=itcst, total=abriv(total_cost), pct=100) + ) + self.info("-" * 80) + # itemization + sgroups = lambda kv: sum(list(kv[-1].values())) + for base, items in sorted(comp_costs.items(), key=sgroups, reverse=True): + # skip if all zeros (allow for net negative costs) + if (subtot := sum([abs(v) for v in items.values()])) > 0: + # self.info(f' {base:<35}| total ---> {abriv(subtot)} | {subtot*100/total_cost:3.0f}%') + pct = subtot * 100 / total_cost + itcst = "{val:>24}".format(val="TOTAL----->") + # todo; add number of items for cost comp + adj_base = base.replace("lifecycle.", "") + self.info( + fmt.format( + key=adj_base, + fmt=itcst, + total=abriv(subtot), + pct=pct, + ) + ) + # Sort costs by value + for key, val in sorted( + items.items(), key=lambda kv: kv[-1], reverse=True + ): + if val == 0 or numpy.isnan(val): + continue # skip zero costs + # self.info(f' \t{key:<32}|{abriv(val)} | {val*100/total_cost:^3.0f}%') + tot = abriv(val) + pct = val * 100 / total_cost + num = comp_nums[base][key] + itcst = ( + f"{abriv(val/num):^18} x {num:3.0f}" if num != 0 else "0" + ) + self.info( + fmt.format(key="-" + key, fmt=itcst, total=tot, pct=pct) + ) + self.info("-" * 80) # section break + self.info("#" * 80) + return costs + @property def lifecycle_output(self) -> dict: """return lifecycle calculations for lcoe""" @@ -599,7 +863,7 @@ def lifecycle_output(self) -> dict: for c in lc.columns: if "category" not in c and "cost" not in c: continue - tot = lc[c].sum() + tot = lc[c].sum() # lifecycle cost if "category" in c: c_ = c.replace("category.", "") lifecat[c_] = tot @@ -609,10 +873,10 @@ def lifecycle_output(self) -> dict: summary["total_cost"] = lc.term_cost.sum() summary["years"] = lc.year.max() + 1 - LC = lc.levalized_cost.sum() - LO = lc.levalized_output.sum() - summary["levalized_cost"] = LC / LO if LO != 0 else numpy.nan - summary["levalized_output"] = LO / LC if LC != 0 else numpy.nan + LC = lc.levelized_cost.sum() + LO = lc.levelized_output.sum() + summary["levelized_cost"] = LC / LO if LO != 0 else numpy.nan + summary["levelized_output"] = LO / LC if LC != 0 else numpy.nan out2 = dict(gend(out)) self._term_output = out2 @@ -641,9 +905,9 @@ def lifecycle_dataframe(self) -> pandas.DataFrame: row["term_cost"] = tc = numpy.nansum( [v(t) for v in self._term_comp_cost.values()] ) - row["levalized_cost"] = tc * (1 + self.discount_rate) ** (-1 * t) + row["levelized_cost"] = tc * (1 + self.discount_rate) ** (-1 * t) row["output"] = output = self.calculate_production(self.parent, t) - row["levalized_output"] = output * (1 + self.discount_rate) ** (-1 * t) + row["levelized_output"] = output * (1 + self.discount_rate) ** (-1 * t) return pandas.DataFrame(out) @@ -675,6 +939,7 @@ def _gather_cost_references(self, parent: "System"): comp_set = set() # reset data + # groupings of components, categories, and pairs of components and categories self._cost_categories = collections.defaultdict(list) self._comp_categories = collections.defaultdict(list) self._comp_costs = dict() @@ -700,7 +965,7 @@ def _gather_cost_references(self, parent: "System"): self.debug(f"checking {key} {comp_key} {kbase}") - # Get Costs Directly From the cost model instance + # 0. Get Costs Directly From the cost model instance if isinstance(conf, CostModel): comps[key] = conf self.debug(f"adding cost model for {kbase}.{comp_key}") @@ -799,6 +1064,7 @@ def _extract_cost_references(self, conf: "CostModel", bse: str): lvl=5, ) # add slot costs with current items (skip class defaults) + # TODO: remove defaults costs for slot_name, slot_value in conf._slot_costs.items(): # Skip items that are internal components if slot_name in comps_act: @@ -826,7 +1092,7 @@ def _extract_cost_references(self, conf: "CostModel", bse: str): elif _key in CST: self.debug(f"skipping key {_key}") - # add base class slot values when comp was none + # add base class slot values when comp was none (recursively) for compnm, comp in conf.internal_configurations(False, none_ok=True).items(): if comp is None: if self.log_level < 5: @@ -834,7 +1100,7 @@ def _extract_cost_references(self, conf: "CostModel", bse: str): f"{conf} looking up base class costs for {compnm}", lvl=5, ) - comp_cls = conf.slots_attributes()[compnm].type.accepted + comp_cls = conf.slots_attributes(attr_type=True)[compnm].accepted for cc in comp_cls: if issubclass(cc, CostModel): if cc._slot_costs: @@ -894,6 +1160,271 @@ def output(self) -> float: return 0 return self._calc_output + @property + def cost_category_store(self): + D = collections.defaultdict(list) + Acat = set() + for catkey, cdict in self._cost_categories.items(): + for ci in cdict: + cprop = getattr(ci.comp.__class__, ci.key) + ccat = set(cprop.cost_categories.copy()) + Acat = Acat.union(ccat) + D[f"{ci.comp.classname}|{ci.key:>36}"] = ccat + return D, Acat + + def create_cost_graph(self, plot=True): + """creates a graph of the cost model using network X and display it""" + import collections + import networkx as nx + + D = collections.defaultdict(dict) + for catkey, cdict in self._cost_categories.items(): + for ci in cdict: + D[catkey][(ci.comp.classname, ci.key)] = ci + + G = nx.Graph() + + for d, dk in D.items(): + print(d.upper()) + cat = d.replace("category.", "") + G.add_node(cat, category=cat) + for kk, r in dk.items(): + cmp = kk[0] + edge = kk[1] + if cmp not in G.nodes: + G.add_node(cmp, component=cmp) + G.add_edge(cmp, cat, cost=edge) + # print(kk) + + # pos = nx.nx_agraph.graphviz_layout(G) + # nx.draw(G, pos=pos) + # nx.draw(G,with_labels=True) + + if plot: + categories = nx.get_node_attributes(G, "category").keys() + components = nx.get_node_attributes(G, "component").keys() + + cm = [] + for nd in G: + if nd in categories: + cm.append("cyan") + else: + cm.append("pink") + + pos = nx.spring_layout(G, k=0.2, iterations=20, scale=1) + nx.draw( + G, + node_color=cm, + with_labels=True, + pos=pos, + arrows=True, + font_size=10, + font_color="0.09", + font_weight="bold", + node_size=200, + ) + + return G + + def cost_matrix(self): + D, Cats = self.cost_category_store + X = list(sorted(Cats)) + C = list(sorted(D.keys())) + M = [] + for k in C: + cats = D[k] + M.append([True if x in cats else numpy.nan for x in X]) + + Mx = numpy.array(M) + X = numpy.array(X) + C = numpy.array(C) + return Mx, X, C + + def create_cost_category_table(self): + """creates a table of costs and categories""" + Mx, X, C = self.cost_matrix() + + fig, ax = subplots(figsize=(12, 12)) + + Mc = numpy.nansum(Mx, axis=0) + x = numpy.argsort(Mc) + + Xs = X[x] + ax.imshow(Mx[:, x]) + ax.set_yticklabels(C, fontdict={"family": "monospace", "size": 8}) + ax.set_yticks(numpy.arange(len(C))) + ax.set_xticklabels(Xs, fontdict={"family": "monospace", "size": 8}) + ax.set_xticks(numpy.arange(len(Xs))) + ax.grid(which="major", linestyle=":", color="k", zorder=0) + xticks(rotation=90) + fig.tight_layout() + + def determine_exclusive_cost_categories( + self, + include_categories=None, + ignore_categories: set = None, + min_groups: int = 2, + max_group_size=None, + min_score=0.95, + include_item_cost=False, + ): + """looks at all possible combinations of cost categories, scoring them based on coverage of costs, and not allowing any double accounting of costs. This is an NP-complete problem and will take a long time for large numbers of items. You can add ignore_categories to ignore certain categories""" + import itertools + + Mx, X, C = self.cost_matrix() + + bad = [] + solutions = [] + inx = {k: i for i, k in enumerate(X)} + + assert include_categories is None or set(X).issuperset( + include_categories + ), "include_categories must be subset of cost categories" + + # ignore categories + if ignore_categories: + X = [x for x in X if x not in ignore_categories] + + if include_categories: + # dont include them in pair since they are added to the group explicitly + X = [x for x in X if x not in include_categories] + + if not include_item_cost: + C = [c for c in C if "item_cost" not in c] + + Num_Costs = len(C) + goal_score = Num_Costs * min_score + NumCats = len(X) // 2 + GroupSize = NumCats if max_group_size is None else max_group_size + for ni in range(min_groups, GroupSize): + print(f"level {ni}/{GroupSize}| {len(solutions)} answers") + for cgs in itertools.combinations(X, ni): + val = None + + # make the set with included if needed + scg = set(cgs) + if include_categories: + scg = scg.union(include_categories) + + # skip bad groups + if any([b.issubset(scg) for b in bad]): + # print(f'skipping {cgs}') + # sys.stdout.write('.') + continue + + good = True # innocent till guilty + for cg in cgs: + xi = Mx[:, inx[cg]].copy() + xi[np.isnan(xi)] = 0 + if val is None: + val = xi + else: + val = val + xi + # determine if any overlap (only pair level) + if np.nanmax(val) > 1: + print(f"bad {cgs}") + bad.append(scg) + good = False + break + + score = np.nansum(val) + if good and score > goal_score: + print(f"found good: {scg}") + solutions.append({"grp": scg, "score": score, "gsize": ni}) + + return solutions + + def cost_categories_from_df(self, df): + categories = set() + for val in df.columns: + m = re.match( + re.compile("economics\.lifecycle\.category\.(?s:[a-z]*)$"), val + ) + if m: + categories.add(val) + return categories + + def plot_cost_categories(self, df, group, cmap="tab20c", make_title=None, ax=None): + categories = self.cost_categories_from_df(df) + from matplotlib import cm + + # if grps: + # assert len(grps) == len(y_vars), 'groups and y_vars must be same length' + # assert all([g in categories for g in grps]), 'all groups must be in categories' + # TODO: project costs onto y_vars + # TODO: ensure groups and y_vars are same length + + color = cm.get_cmap(cmap) + styles = { + c.replace("economics.lifecycle.category.", ""): { + "color": color(i / len(categories)) + } + for i, c in enumerate(categories) + } + + if make_title is None: + + def make_title(row): + return f'{row["name"]}x{row["num_items"]} @{"floating" if row["ldepth"]>50 else "fixed"}' + + # for j,grp in enumerate(groups): + figgen = False + if ax is None: + figgen = True + fig, ax = subplots(figsize=(12, 8)) + else: + fig = ax.get_figure() + + titles = [] + xticks = [] + data = {} + i = 0 + + for inx, row in df.iterrows(): + i += 1 + tc = row["economics.summary.total_cost"] + cat_costs = { + k.replace("economics.lifecycle.category.", ""): row[k] + for k in categories + } + # print(i,cat_costs) + + spec_costs = {k: v for k, v in cat_costs.items() if k in group} + pos_costs = {k: v for k, v in spec_costs.items() if v >= 0} + neg_costs = {k: v for k, v in spec_costs.items() if k not in pos_costs} + neg_amt = sum(list(neg_costs.values())) + pos_amt = sum(list(pos_costs.values())) + + data[i] = spec_costs.copy() + + com = {"x": i, "width": 0.5, "linewidth": 0} + cur = neg_amt + for k, v in neg_costs.items(): + opt = {} if i != 1 else {"label": k} + ax.bar(height=abs(v), bottom=cur, **com, **styles[k], **opt) + cur += abs(v) + for k, v in pos_costs.items(): + opt = {} if i != 1 else {"label": k} + ax.bar(height=abs(v), bottom=cur, **com, **styles[k], **opt) + cur += abs(v) + xticks.append(com["x"]) + titles.append(make_title(row)) + + # Format the chart + ax.legend(loc="upper right") + ax.set_xlim([0, i + max(2, 0.2 * i)]) + ax.set_xticks(xticks) + + ax.set_xticklabels(titles, rotation=90) + ylim = ax.get_ylim() + ylim = ylim[0] - 0.05 * abs(ylim[0]), ylim[1] + 0.05 * abs(ylim[1]) + ax.set_yticks(numpy.linspace(*ylim, 50), minor=True) + ax.grid(which="major", linestyle="--", color="k", zorder=0) + ax.grid(which="minor", linestyle=":", color="k", zorder=0) + if figgen: + fig.tight_layout() + return {"fig": fig, "ax": ax, "data": data} + # TODO: add costs for iterable components (wide/narrow modes) # if isinstance(conf,ComponentIter): diff --git a/engforge/eng/geometry.py b/engforge/eng/geometry.py index 75c33a4..f375ede 100644 --- a/engforge/eng/geometry.py +++ b/engforge/eng/geometry.py @@ -36,14 +36,16 @@ # generic cross sections from # https://mechanicalbase.com/area-moment-of-inertia-calculator-of-certain-cross-sectional-shapes/ -temp_path = os.path.join(tempfile.gettempdir(), "shapely_sections") +_temp = os.path.join(os.path.expanduser("~"), ".forge_tmp") +temp_path = os.path.join(_temp, "shapely_sections") + section_cache = EnvVariable( - "FORGE_SECTION_CACHE", + "FORGE_CACHE", default=temp_path, desc="directory to cache section properties", ) -if "FORGE_SECTION_CACHE" not in os.environ and not os.path.exists(temp_path): - os.mkdir(temp_path) +if "FORGE_CACHE" not in os.environ and not os.path.exists(temp_path): + os.makedirs(temp_path, 0o741, exist_ok=True) section_cache.info(f"loading section from {section_cache.secret}") @@ -620,7 +622,7 @@ def mesh_section(self): self.calculate_bounds() def calculate_bounds(self): - self.info(f"calculating shape bounds!") + self.debug(f"calculating shape bounds!") xcg, ycg = self._geo.calculate_centroid() minx, maxx, miny, maxy = self._geo.calculate_extents() self.y_bounds = (miny - ycg, maxy - ycg) diff --git a/engforge/eng/solid_materials.py b/engforge/eng/solid_materials.py index 509c5e1..62f9f8d 100644 --- a/engforge/eng/solid_materials.py +++ b/engforge/eng/solid_materials.py @@ -41,7 +41,7 @@ class SolidMaterial(SectionMaterial, PyNiteMat.Material, Configuration): __metaclass__ = SecMat name: str = attr.ib(default="solid material") - color: float = attr.ib(factory=random_color, hash=False, eq=ih) + color: tuple = attr.ib(factory=random_color, hash=False, eq=ih) # Structural Properties density: float = attr.ib(default=1.0) diff --git a/engforge/eng/structure.py b/engforge/eng/structure.py index 440bc3b..62dc6d2 100644 --- a/engforge/eng/structure.py +++ b/engforge/eng/structure.py @@ -835,6 +835,7 @@ def add_member_with( return beam # OUTPUT + def beam_dataframe(self, univ_parms: list = None, add_columns: list = None): """creates a dataframe entry for each beam and combo :param univ_parms: keys represent the dataframe parm, and the values represent the lookup value @@ -923,8 +924,18 @@ def structure_cost_panels(self) -> float: """sum of all panels cost""" return sum([sum(self.costs["quads"].values())]) - @cost_property(category="mfg,material,structure") - def structure_cost(self): + @cost_property(category="mfg,material,panels") + def panels_cost(self): + return self.structure_cost_panels + + @cost_property(category="mfg,material,beams") + def beam_cost(self) -> float: + """sum of all beams cost""" + return sum([bm.cost for bm in self.beams.values()]) + + @system_property + def structure_cost(self) -> float: + """sum of all beams and quad cost""" return self.structure_cost_beams + self.structure_cost_panels @solver_cached diff --git a/engforge/eng/structure_beams.py b/engforge/eng/structure_beams.py index a9db704..a6a7056 100644 --- a/engforge/eng/structure_beams.py +++ b/engforge/eng/structure_beams.py @@ -19,7 +19,7 @@ from engforge.system import System from engforge.eng.solid_materials import * from engforge.common import * -import engforge.eng.geometry as ottgeo +import engforge.eng.geometry as enggeo from engforge.eng.costs import CostModel, cost_property import sectionproperties @@ -55,7 +55,7 @@ def rotation_matrix_from_vectors(vec1, vec2): @forge -class Beam(Component, CostModel): +class Beam(Component): """Beam is a wrapper for emergent useful properties of the structure""" # parent structure, will be in its _beams @@ -69,7 +69,7 @@ class Beam(Component, CostModel): validator=attr.validators.instance_of( ( geometry.Geometry, - ottgeo.Profile2D, + enggeo.Profile2D, type(None), ) ) @@ -109,7 +109,7 @@ class Beam(Component, CostModel): def __on_init__(self): self.debug("initalizing...") # update material - if isinstance(self.section, ottgeo.ShapelySection): + if isinstance(self.section, enggeo.ShapelySection): if self.section.material is None: raise Exception("No Section Material") else: @@ -126,11 +126,11 @@ def update_section(self, section): material = self.material else: material = self.section.material - self.section = ottgeo.ShapelySection( + self.section = enggeo.ShapelySection( shape=self.section, material=self.material ) - if isinstance(self.section, ottgeo.ShapelySection): + if isinstance(self.section, enggeo.ShapelySection): self.debug(f"determining profile {section} properties...") self.in_Ix = self.section.Ixx self.in_Iy = self.section.Iyy @@ -138,7 +138,7 @@ def update_section(self, section): self.in_A = self.section.A self.in_Ixy = self.section.Ixy - elif isinstance(self.section, ottgeo.Profile2D): + elif isinstance(self.section, enggeo.Profile2D): self.warning(f"use shapely section instead {section} properties...") # raise Exception("use shapely section instead") self.in_Ix = self.section.Ixx @@ -220,7 +220,7 @@ def A(self) -> float: @property def Ao(self): """outside area, over ride for hallow sections""" - if isinstance(self.section, ottgeo.Profile2D): + if isinstance(self.section, enggeo.Profile2D): return self.section.Ao return self.A @@ -302,7 +302,8 @@ def section_mass(self) -> float: def mass(self) -> float: return self.material.density * self.Vol - @cost_property(category="mfg,material,beams") + # @system_property(category="mfg,material,beams") + @system_property def cost(self) -> float: return self.mass * self.material.cost_per_kg @@ -389,7 +390,7 @@ def show_mesh(self): def estimate_stress(self, force_calc=True, **forces): """uses the best available method to determine max stress in the beam, for ShapelySections this is done through a learning process, for other sections it is done through a simple calculation aimed at providing a conservative estimate""" - if isinstance(self.section, ottgeo.ShapelySection): + if isinstance(self.section, enggeo.ShapelySection): return self.section.estimate_stress(**forces, force_calc=force_calc) else: return self._fallback_estimate_stress(**forces) diff --git a/engforge/engforge_attributes.py b/engforge/engforge_attributes.py index f743879..9672cde 100644 --- a/engforge/engforge_attributes.py +++ b/engforge/engforge_attributes.py @@ -71,7 +71,7 @@ def collect_inst_attributes(self, **kw): return out @classmethod - def _get_init_attrs_data(cls, subclass_of: type, exclude=False): + def _get_init_attrs_data(cls, subclass_of: type, exclude=False, attr_type=False): choose = issubclass if exclude: choose = lambda ty, type_set: not issubclass(ty, type_set) @@ -80,7 +80,7 @@ def _get_init_attrs_data(cls, subclass_of: type, exclude=False): if "__attrs_attrs__" in cls.__dict__: # Handle Attrs Class for k, v in attrs.fields_dict(cls).items(): if isinstance(v.type, type) and choose(v.type, subclass_of): - attrval[k] = v + attrval[k] = v.type if attr_type else v # else: # Handle Pre-Attrs Class # FIXME: should this run first? @@ -167,9 +167,9 @@ def slot_refs(cls, recache=False): return o @classmethod - def slots_attributes(cls) -> typing.Dict[str, "Attribute"]: + def slots_attributes(cls, attr_type=False) -> typing.Dict[str, "Attribute"]: """Lists all slots attributes for class""" - return cls._get_init_attrs_data(Slot) + return cls._get_init_attrs_data(Slot, attr_type=attr_type) @classmethod def signals_attributes(cls) -> typing.Dict[str, "Attribute"]: @@ -198,6 +198,7 @@ def plot_attributes(cls) -> typing.Dict[str, "Attribute"]: @classmethod def input_attrs(cls): + """Lists all input attributes for class""" return attr.fields_dict(cls) @classmethod @@ -215,6 +216,7 @@ def input_fields(cls, add_ign_types: list = None): @classmethod def numeric_fields(cls): + """no tuples,lists, dicts, strings, or attr base types""" ignore_types = ( ATTR_BASE, str, @@ -227,7 +229,9 @@ def numeric_fields(cls): @classmethod def table_fields(cls): - keeps = (str, float, int) # TODO: add numpy fields + """the table attributes corresponding to""" + # TODO: add list/numpy fields with vector stats + keeps = (str, float, int) typ = cls._get_init_attrs_data(keeps) return {k: v for k, v in typ.items()} @@ -246,6 +250,7 @@ def as_dict(self): o = {k: getattr(self, k, None) for k, v in inputs.items()} return o + # TODO: refactor this, allowing a nesting return option for sub components, by default True (later to be reverted to False, as a breaking change). this messes up hashing and we can just the other object hash @property def input_as_dict(self): """returns values as they are in the class instance, but converts classes inputs to their input_as_dict""" @@ -258,6 +263,14 @@ def input_as_dict(self): } return o + @property + def table_row_dict(self): + """returns values as they would be put in a table row from this instance ignoring any sub components""" + from engforge.configuration import Configuration + + o = {k: getattr(self, k, None) for k in self.table_fields()} + return o + @property def numeric_as_dict(self): """recursively gets internal components numeric_as_dict as well as its own numeric values""" @@ -272,7 +285,24 @@ def numeric_as_dict(self): # Hashes # TODO: issue with logging sub-items + def hash(self, *args, **input_kw): + """hash by parm or by input_kw, only input can be hashed by lookup as system properties can create a recursive loop and should be deterministic from input""" + d = {k: v for k, v in self.input_as_dict.items() if k in args} + d.update(input_kw) # override with input_kw + return d, deepdiff.DeepHash( + d, ignore_encoding_errors=True, significant_digits=6 + ) + def hash_with(self, **input_kw): + """ + Generates a hash for the object's dictionary representation, updated with additional keyword arguments. + Args: + **input_kw: Arbitrary keyword arguments to update the object's dictionary representation. + Returns: + The hash value of the updated dictionary. + Raises: + Any exceptions raised by deepdiff.DeepHash if hashing fails. + """ d = self.as_dict d.update(input_kw) return deepdiff.DeepHash(d, ignore_encoding_errors=True)[d] @@ -292,6 +322,11 @@ def input_hash(self): d = self.input_as_dict return deepdiff.DeepHash(d, ignore_encoding_errors=True)[d] + @property + def table_hash(self): + d, hsh = self.hash(**self.table_row_dict) + return hsh[d] + @property def numeric_hash(self): d = self.numeric_as_dict diff --git a/engforge/logging.py b/engforge/logging.py index ffbddc8..2437e0c 100644 --- a/engforge/logging.py +++ b/engforge/logging.py @@ -18,7 +18,7 @@ LOG_LEVEL = logging.INFO -def change_all_log_levels(new_log_level: int = 20, inst=None, check_function=None): +def change_all_log_levels(new_log_level: int = 20, inst=None, check_function=None): """Changes All Log Levels With pyee broadcast before reactor is running :param new_log_level: int - changes unit level log level (10-msg,20-debug,30-info,40-warning,50-error,60-crit) :param check_function: callable -> bool - (optional) if provided if check_function(unit) is true then the new_log_level is applied @@ -55,7 +55,7 @@ class LoggingMixin(logging.Filter): slack_webhook_url = None # log_silo = False - change_all_log_lvl = lambda s, *a, **kw: change_all_log_levels(*a, inst=s,**kw) + change_all_log_lvl = lambda s, *a, **kw: change_all_log_levels(*a, inst=s, **kw) @property def logger(self): diff --git a/engforge/problem_context.py b/engforge/problem_context.py index 93b2d2f..a26de9d 100644 --- a/engforge/problem_context.py +++ b/engforge/problem_context.py @@ -126,7 +126,7 @@ class ProbLog(LoggingMixin): save_mode="all", x_start=None, save_on_exit=False, - enter_refresh=False, + enter_refresh=True, ) # can be found on session._ or session. root_defined = dict( @@ -210,6 +210,8 @@ class ProblemExec: """ + full_update = True # TODO: cant justify setting this to false for performance gains. accuracy comes first. see event based update + # TODO: convert this to a system based cache where there is a unique problem for each system instance. On subprobem copy a system and add o dictionary. class_cache = None # ProblemExec is assigned below @@ -244,7 +246,9 @@ class ProblemExec: _converged: bool # Interior Context Options - enter_refresh: bool = False + enter_refresh: bool = ( + True # TODO: allow this off (or lower impact) with event system + ) save_on_exit: bool = False save_mode: str = "all" level_name: str = None # target this context with the level name @@ -269,7 +273,9 @@ class ProblemExec: ) def __getattr__(self, name): - """This is a special method that is called when an attribute is not found in the usual places, like when interior contexts (anything not the root (session_id=True)) are created that dont have the top level's attributes. some attributes will look to the parent session""" + """ + This is a special method that is called when an attribute is not found in the usual places, like when interior contexts (anything not the root (session_id=True)) are created that dont have the top level's attributes. some attributes will look to the parent session + """ # interior context lookup (when in active context, ie session exists) if hasattr(self.class_cache, "session") and name in root_possible: @@ -333,8 +339,8 @@ def __init__(self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts): if opts.pop("persist", False) or kw_dict.pop("persist", False): self.persist_contexts() - # temp solver storage - self.solver_hist = expiringdict.ExpiringDict(100, 60) + # temp solver storage #TODO + # self.solver_hist = expiringdict.ExpiringDict(100, 60) if self.log_level < 5: if hasattr(self.class_cache, "session"): @@ -425,6 +431,8 @@ def __init__(self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts): self.class_cache.session._prob_levels[self.level_name] = self # error if the system is different (it shouldn't be!) if self.system is not system: + # TODO: subproblems allow different systems, but the top level should be the same + # idea - use (system,pid) as key for problems_dict, (system,True) would be root problem. This breaks checking for `class_cache.session` though one could gather that from the root problem key` raise IllegalArgument( f"somethings wrong! change of comp! {self.system} -> {system}" ) @@ -589,19 +597,14 @@ def get_sesh(self, sesh=None): # Update Methods def refresh_references(self, sesh=None): """refresh the system references""" - sesh = self.sesh + + if sesh is None: + sesh = self.sesh if self.log_level < 5: self.warning(f"refreshing system references") - check_dynamics = sesh.check_dynamics - sesh._num_refs = sesh.system.system_references(numeric_only=True) - sesh._sys_refs = sesh.system.solver_vars( - check_dynamics=check_dynamics, - addable=sesh._num_refs, - **sesh._slv_kw, - ) - sesh.update_methods(sesh=sesh) + sesh.full_refresh(sesh=sesh) sesh.min_refresh(sesh=sesh) def update_methods(self, sesh=None): @@ -621,16 +624,46 @@ def update_dynamics(self, sesh=None): self.info(f"update dynamics") self.system.setup_global_dynamics() + def full_refresh(self, sesh=None): + """a more time consuming but throughout refresh of the system""" + if self.log_level < 5: + self.info(f"full refresh") + + check_dynamics = sesh.check_dynamics + sesh._num_refs = sesh.system.system_references( + numeric_only=True, + none_ok=True, + only_inst=False, + ignore_none_comp=False, + recache=True, + ) + sesh._sys_refs = sesh.system.solver_vars( + check_dynamics=check_dynamics, + addable=sesh._num_refs, + **sesh._slv_kw, + ) + sesh.update_methods(sesh=sesh) + def min_refresh(self, sesh=None): + """what things need to be refreshed per execution, this is important whenever items are replaced""" + # TODO: replace this function with an event based responsiblity model. sesh = sesh if sesh is not None else self.sesh if self.log_level < 5: self.info(f"min refresh") + if sesh.full_update: + # TODO: dont require this + sesh.full_refresh(sesh=sesh) + # final ref's after update # after updates sesh._all_refs = sesh.system.system_references( - recache=True, check_config=False, ignore_none_comp=False + recache=True, + check_config=False, + ignore_none_comp=False, + none_ok=True, + only_inst=False, ) # sesh._attr_sys_key_map = sesh.attribute_sys_key_map @@ -641,6 +674,41 @@ def min_refresh(self, sesh=None): cons = {} # TODO: parse additional constraints sesh.constraints = sesh.sys_solver_constraints(cons) + def print_all_info(self, keys: str = None, comps: str = None): + """ + Print all the information of each component's dictionary. + Parameters: + key_sch (str, optional): A pattern to match dictionary keys. Only keys matching this pattern will be included in the output. + comps (list, optional): A list of component sys names to filter. Only information of these components will be printed. + Returns: None (except stdout :) + """ + from pprint import pprint + + keys = keys.split(",") + comps = (comps + ",").split(",") # always top level + print(f"CONTEXT: {self}") + + mtch = lambda key, ptrns: any( + [fnmatch.fnmatch(key.lower(), ptn.lower()) for ptn in ptrns] + ) + + # check your comps + itrs = self.all_comps.copy() + itrs[""] = Ref(self.system, "", True, False) + + # check your comps + for cn, comp in itrs.items(): + if comps is not None and not mtch(cn, comps): + continue + + dct = comp.value().as_dict + if keys: # filter keys + dct = {k: v for k, v in dct.items() if mtch(k, keys)} + if dct: + print(f'INFO: {cn if cn else ""}') + pprint(dct) + print("-" * 80) + @property def check_dynamics(self): sesh = self.sesh @@ -672,7 +740,6 @@ def __enter__(self): # transients wont update components/ methods dynamically (or shouldn't) so we can just update the system references once and be done with it for other cases, but that is not necessary unless a component changes or a component has in general a unique reference update system (economics / component-iterators) sesh = self.sesh if not sesh._dxdt is True and self.enter_refresh: - sesh.update_methods(sesh=sesh) sesh.min_refresh(sesh=sesh) elif sesh.dynamics_updated: @@ -802,6 +869,7 @@ def debug_levels(self): raise IllegalArgument(f"no session available") # Multi Context Exiting: + # TODO: rethink this def persist_contexts(self): """convert all contexts to a new storage format""" self.info(f"persisting contexts!") @@ -1066,7 +1134,7 @@ def integral_rate(self, t, x, dt, Xss=None, Yobj=None, **kw): self.info( f'exiting solver {t} {ss_out["Xans"]} {ss_out["Xstart"]}' ) - pbx.set_ref_values(ss_out["Xans"]) + pbx.set_ref_values(ss_out["Xans"], scope="intgrl") pbx.exit_to_level("ss_slvr", False) else: self.warning( @@ -1171,7 +1239,7 @@ def handle_solution(self, answer, Xref, Yref, output): # Output Results Xa = {p: answer.x[i] for i, p in enumerate(vars)} output["Xans"] = Xa - Ref.refset_input(Xref, Xa) + Ref.refset_input(Xref, Xa, scope="solvd") Yout = {p: yit.value(yit.comp, self) for p, yit in Yref.items()} output["Yobj"] = Yout @@ -1660,13 +1728,13 @@ def get_ref_values(self, refs=None): refs = sesh.all_system_references return Ref.refset_get(refs, sys=self.system, prob=self) - def set_ref_values(self, values, refs=None): + def set_ref_values(self, values, refs=None, scope="sref"): """returns the values of the refs""" # TODO: add checks for the refs if refs is None: sesh = self.sesh refs = sesh.all_comps_and_vars - return Ref.refset_input(refs, values) + return Ref.refset_input(refs, values, scope=scope) def change_sys_var(self, key, value, refs=None, doset=True, attr_key_map=None): """use this function to change the value of a system var and update the start state, multiple uses in the same context will not change the record preserving the start value @@ -1711,7 +1779,9 @@ def revert_to_start(self): rs = list(self.record_state.values()) self.debug(f"reverting to start: {xs} -> {rs}") # TODO: STRICT MODE Fail for refset_input - Ref.refset_input(sesh.all_comps_and_vars, self.x_start, fail=False) + Ref.refset_input( + sesh.all_comps_and_vars, self.x_start, fail=False, scope="rvtst" + ) def activate_temp_state(self, new_state=None): # TODO: determine when components change, and update refs accordingly! @@ -1720,11 +1790,18 @@ def activate_temp_state(self, new_state=None): if new_state: if self.log_level < 3: self.debug(f"new-state: {self.temp_state}") - Ref.refset_input(sesh.all_comps_and_vars, new_state, fail=False) + Ref.refset_input( + sesh.all_comps_and_vars, new_state, fail=False, scope="ntemp" + ) elif self.temp_state: if self.log_level < 3: self.debug(f"act-state: {self.temp_state}") - Ref.refset_input(sesh.all_comps_and_vars, self.temp_state, fail=False) + Ref.refset_input( + sesh.all_comps_and_vars, + self.temp_state, + fail=False, + scope="atemp", + ) elif self.log_level < 3: self.debug(f"no-state: {new_state}") @@ -1974,6 +2051,13 @@ def is_active(self): """checks if the context has been entered and not exited""" return self.entered and not self.exited + @classmethod + def cls_is_active(cls): + """checks if the cache has a session""" + if cls.class_cache and hasattr(cls.class_cache, "session"): + return True + return False + @property def solveable(self): """checks the system's references to determine if its solveabl""" diff --git a/engforge/properties.py b/engforge/properties.py index ed4dcd4..29b4282 100644 --- a/engforge/properties.py +++ b/engforge/properties.py @@ -21,6 +21,8 @@ class PropertyLog(LoggingMixin): log = PropertyLog() +_basic_valild_types = (int, str, float) # check for return_type + class engforge_prop: """ @@ -28,15 +30,16 @@ class engforge_prop: Use as follows: @engforge_prop - def our_custom_function(self): + def our_custom_function(self) -> return_type: pass """ + valid_types = _basic_valild_types must_return = False def __init__(self, fget=None, fset=None, fdel=None, *args, **kwargs): """call with the function you want to wrap in a decorator""" - + self.valild_types = (int, str, float) # check for return_type self.fget = fget if fget: self.gname = fget.__name__ @@ -63,7 +66,7 @@ def get_func_return(self, func): """ensures that the function has a return annotation, and that return annotation is in valid sort types""" anno = func.__annotations__ typ = anno.get("return", None) - if not typ in (int, str, float) and self.must_return: + if not typ in self.valid_types and self.must_return: raise Exception( f"system_property input: function {func.__name__} must have valid return annotation of type: {(int,str,float)}" ) @@ -102,6 +105,7 @@ class cache_prop(engforge_prop): def __init__(self, *args, **kwargs): self.allow_set = True + self.valild_types = (int, str, float) # check for return_type super().__init__(*args, **kwargs) def __set__(self, instance, value): @@ -155,7 +159,7 @@ def function(...): < this uses __init__ to assign function @system_property(desc='really nice',label='funky function') def function(...): < this uses __call__ to assign function """ - + self.valild_types = (int, str, float) # check for return_type self.fget = fget if fget: self.get_func_return(fget) diff --git a/engforge/solveable.py b/engforge/solveable.py index 9deb692..486cd13 100644 --- a/engforge/solveable.py +++ b/engforge/solveable.py @@ -36,12 +36,12 @@ class SolvableLog(LoggingMixin): def _update_func(comp, eval_kw): def updt(*args, **kw): eval_kw.update(kw) - if log.log_level <= 5: - log.msg(f"update| {comp.name} ", lvl=5) + if log.log_level <= 12: + log.info(f"update| {comp.name} ") # =5) return comp.update(comp.parent, *args, **eval_kw) - if log.log_level <= 5: - log.msg(f"create method| {comp.name}| {eval_kw}") + if log.log_level <= 12: + log.info(f"create method| {comp.name}| {eval_kw}") updt.__name__ = f"{comp.name}_update" return updt @@ -51,8 +51,8 @@ def updt(*args, **kw): eval_kw.update(kw) return comp.post_update(comp.parent, *args, **eval_kw) - if log.log_level <= 5: - log.msg(f"create post method| {comp.name}| {eval_kw}") + if log.log_level <= 12: + log.info(f"create post method| {comp.name}| {eval_kw}") updt.__name__ = f"{comp.name}_post_update" return updt @@ -63,24 +63,25 @@ def _cost_update(comp): if isinstance(comp, Economics): def updt(*args, **kw): - if log.log_level <= 8: - log.debug(f"update economics {comp.name} | {comp.term_length} ") + if log.log_level <= 12: + log.info(f"update economics {comp.name} | {comp.term_length} ") comp.system_properties_classdef(True) comp.update(comp.parent, *args, **kw) - if log.log_level <= 8: - log.debug(f"economics update cb {comp.name} | {comp.term_length} ") + if log.log_level <= 12: + log.info(f"economics update cb {comp.name} | {comp.term_length} ") updt.__name__ = f"{comp.name}_econ_update" else: def updt(*args, **kw): - if log.log_level <= 7: - log.msg(f"update costs {comp.name} ", lvl=5) + if log.log_level <= 12: + log.info(f"update costs {comp.name} ") # =5) comp.system_properties_classdef(True) + # comp.update(comp.parent, *args, **kw) #called as update without cm return comp.update_dflt_costs() - if log.log_level <= 7: - log.debug(f"cost update cb {comp.name} ") + if log.log_level <= 12: + log.info(f"cost update cb {comp.name} ") updt.__name__ = f"{comp.name}_cost_update" return updt @@ -123,13 +124,13 @@ class SolveableMixin(AttributedBaseMixin): #'Configuration' # TODO: pass the problem vs the parent component, then locate this component in the problem and update any references def update(self, parent, *args, **kwargs): """Kwargs comes from eval_kw in solver""" - if log.log_level <= 5: - log.debug(f"void updating {self.__class__.__name__}.{self}") + if log.log_level <= 12: + log.info(f"void updating {self.__class__.__name__}.{self}") def post_update(self, parent, *args, **kwargs): """Kwargs comes from eval_kw in solver""" - if log.log_level <= 5: - log.debug(f"void post-updating {self.__class__.__name__}.{self}") + if log.log_level <= 12: + log.info(f"void post-updating {self.__class__.__name__}.{self}") def collect_update_refs(self, eval_kw=None, ignore=None): """checks all methods and creates ref's to execute them later""" @@ -145,7 +146,17 @@ def collect_update_refs(self, eval_kw=None, ignore=None): elif self in ignore: return + key = "top" + if self.__class__.update != SolveableMixin.update: + ref = Ref(self, _update_func(self, eval_kw if eval_kw else {})) + updt_refs[key] = ref + + if isinstance(self, (CostModel, Economics)): + ref = Ref(self, _cost_update(self)) + updt_refs[key + "._cost_model_"] = ref + for key, comp in self.internal_configurations(False).items(): + # for key,lvl,comp in self.go_through_configurations(check_config=False): if ignore is not None and comp in ignore: continue @@ -164,7 +175,7 @@ def collect_update_refs(self, eval_kw=None, ignore=None): ref = Ref(comp, _cost_update(comp)) updt_refs[key + "._cost_model_"] = ref - elif comp.__class__.update != SolveableMixin.update: + if comp.__class__.update != SolveableMixin.update: ref = Ref(comp, _update_func(comp, ekw)) updt_refs[key] = ref @@ -185,7 +196,12 @@ def collect_post_update_refs(self, eval_kw=None, ignore=None): elif self in ignore: return + if self.__class__.post_update != SolveableMixin.post_update: + ref = Ref(self, _post_update_func(self, eval_kw if eval_kw else {})) + updt_refs["top"] = ref + for key, comp in self.internal_configurations(False).items(): + # for key,lvl,comp in self.go_through_configurations(check_config=False): if ignore is not None and comp in ignore: continue @@ -336,6 +352,7 @@ def comp_references(self, ignore_none_comp=True, **kw): out = {} for key, lvl, comp in self.go_through_configurations(parent_level=1, **kw): if ignore_none_comp and not isinstance(comp, SolveableMixin): + self.warning(f"ignoring {key} {lvl}|{comp}") continue out[key] = comp return out @@ -351,6 +368,7 @@ def _iterate_input_matrix( force_solve=False, return_results=False, method_kw: dict = None, + print_kw: dict = None, **kwargs, ): """applies a permutation of input vars for vars. runs the system instance by applying input to the system and its slot-components, ensuring that the targeted attributes actualy exist. @@ -360,6 +378,8 @@ def _iterate_input_matrix( :param sequence: a list of dictionaries that should be run in order per the outer-product of kwargs :param eval_kw: a dictionary of keyword arguments to pass to the eval function of each component by their name and a set of keyword args. Use this to set values in the component that are not inputs to the system. No iteration occurs upon these values, they are static and irrevertable :param sys_kw: a dictionary of keyword arguments to pass to the eval function of each system by their name and a set of keyword args. Use this to set values in the component that are not inputs to the system. No iteration occurs upon these values, they are static and irrevertable + :param print_kw: a dictionary of keyword arguments to pass to the print_all_info function of the current context + :param kwargs: inputs are run on a product basis asusming they correspond to actual scoped vars (system.var or system.slot.var) @@ -415,6 +435,13 @@ def _iterate_input_matrix( # Run The Method with inputs provisioned out = method(icur, eval_kw, sys_kw, cb=cb, **method_kw) + if ( + print_kw + and hasattr(self, "last_context") + and self.last_context + ): + self.last_context.print_all_info(**print_kw) + if return_results: # store the output output[max(output) + 1 if output else 0] = out @@ -530,7 +557,7 @@ def locate(cls, key, fail=True) -> type: args = key.split(".") comp, sub = args[0], ".".join(args[1:]) assert comp in cls.slots_attributes(), f"invalid {comp} in {key}" - comp_cls = cls.slots_attributes()[comp].type.accepted[0] + comp_cls = cls.slots_attributes(attr_type=True)[comp].accepted[0] val = comp_cls.locate(sub, fail=True) elif key in cls.input_fields(): @@ -606,6 +633,8 @@ def system_references(self, recache=False, numeric_only=False, **kw): ): return self._prv_system_references + # TODO: system references are really important and nesting them together complicates the refresh process. Each component should be able to refresh itself and its children on set_state, as well as alert parents to change. Ideally the `Ref` objects could stay the same and no `recache` would need to occur. This would be a huge performance boost and fix a lot of the issues with the current system. + out = self.internal_references(recache, numeric_only=numeric_only) tatr = out["attributes"] tprp = out["properties"] @@ -614,6 +643,8 @@ def system_references(self, recache=False, numeric_only=False, **kw): # component iternals for key, comp in self.comp_references(**kw).items(): + if comp is None: + continue sout = comp.internal_references(recache, numeric_only=numeric_only) satr = sout["attributes"] sprp = sout["properties"] diff --git a/engforge/solver.py b/engforge/solver.py index c192579..bb64337 100644 --- a/engforge/solver.py +++ b/engforge/solver.py @@ -244,8 +244,6 @@ def solver(self, enter_refresh=True, save_on_exit=True, **kw): """ runs the system solver using the current system state and modifying it. This is the default solver for the system, and it is recommended to add additional options or methods via the execute method. - - :param obj: the objective function to minimize, by default will minimize the sum of the squares of the residuals. Objective function should be a function(system,Xs,Xt) where Xs is the system state and Xt is the system transient state. The objective function will be argmin(X)|(1+custom_objective)*residual_RSS when `add_obj` is True in kw otherwise argmin(X)|custom_objective with constraints on the system as balances instead of first objective being included. :param cons: the constraints to be used in the solver, by default will use the system's constraints will be enabled when True. If a dictionary is passed the solver will use the dictionary as the constraints in addition to system constraints. These can be individually disabled by key=None in the dictionary. @@ -286,7 +284,7 @@ def solver(self, enter_refresh=True, save_on_exit=True, **kw): # depending on the solver success, failure or no solution, we can exit the solver if has_ans and out["ans"] and out["ans"].success: # this is where you want to be! <<< - pbx.set_ref_values(out["Xans"]) + pbx.set_ref_values(out["Xans"], scope="solvr") pbx.exit_to_level("sys_slvr", False) elif has_ans and out["ans"] is None: diff --git a/engforge/system_reference.py b/engforge/system_reference.py index ec17780..196c65a 100644 --- a/engforge/system_reference.py +++ b/engforge/system_reference.py @@ -13,6 +13,7 @@ from contextlib import contextmanager from engforge.properties import * import copy +from difflib import get_close_matches class RefLog(LoggingMixin): @@ -22,32 +23,39 @@ class RefLog(LoggingMixin): log = RefLog() -def refset_input(refs, delta_dict, chk=True, fail=True, warn=True): +def refset_input(refs, delta_dict, chk=True, fail=True, warn=True, scope="ref"): """change a set of refs with a dictionary of values. If chk is True k will be checked for membership in refs""" + keys = set(refs.keys()) for k, v in delta_dict.items(): if isinstance(k, Ref): k.set_value(v) continue memb = k in refs + # if a match or not checking go ahead. if not chk or memb: refs[k].set_value(v) + + # TODO: handle setting non-ref values such as dictionaries + elif fail and chk and not memb: - raise KeyError(f"key {k} not in refs {refs.keys()}") + close = get_close_matches(k, keys) + raise KeyError(f"{scope}| key {k} not in refs. did you mean {close}?") elif warn and chk and not memb: - log.warning(f"key {k} not in refs {refs.keys()}") + close = get_close_matches(k, keys) + log.warning(f"{scope}| key {k} not in refs. did you mean {close}") def refset_get(refs, *args, **kw): out = {} + scope = kw.get("scope", "ref") for k in refs: try: # print(k,refs[k]) - out[k] = refs[k].value(refs[k].comp, **kw) except Exception as e: rf = refs[k] - log.error(e, f"issue with ref: {rf}|{rf.key}|{rf.comp}") + log.error(e, f"{scope}| issue with ref: {rf}|{rf.key}|{rf.comp}") return out @@ -196,6 +204,7 @@ def set(self, comp, key, use_call=True, allow_set=True, eval_f=None): self.hxd = str(hex(id(self)))[-6:] + # TODO: update with change (pyee?) self.setup_calls() def setup_calls(self): @@ -215,7 +224,10 @@ def setup_calls(self): self._value_eval = lambda *a, **kw: self.key(*a, **kw) else: # do not cross reference vars! - if self.use_dict: + # TODO: allo for comp/key changes with events + if self.key == "": # return the component + p = lambda *a, **kw: self.comp + elif self.use_dict: p = lambda *a, **kw: self.comp.get(self.key) elif self.key in self.comp.__dict__: p = lambda *a, **kw: self.comp.__dict__[self.key] diff --git a/engforge/tabulation.py b/engforge/tabulation.py index 5c08f98..9df1dd2 100644 --- a/engforge/tabulation.py +++ b/engforge/tabulation.py @@ -83,10 +83,30 @@ def last_context(self): # @solver_cached #FIXME: not caching correctly @property # FIXME: this is slow def dataframe(self): + """ + Returns a pandas DataFrame based on the current context. + + This method checks for the presence of `last_context` and its `dataframe` attribute. + If they exist, it returns the `dataframe` from `last_context`. + If not, it checks for the `_patch_dataframe` attribute and returns it if it exists. + If neither condition is met, it returns an empty DataFrame. + + :return: A pandas DataFrame based on the current context or an empty DataFrame if no context is available. + :rtype: pandas.DataFrame + """ + """""" if hasattr(self, "last_context") and hasattr(self.last_context, "dataframe"): return self.last_context.dataframe + if hasattr(self, "_patch_dataframe") and self._patch_dataframe is not None: + return self._patch_dataframe return pandas.DataFrame([]) + @dataframe.setter + def dataframe(self, input_dataframe): + if hasattr(self, "last_context") and hasattr(self.last_context, "dataframe"): + raise Exception(f"may not set dataframe on run component") + self._patch_dataframe = input_dataframe + @property def plotable_variables(self): """Checks columns for ones that only contain numeric types or haven't been explicitly skipped""" @@ -212,9 +232,10 @@ def system_properties_classdef(cls, recache=False): """Combine other parent-classes table properties into this one, in the case of subclassed system_properties""" from engforge.tabulation import TabulationMixin + cls_key = f"_{cls.__name__}_system_properties" # Use a cache for deep recursion - if not recache and hasattr(cls, "_{cls.__name__}_system_properties"): - res = getattr(cls, "_{cls.__name__}_system_properties") + if not recache and hasattr(cls, cls_key): + res = getattr(cls, cls_key) if res is not None: return res @@ -246,9 +267,10 @@ def system_properties_classdef(cls, recache=False): prop = getattr(cls, k, None) if prop and isinstance(prop, system_property): __system_properties[k] = prop - log.msg(f"adding system property {mrv.__name__}.{k}") + if log.log_level <= 3: + log.msg(f"adding system property {mrv.__name__}.{k}") - setattr(cls, "_{cls.__name__}_system_properties", __system_properties) + setattr(cls, cls_key, __system_properties) return __system_properties diff --git a/engforge/test/test_structures.py b/engforge/test/_pre_test_structures.py similarity index 100% rename from engforge/test/test_structures.py rename to engforge/test/_pre_test_structures.py diff --git a/engforge/test/test_comp_iter.py b/engforge/test/test_comp_iter.py index 1d95680..737bff8 100644 --- a/engforge/test/test_comp_iter.py +++ b/engforge/test/test_comp_iter.py @@ -102,7 +102,7 @@ def test_keys(self): for p in props: tkn = f"{ck}.{v}.{p}" should_keys.add(tkn) - dataframe_keys.add(tkn.replace(".", "_")) + dataframe_keys.add(tkn) sys_key = set(self.system.data_dict.keys()) mtch = should_keys.issubset(sys_key) @@ -161,7 +161,7 @@ def test_keys(self): for p in props: tkn = f"{ck}.{p}" should_keys.add(tkn) - dataframe_keys.add(tkn.replace(".", "_")) + dataframe_keys.add(tkn) sys_key = set(self.system.data_dict.keys()) mtch = should_keys.issubset(sys_key) @@ -177,18 +177,18 @@ def test_keys(self): self.assertTrue(dataframe_keys.issubset(set(df.keys()))) # test item existence - v1 = set(self.system.dataframe["cdict_current_item"]) + v1 = set(self.system.dataframe["cdict.current_item"]) v2 = set(comps["cdict"]) self.assertEqual(v1, v2) - d1 = set(self.system.dataframe["citer_current_item"]) + d1 = set(self.system.dataframe["citer.current_item"]) d2 = set(comps["citer"]) self.assertEqual(d1, d2) - dvs = self.system.dataframe["cdict_current_item"] - cvs = self.system.dataframe["citer_current_item"] + dvs = self.system.dataframe["cdict.current_item"] + cvs = self.system.dataframe["citer.current_item"] al = set(zip(dvs, cvs)) sh = set(itertools.product(v2, d2)) diff --git a/engforge/test/test_composition.py b/engforge/test/test_composition.py index 9ae926b..4db8de7 100644 --- a/engforge/test/test_composition.py +++ b/engforge/test/test_composition.py @@ -121,8 +121,8 @@ def test_signals(self): def test_input_and_run(self): self.system.run(**{"comp.aux": 5, "comp.comp.aux": 6}) self.assertEqual(len(self.system.dataframe), 1, f"wrong run config") - self.assertEqual(set(self.system.dataframe["comp_aux"]), set([5])) - self.assertEqual(set(self.system.dataframe["comp_comp_aux"]), set([6])) + self.assertEqual(set(self.system.dataframe["comp.aux"]), set([5])) + self.assertEqual(set(self.system.dataframe["comp.comp.aux"]), set([6])) # internal storage # self.assertEqual(set(self.system.comp.dataframe["aux"]), set([5])) diff --git a/engforge/test/test_costs.py b/engforge/test/test_costs.py index 38a4a94..2399c51 100644 --- a/engforge/test/test_costs.py +++ b/engforge/test/test_costs.py @@ -96,7 +96,7 @@ def test_econ_array(self): df = er.dataframe tc = ( - df["econ_summary_total_cost"] == np.array([161.0, 220.0, 305.0, 390.0]) + df["econ.summary.total_cost"] == np.array([161.0, 220.0, 305.0, 390.0]) ).all() self.assertTrue(tc) @@ -123,7 +123,7 @@ def test_econ_defaults(self): d = er.data_dict self.assertEqual(78, d["econ.summary.total_cost"]) self.assertEqual(78, d["econ.lifecycle.annualized.term_cost"]) - self.assertEqual(78, d["econ.lifecycle.annualized.levalized_cost"]) + self.assertEqual(78, d["econ.lifecycle.annualized.levelized_cost"]) self.assertEqual(78, d["econ.lifecycle.term_cost"]) def test_recursive_null(self, ANS=75): @@ -273,14 +273,14 @@ def test_dataframe(self): dfc = df_complete match = ( - dfc["fan_blade_cost"] + dfc["motor_motor_cost"] - == dfc["econ_lifecycle_category_capex"] + dfc["fan.blade_cost"] + dfc["motor.motor_cost"] + == dfc["econ.lifecycle.category.capex"] ).all() self.assertTrue(match) match = ( - df_complete["fan_area"] * df_complete["fan_v"] - == df_complete["fan_volumetric_flow"] + df_complete["fan.area"] * df_complete["fan.v"] + == df_complete["fan.volumetric_flow"] ).all() self.assertTrue(match) diff --git a/engforge/typing.py b/engforge/typing.py index 51fc816..9109207 100644 --- a/engforge/typing.py +++ b/engforge/typing.py @@ -53,7 +53,10 @@ def Options(*choices, **kwargs): :param kwargs: keyword args passed to attrs field""" assert choices, f"must have some choices!" assert "type" not in kwargs, "options type set is str" - assert set([type(c) for c in choices]) == set((str,)), "choices must be str" + valids = set((str, int, float, bool, type(None))) + assert set([type(c) for c in choices]).issubset( + valids + ), f"choices must be in {valids}" assert "on_setattr" not in kwargs validators = [attrs.validators.in_(choices)] diff --git a/examples/air_filter.py b/examples/air_filter.py index c28d030..1c98fc6 100644 --- a/examples/air_filter.py +++ b/examples/air_filter.py @@ -78,7 +78,7 @@ def post_process(self, *run_args, **run_kwargs): this_dir = str(pathlib.Path(__file__).parent) this_dir = os.path.join(this_dir, "airfilter_report") if not os.path.exists(this_dir): - os.makedirs(this_dir, 754, exist_ok=True) + os.makedirs(this_dir, 0o754, exist_ok=True) csv = CSVReporter(path=this_dir, report_mode="daily") csv_latest = CSVReporter(path=this_dir, report_mode="single") diff --git a/requirements.txt b/requirements.txt index 89727e8..838e8a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ jupyter pandas scikit-learn~=1.3.2 seaborn +tabulate #Engineering sectionproperties~=3.1.2 diff --git a/setup.py b/setup.py index a58fc6f..f1e61f2 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup import setuptools -__version__ = "0.0.9" +__version__ = "0.1.0" def parse_requirements(filename):