From 8a6e56eb582a4b3842707ca4e0047f1542294e71 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Sat, 21 Sep 2024 13:48:16 -0400 Subject: [PATCH 01/15] Cache Changes & Permisisons Fixes: - need to use binary value for makedirs default was ok - move cache to user homedir hidden folder - remove ottermaticslib references --- engforge/configuration.py | 4 ++-- engforge/eng/geometry.py | 10 ++++++---- engforge/eng/solid_materials.py | 2 +- engforge/eng/structure_beams.py | 16 ++++++++-------- engforge/engforge_attributes.py | 28 ++++++++++++++++++++++++++++ examples/air_filter.py | 2 +- 6 files changed, 46 insertions(+), 16 deletions(-) diff --git a/engforge/configuration.py b/engforge/configuration.py index ec93767..ec38a1d 100644 --- a/engforge/configuration.py +++ b/engforge/configuration.py @@ -546,9 +546,9 @@ def __attrs_post_init__(self): # TODO: allow multiple parents if (hasattr(comp, "parent")) and (comp.parent is not None): self.warning( - f"Component {compnm} already has a parent {comp.parent} copying, and assigning to {self}" + f"Component {compnm} already has a parent {comp.parent} #copying, and assigning to {self}" ) - setattr(self, compnm, attrs.evolve(comp, parent=self)) + #setattr(self, compnm, attrs.evolve(comp, parent=self)) else: comp.parent = self diff --git a/engforge/eng/geometry.py b/engforge/eng/geometry.py index 75c33a4..98b9de7 100644 --- a/engforge/eng/geometry.py +++ b/engforge/eng/geometry.py @@ -36,14 +36,16 @@ # generic cross sections from # https://mechanicalbase.com/area-moment-of-inertia-calculator-of-certain-cross-sectional-shapes/ -temp_path = os.path.join(tempfile.gettempdir(), "shapely_sections") +_temp = os.path.join(os.path.expanduser('~'),'.forge_tmp') +temp_path = os.path.join(_temp, "shapely_sections") + section_cache = EnvVariable( - "FORGE_SECTION_CACHE", + "FORGE_CACHE", default=temp_path, desc="directory to cache section properties", ) -if "FORGE_SECTION_CACHE" not in os.environ and not os.path.exists(temp_path): - os.mkdir(temp_path) +if "FORGE_CACHE" not in os.environ and not os.path.exists(temp_path): + os.makedirs(temp_path,0o741,exist_ok=True) section_cache.info(f"loading section from {section_cache.secret}") diff --git a/engforge/eng/solid_materials.py b/engforge/eng/solid_materials.py index 509c5e1..62f9f8d 100644 --- a/engforge/eng/solid_materials.py +++ b/engforge/eng/solid_materials.py @@ -41,7 +41,7 @@ class SolidMaterial(SectionMaterial, PyNiteMat.Material, Configuration): __metaclass__ = SecMat name: str = attr.ib(default="solid material") - color: float = attr.ib(factory=random_color, hash=False, eq=ih) + color: tuple = attr.ib(factory=random_color, hash=False, eq=ih) # Structural Properties density: float = attr.ib(default=1.0) diff --git a/engforge/eng/structure_beams.py b/engforge/eng/structure_beams.py index a9db704..fa5f411 100644 --- a/engforge/eng/structure_beams.py +++ b/engforge/eng/structure_beams.py @@ -19,7 +19,7 @@ from engforge.system import System from engforge.eng.solid_materials import * from engforge.common import * -import engforge.eng.geometry as ottgeo +import engforge.eng.geometry as enggeo from engforge.eng.costs import CostModel, cost_property import sectionproperties @@ -69,7 +69,7 @@ class Beam(Component, CostModel): validator=attr.validators.instance_of( ( geometry.Geometry, - ottgeo.Profile2D, + enggeo.Profile2D, type(None), ) ) @@ -109,7 +109,7 @@ class Beam(Component, CostModel): def __on_init__(self): self.debug("initalizing...") # update material - if isinstance(self.section, ottgeo.ShapelySection): + if isinstance(self.section, enggeo.ShapelySection): if self.section.material is None: raise Exception("No Section Material") else: @@ -126,11 +126,11 @@ def update_section(self, section): material = self.material else: material = self.section.material - self.section = ottgeo.ShapelySection( + self.section = enggeo.ShapelySection( shape=self.section, material=self.material ) - if isinstance(self.section, ottgeo.ShapelySection): + if isinstance(self.section, enggeo.ShapelySection): self.debug(f"determining profile {section} properties...") self.in_Ix = self.section.Ixx self.in_Iy = self.section.Iyy @@ -138,7 +138,7 @@ def update_section(self, section): self.in_A = self.section.A self.in_Ixy = self.section.Ixy - elif isinstance(self.section, ottgeo.Profile2D): + elif isinstance(self.section, enggeo.Profile2D): self.warning(f"use shapely section instead {section} properties...") # raise Exception("use shapely section instead") self.in_Ix = self.section.Ixx @@ -220,7 +220,7 @@ def A(self) -> float: @property def Ao(self): """outside area, over ride for hallow sections""" - if isinstance(self.section, ottgeo.Profile2D): + if isinstance(self.section, enggeo.Profile2D): return self.section.Ao return self.A @@ -389,7 +389,7 @@ def show_mesh(self): def estimate_stress(self, force_calc=True, **forces): """uses the best available method to determine max stress in the beam, for ShapelySections this is done through a learning process, for other sections it is done through a simple calculation aimed at providing a conservative estimate""" - if isinstance(self.section, ottgeo.ShapelySection): + if isinstance(self.section, enggeo.ShapelySection): return self.section.estimate_stress(**forces, force_calc=force_calc) else: return self._fallback_estimate_stress(**forces) diff --git a/engforge/engforge_attributes.py b/engforge/engforge_attributes.py index f743879..aa2c3d3 100644 --- a/engforge/engforge_attributes.py +++ b/engforge/engforge_attributes.py @@ -246,6 +246,7 @@ def as_dict(self): o = {k: getattr(self, k, None) for k, v in inputs.items()} return o + #TODO: refactor this, allowing a nesting return option for sub components, by default True (later to be reverted to False, as a breaking change). this messes up hashing and we can just the other object hash @property def input_as_dict(self): """returns values as they are in the class instance, but converts classes inputs to their input_as_dict""" @@ -258,6 +259,13 @@ def input_as_dict(self): } return o + @property + def table_row_dict(self): + """returns values as they would be put in a table row from this instance ignoring any sub components""" + from engforge.configuration import Configuration + o = {k: getattr(self, k, None) for k in self.table_fields()} + return o + @property def numeric_as_dict(self): """recursively gets internal components numeric_as_dict as well as its own numeric values""" @@ -272,7 +280,22 @@ def numeric_as_dict(self): # Hashes # TODO: issue with logging sub-items + def hash(self,*args, **input_kw): + """hash by parm or by input_kw""" + d = {k:v for k,v in self.as_dict.items() if k in args} + d.update(input_kw) + return d,deepdiff.DeepHash(d, ignore_encoding_errors=True,significant_digits = 6) + def hash_with(self, **input_kw): + """ + Generates a hash for the object's dictionary representation, updated with additional keyword arguments. + Args: + **input_kw: Arbitrary keyword arguments to update the object's dictionary representation. + Returns: + The hash value of the updated dictionary. + Raises: + Any exceptions raised by deepdiff.DeepHash if hashing fails. + """ d = self.as_dict d.update(input_kw) return deepdiff.DeepHash(d, ignore_encoding_errors=True)[d] @@ -292,6 +315,11 @@ def input_hash(self): d = self.input_as_dict return deepdiff.DeepHash(d, ignore_encoding_errors=True)[d] + @property + def table_hash(self): + d,hsh = self.hash(**self.table_row_dict) + return hsh[d] + @property def numeric_hash(self): d = self.numeric_as_dict diff --git a/examples/air_filter.py b/examples/air_filter.py index c28d030..1c98fc6 100644 --- a/examples/air_filter.py +++ b/examples/air_filter.py @@ -78,7 +78,7 @@ def post_process(self, *run_args, **run_kwargs): this_dir = str(pathlib.Path(__file__).parent) this_dir = os.path.join(this_dir, "airfilter_report") if not os.path.exists(this_dir): - os.makedirs(this_dir, 754, exist_ok=True) + os.makedirs(this_dir, 0o754, exist_ok=True) csv = CSVReporter(path=this_dir, report_mode="daily") csv_latest = CSVReporter(path=this_dir, report_mode="single") From 49e60384913e3d464203f71d92302c16f0787e90 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Wed, 25 Sep 2024 22:43:39 -0400 Subject: [PATCH 02/15] only cache input --- engforge/engforge_attributes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engforge/engforge_attributes.py b/engforge/engforge_attributes.py index aa2c3d3..dba56e6 100644 --- a/engforge/engforge_attributes.py +++ b/engforge/engforge_attributes.py @@ -281,9 +281,9 @@ def numeric_as_dict(self): # Hashes # TODO: issue with logging sub-items def hash(self,*args, **input_kw): - """hash by parm or by input_kw""" - d = {k:v for k,v in self.as_dict.items() if k in args} - d.update(input_kw) + """hash by parm or by input_kw, only input can be hashed by lookup as system properties can create a recursive loop and should be deterministic from input""" + d = {k:v for k,v in self.input_as_dict.items() if k in args} + d.update(input_kw) #override with input_kw return d,deepdiff.DeepHash(d, ignore_encoding_errors=True,significant_digits = 6) def hash_with(self, **input_kw): From 8a12d1f6fd94e527fd304ae40f0fe013f45b68da Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Sat, 28 Sep 2024 12:38:52 -0400 Subject: [PATCH 03/15] Allow Setting Of Dataframes: - raises exception if a context exists on component ensuring no generated data is ovewritten. --- engforge/tabulation.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/engforge/tabulation.py b/engforge/tabulation.py index 5c08f98..f38fffc 100644 --- a/engforge/tabulation.py +++ b/engforge/tabulation.py @@ -83,10 +83,31 @@ def last_context(self): # @solver_cached #FIXME: not caching correctly @property # FIXME: this is slow def dataframe(self): + """ + Returns a pandas DataFrame based on the current context. + + This method checks for the presence of `last_context` and its `dataframe` attribute. + If they exist, it returns the `dataframe` from `last_context`. + If not, it checks for the `_patch_dataframe` attribute and returns it if it exists. + If neither condition is met, it returns an empty DataFrame. + + :return: A pandas DataFrame based on the current context or an empty DataFrame if no context is available. + :rtype: pandas.DataFrame + """ + """""" if hasattr(self, "last_context") and hasattr(self.last_context, "dataframe"): return self.last_context.dataframe + if hasattr(self,'_patch_dataframe') and self._patch_dataframe is not None: + return self._patch_dataframe return pandas.DataFrame([]) + @dataframe.setter + def dataframe(self,input_dataframe): + if hasattr(self, "last_context") and hasattr(self.last_context, "dataframe"): + raise Exception(f'may not set dataframe on run component') + self._patch_dataframe = input_dataframe + + @property def plotable_variables(self): """Checks columns for ones that only contain numeric types or haven't been explicitly skipped""" From 0259e02436c8d936ba64f846637a373b64c48d9f Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 1 Oct 2024 22:32:23 -0400 Subject: [PATCH 04/15] Passing Test Suite: - changed dataframe scoping back to `sys.comp.prop` from `sys_comp_prop` --- engforge/attributes.py | 3 +++ engforge/configuration.py | 5 +++-- engforge/dataframe.py | 3 ++- engforge/engforge_attributes.py | 6 +++++- engforge/problem_context.py | 10 +++++++--- engforge/test/test_comp_iter.py | 12 ++++++------ engforge/test/test_composition.py | 4 ++-- engforge/test/test_costs.py | 10 +++++----- 8 files changed, 33 insertions(+), 20 deletions(-) diff --git a/engforge/attributes.py b/engforge/attributes.py index 697a470..259cbda 100644 --- a/engforge/attributes.py +++ b/engforge/attributes.py @@ -186,11 +186,14 @@ def define(cls, **kwargs): def _setup_cls(cls, name, new_dict, **kwargs): # randomize name for specifics reasons uid = str(uuid.uuid4()) + #base info name = name + "_" + uid.replace("-", "")[0:16] new_dict["uuid"] = uid new_dict["default_options"] = cls.default_options.copy() new_dict["template_class"] = False new_dict["name"] = name + + #create a new type of attribute new_slot = type(name, (cls,), new_dict) new_slot.default_options["default"] = new_slot.make_factory() new_slot.default_options["validator"] = new_slot.configure_instance diff --git a/engforge/configuration.py b/engforge/configuration.py index ec38a1d..dfba880 100644 --- a/engforge/configuration.py +++ b/engforge/configuration.py @@ -170,12 +170,12 @@ def property_changed(instance, variable, value): # elif session: # notify session that this variable has changed # log.info(f'property changed {variable.name} {value}') + # TODO: notify change input system here, perhaps with & without a session # session.change_sys_var(variable,value,doset=False) - attrs = attr.fields(instance.__class__) # check identity of variable cur = getattr(instance, variable.name) - is_different = value != cur + is_different = value != cur if isinstance(value, (int, float, str)) else True is_var = variable in attrs chgnw = instance._anything_changed @@ -183,6 +183,7 @@ def property_changed(instance, variable, value): log.debug( f"checking property changed {instance}{variable.name} {value}|invar: {is_var}| nteqval: {is_different}" ) + print(f"checking property changed {instance}{variable.name} {value}|invar: {is_var}| nteqval: {is_different}") # Check if should be updated if not chgnw and is_var and is_different: diff --git a/engforge/dataframe.py b/engforge/dataframe.py index a72bb28..3db1672 100644 --- a/engforge/dataframe.py +++ b/engforge/dataframe.py @@ -166,8 +166,9 @@ def dataframe_variants(self): return o def format_columns(self, dataframe: pandas.DataFrame): + #replace(".", "_") dataframe.rename( - lambda x: x.replace(".", "_").lower(), axis="columns", inplace=True + lambda x: x.lower(), axis="columns", inplace=True ) # Plotting Interface diff --git a/engforge/engforge_attributes.py b/engforge/engforge_attributes.py index dba56e6..b8445fa 100644 --- a/engforge/engforge_attributes.py +++ b/engforge/engforge_attributes.py @@ -198,6 +198,7 @@ def plot_attributes(cls) -> typing.Dict[str, "Attribute"]: @classmethod def input_attrs(cls): + """Lists all input attributes for class""" return attr.fields_dict(cls) @classmethod @@ -215,6 +216,7 @@ def input_fields(cls, add_ign_types: list = None): @classmethod def numeric_fields(cls): + """no tuples,lists, dicts, strings, or attr base types""" ignore_types = ( ATTR_BASE, str, @@ -227,7 +229,9 @@ def numeric_fields(cls): @classmethod def table_fields(cls): - keeps = (str, float, int) # TODO: add numpy fields + """the table attributes corresponding to """ + #TODO: add list/numpy fields with vector stats + keeps = (str, float, int) typ = cls._get_init_attrs_data(keeps) return {k: v for k, v in typ.items()} diff --git a/engforge/problem_context.py b/engforge/problem_context.py index 93b2d2f..efc219a 100644 --- a/engforge/problem_context.py +++ b/engforge/problem_context.py @@ -269,7 +269,9 @@ class ProblemExec: ) def __getattr__(self, name): - """This is a special method that is called when an attribute is not found in the usual places, like when interior contexts (anything not the root (session_id=True)) are created that dont have the top level's attributes. some attributes will look to the parent session""" + """ + This is a special method that is called when an attribute is not found in the usual places, like when interior contexts (anything not the root (session_id=True)) are created that dont have the top level's attributes. some attributes will look to the parent session + """ # interior context lookup (when in active context, ie session exists) if hasattr(self.class_cache, "session") and name in root_possible: @@ -333,8 +335,8 @@ def __init__(self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts): if opts.pop("persist", False) or kw_dict.pop("persist", False): self.persist_contexts() - # temp solver storage - self.solver_hist = expiringdict.ExpiringDict(100, 60) + # temp solver storage #TODO + #self.solver_hist = expiringdict.ExpiringDict(100, 60) if self.log_level < 5: if hasattr(self.class_cache, "session"): @@ -425,6 +427,8 @@ def __init__(self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts): self.class_cache.session._prob_levels[self.level_name] = self # error if the system is different (it shouldn't be!) if self.system is not system: + #TODO: subproblems allow different systems, but the top level should be the same + #idea - use (system,pid) as key for problems_dict, (system,True) would be root problem. This breaks checking for `class_cache.session` though one could gather that from the root problem key` raise IllegalArgument( f"somethings wrong! change of comp! {self.system} -> {system}" ) diff --git a/engforge/test/test_comp_iter.py b/engforge/test/test_comp_iter.py index 1d95680..737bff8 100644 --- a/engforge/test/test_comp_iter.py +++ b/engforge/test/test_comp_iter.py @@ -102,7 +102,7 @@ def test_keys(self): for p in props: tkn = f"{ck}.{v}.{p}" should_keys.add(tkn) - dataframe_keys.add(tkn.replace(".", "_")) + dataframe_keys.add(tkn) sys_key = set(self.system.data_dict.keys()) mtch = should_keys.issubset(sys_key) @@ -161,7 +161,7 @@ def test_keys(self): for p in props: tkn = f"{ck}.{p}" should_keys.add(tkn) - dataframe_keys.add(tkn.replace(".", "_")) + dataframe_keys.add(tkn) sys_key = set(self.system.data_dict.keys()) mtch = should_keys.issubset(sys_key) @@ -177,18 +177,18 @@ def test_keys(self): self.assertTrue(dataframe_keys.issubset(set(df.keys()))) # test item existence - v1 = set(self.system.dataframe["cdict_current_item"]) + v1 = set(self.system.dataframe["cdict.current_item"]) v2 = set(comps["cdict"]) self.assertEqual(v1, v2) - d1 = set(self.system.dataframe["citer_current_item"]) + d1 = set(self.system.dataframe["citer.current_item"]) d2 = set(comps["citer"]) self.assertEqual(d1, d2) - dvs = self.system.dataframe["cdict_current_item"] - cvs = self.system.dataframe["citer_current_item"] + dvs = self.system.dataframe["cdict.current_item"] + cvs = self.system.dataframe["citer.current_item"] al = set(zip(dvs, cvs)) sh = set(itertools.product(v2, d2)) diff --git a/engforge/test/test_composition.py b/engforge/test/test_composition.py index 9ae926b..4db8de7 100644 --- a/engforge/test/test_composition.py +++ b/engforge/test/test_composition.py @@ -121,8 +121,8 @@ def test_signals(self): def test_input_and_run(self): self.system.run(**{"comp.aux": 5, "comp.comp.aux": 6}) self.assertEqual(len(self.system.dataframe), 1, f"wrong run config") - self.assertEqual(set(self.system.dataframe["comp_aux"]), set([5])) - self.assertEqual(set(self.system.dataframe["comp_comp_aux"]), set([6])) + self.assertEqual(set(self.system.dataframe["comp.aux"]), set([5])) + self.assertEqual(set(self.system.dataframe["comp.comp.aux"]), set([6])) # internal storage # self.assertEqual(set(self.system.comp.dataframe["aux"]), set([5])) diff --git a/engforge/test/test_costs.py b/engforge/test/test_costs.py index 38a4a94..3d7cd5b 100644 --- a/engforge/test/test_costs.py +++ b/engforge/test/test_costs.py @@ -96,7 +96,7 @@ def test_econ_array(self): df = er.dataframe tc = ( - df["econ_summary_total_cost"] == np.array([161.0, 220.0, 305.0, 390.0]) + df["econ.summary.total_cost"] == np.array([161.0, 220.0, 305.0, 390.0]) ).all() self.assertTrue(tc) @@ -273,14 +273,14 @@ def test_dataframe(self): dfc = df_complete match = ( - dfc["fan_blade_cost"] + dfc["motor_motor_cost"] - == dfc["econ_lifecycle_category_capex"] + dfc["fan.blade_cost"] + dfc["motor.motor_cost"] + == dfc["econ.lifecycle.category.capex"] ).all() self.assertTrue(match) match = ( - df_complete["fan_area"] * df_complete["fan_v"] - == df_complete["fan_volumetric_flow"] + df_complete["fan.area"] * df_complete["fan.v"] + == df_complete["fan.volumetric_flow"] ).all() self.assertTrue(match) From 81174e29da3f87bb362b5fd2c242daed69926685 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Sat, 5 Oct 2024 00:46:29 -0400 Subject: [PATCH 05/15] dont test structures in default package --- engforge/test/_pre_test_structures.py | 221 ++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 engforge/test/_pre_test_structures.py diff --git a/engforge/test/_pre_test_structures.py b/engforge/test/_pre_test_structures.py new file mode 100644 index 0000000..4851881 --- /dev/null +++ b/engforge/test/_pre_test_structures.py @@ -0,0 +1,221 @@ +import unittest +from matplotlib import pylab +from engforge.eng.structure import * +from engforge.eng.geometry import ShapelySection +from sectionproperties.pre.library.steel_sections import i_section + +from numpy import * + + +class test_cantilever(unittest.TestCase): + # tests the first example here + # https://www.engineeringtoolbox.com/cantilever-beams-d_1848.html + + def setUp(self): + self.st = Structure(name="cantilever_beam", add_gravity_force=False) + self.st.add_node("wall", 0, 0, 0) + self.st.add_node("free", 0, 5, 0) + + self.ibeam = i_section(0.3072, 0.1243, 0.0121, 0.008, 0.0089, 4) + beam = ShapelySection(shape=self.ibeam, material=ANSI_4130()) + self.bm = self.st.add_member("mem", "wall", "free", section=beam) + + self.st.def_support( + "wall", + support_DX=True, + support_DY=True, + support_DZ=True, + support_RY=True, + support_RX=True, + support_RZ=True, + ) + self.st.frame.add_node_load("free", "FX", 3000, case="gravity") + + self.st.run(check_statics=True) + + def test_beam(self): + self.subtest_assert_near(self.bm.A, 53.4 / (100**2)) + self.subtest_assert_near(self.bm.Ix, 8196 / (100**4)) + self.subtest_assert_near(self.bm.Iy, 388.8 / (100**4)) + self.subtest_assert_near(self.bm.section_mass, 41.9) + + self.subtest_assert_near(self.bm.max_von_mises(), 27.4e6) + self.subtest_assert_near(float(self.bm.data_dict["min_deflection_y"]), -0.0076) + # self.subtest_assert_near(float(self.bm.data_dict["max_shear_y"]), 3000) + self.subtest_assert_near(float(self.bm.data_dict["max_shear_y"]), 3000) + + df = self.st.node_dataframe.loc["gravity"] + + dfw = df.loc["wall"] + dff = df.loc["free"] + + self.subtest_assert_near(float(dfw["rxfx"]), -3000, msg="wall rxfx") + self.subtest_assert_near(float(dfw["rxmz"]), 15000, msg="wall rxmz") + self.subtest_assert_near(float(dfw["dx"]), 0, msg="wall dx") + + self.subtest_assert_near(float(dff["dx"]), 0.0076, msg="dx") + self.subtest_assert_near(float(dff["rxfx"]), 0, msg="rxfx") + self.subtest_assert_near(float(dff["rxmz"]), 0, msg="rxmz") + + stress_obj = self.bm.get_stress_at(0, "gravity") + # stress_obj.plot_stress_vm() + + def subtest_assert_near(self, value, truth, pct=0.025, **kw): + with self.subTest(**kw): + self.assertAlmostEqual(value, truth, delta=max(abs(truth * pct), abs(pct))) + + +class test_truss(unittest.TestCase): + # Match this example, no beam stresses + # https://engineeringlibrary.org/reference/trusses-air-force-stress-manual + + def setUp(self): + self.st = Structure(name="truss", add_gravity_force=False) + self.st.add_node("A", 0, 0, 0) + self.st.add_node("B", 15, 30 * sqrt(3) / 2, 0) + self.st.add_node("C", 45, 30 * sqrt(3) / 2, 0) + self.st.add_node("D", 75, 30 * sqrt(3) / 2, 0) + self.st.add_node("E", 90, 0, 0) + self.st.add_node("F", 60, 0, 0) + self.st.add_node("G", 30, 0, 0) + + pairs = set() + Lmin = 30 * sqrt(3) / 2 + Lmax = 30.1 + + for n1 in self.st.nodes.values(): + for n2 in self.st.nodes.values(): + L = numpy.sqrt( + (n1.X - n2.X) ** 2.0 + (n1.Y - n2.Y) ** 2.0 + (n1.Z - n2.Z) ** 2.0 + ) + + if ( + L >= Lmin + and L <= Lmax + and (n1.name, n2.name) not in pairs + and (n2.name, n1.name) not in pairs + ): + pairs.add((n1.name, n2.name)) + + elif (n1.name, n2.name) in pairs or ( + n2.name, + n1.name, + ) in pairs: + pass + + self.beam = sections.rectangular_section(1.5, 1.5) + material = ANSI_4130() + self.section = ShapelySection(shape=self.beam, material=material) + + constrained = ("A", "E") + for n1, n2 in pairs: + bkey = f"{n1}_{n2}" + self.bm = self.st.add_member( + bkey, + n1, + n2, + # material=ANSI_4130(), + section=self.section, + # min_mesh_size=0.01, + ) + + # if n1 not in constrained: + # print(f'releasing {bkey}') + # self.st.frame.DefineReleases(bkey, Rzi=True) + + # if n2 not in constrained: + # print(f'releasing {bkey} J') + # self.st.frame.DefineReleases(bkey, Rzj=True) + + self.st.def_support( + "A", + support_DX=True, + support_DY=True, + support_DZ=True, + support_RY=True, + support_RX=True, + support_RZ=True, + ) + self.st.def_support( + "E", + support_DX=False, + support_DY=True, + support_DZ=True, + support_RY=False, + support_RX=False, + support_RZ=True, + ) + # for node in self.st.nodes: + # self.st.frame.Definesupport_(node,support_DZ=True,support_RZ=True) + + self.st.add_node_load("F", "FY", -1000, case="gravity") + self.st.add_node_load("G", "FY", -2000, case="gravity") + + self.st.run(check_statics=True) + # self.st.visulize() + + def test_reactions(self): + df = self.st.node_dataframe.loc["gravity"] + # print(df) + + dfa = df.loc["A"] + dfe = df.loc["E"] + + # print(dfa) + # print(dfe) + + self.subtest_assert_near(float(dfa["rxfy"]), 1667) + self.subtest_assert_near(float(dfe["rxfy"]), 1333) + + self.subtest_member("A", "B", "max_axial", 1925) + self.subtest_member("A", "G", "max_axial", -949) # -926 textbook + self.subtest_member("B", "C", "max_axial", 1925) + self.subtest_member("B", "G", "max_axial", -1925) + self.subtest_member("C", "D", "max_axial", 1541) + self.subtest_member("F", "G", "max_axial", -1734) + self.subtest_member("C", "F", "max_axial", 382) + self.subtest_member("C", "G", "max_axial", -382) + self.subtest_member("C", "G", "max_axial", -378) # 1541 textbook + self.subtest_member("D", "F", "max_axial", -1541) + self.subtest_member("E", "F", "max_axial", -770) + + # Visualization.RenderModel( self.st.frame ) + + def subtest_member(self, nodea, nodeb, result_key, truth, pct=0.025): + with self.subTest(msg=f"test {nodea}->{nodeb}"): + key_1 = f"{nodea}_{nodeb}" + key_2 = f"{nodeb}_{nodea}" + + if key_1 in self.st.beams: + key = key_1 + elif key_2 in self.st.beams: + key = key_2 + else: + raise + + value = self.get_member_result(nodea, nodeb, result_key) + dopasst = abs(value - truth) <= abs(truth) * pct + + if not dopasst: + print(f"fails {key} {result_key}| {value:3.5f} == {truth:3.5f}?") + self.subtest_assert_near(value, truth, pct=pct) + + def subtest_assert_near(self, value, truth, pct=0.025): + with self.subTest(): + self.assertAlmostEqual(value, truth, delta=abs(truth * pct)) + + def get_member_result(self, nodea, nodeb, result_key): + key_1 = f"{nodea}_{nodeb}" + key_2 = f"{nodeb}_{nodea}" + + if key_1 in self.st.beams: + mem = self.st.beams[key_1] + if result_key in mem.data_dict: + return float(mem.data_dict[result_key]) + + elif key_2 in self.st.beams: + mem = self.st.beams[key_2] + if result_key in mem.data_dict: + return float(mem.data_dict[result_key]) + + return numpy.nan # shouod fail, nan is not comparable From 517e5db9b0ed424f6c687c929af5356ca010c475 Mon Sep 17 00:00:00 2001 From: Kevin Russell Date: Sun, 6 Oct 2024 03:11:19 -0400 Subject: [PATCH 06/15] Always Update + Num-Costs! -costs supports a number of items concept on class witth on prperty override. - set `entter_refresh` to true always with a full refresh. There will be a performance hit but its not ok to not work right. - set attr type for attributes - get none values in components - set types in engforge_props - correct class cache name for system propclass --- engforge/_testing_components.py | 2 +- engforge/configuration.py | 4 + engforge/eng/costs.py | 207 +++++++++++++++++++++++++---- engforge/engforge_attributes.py | 8 +- engforge/problem_context.py | 44 ++++-- engforge/properties.py | 12 +- engforge/solveable.py | 9 +- engforge/solver.py | 2 - engforge/tabulation.py | 7 +- engforge/test/test_costs.py | 2 +- engforge/test/test_structures.py | 221 ------------------------------- 11 files changed, 242 insertions(+), 276 deletions(-) delete mode 100644 engforge/test/test_structures.py diff --git a/engforge/_testing_components.py b/engforge/_testing_components.py index 48512d7..4a966db 100644 --- a/engforge/_testing_components.py +++ b/engforge/_testing_components.py @@ -852,7 +852,7 @@ def calculate_production(self, parent, term): @forge class FanSystem(System, CostModel): - base = Slot.define(Component) + base = Slot.define(Component) #FIXME: not showing in "econ" (due to base default cost?) fan = Slot.define(Fan) motor = Slot.define(Motor) diff --git a/engforge/configuration.py b/engforge/configuration.py index dfba880..0113819 100644 --- a/engforge/configuration.py +++ b/engforge/configuration.py @@ -318,6 +318,7 @@ def signals_slots_handler( cls_properties = cls.system_properties_classdef(True) else: cls_properties = {} + cls_dict = cls.__dict__.copy() cls.__anony_store = {} # print(f'tab found!! {cls_properties.keys()}') @@ -461,6 +462,7 @@ def copy_config_at_state( new_sys.system_references(recache=True) # update the parents + #TODO: use pyee to broadcast change if hasattr(self, "parent"): if self.parent in changed: new_sys.parent = changed[self.parent] @@ -482,6 +484,8 @@ def go_through_configurations( :return: level,config""" from engforge.configuration import Configuration + #TODO: instead of a recursive loop a global map per problem context should be used, with a static map of slots, updating with every change per note in system_references. This function could be a part of that but each system shouldn't be responsible for it. + should_yield_level = lambda level: all( [ level >= parent_level, diff --git a/engforge/eng/costs.py b/engforge/eng/costs.py index d116222..90adb18 100644 --- a/engforge/eng/costs.py +++ b/engforge/eng/costs.py @@ -4,16 +4,22 @@ CostModel's can have cost_property's which detail how and when a cost should be applied & grouped. By default each CostModel has a `cost_per_item` which is reflected in `item_cost` cost_property set on the `initial` term as a `unit` category. Multiple categories of cost are also able to be set on cost_properties as follows +Cost Models can represent multiple instances of a component, and can be set to have a `num_items` multiplier to account for multiple instances of a component. CostModels can have a `term_length` which will apply costs over the term, using the `cost_property.mode` to determine at which terms a cost should be applied. + ``` @forge class Widget(Component,CostModel): + + #use num_items as a multiplier for costs, `cost_properties` can have their own custom num_items value. + num_items:float = 100 + + @cost_property(mode='initial',category='capex,manufacturing',num_items=1) + def cost_of_XYZ(self) -> float: + return cost - @cost_property(mode='initial',category='capex,manufacturing') - def cost_of_XYZ(self): - return ... ``` -Economics models sum CostModel.cost_properties recursively on the parent they are defined. Economics computes the grouped category costs for each item recursively as well as summary properties like annualized values and levalized cost. Economic output is determined by a `fixed_output` or overriding `calculate_production(self,parent)` to dynamically calculate changing economics based on factors in the parent. +Economics models sum CostModel.cost_properties recursively on the parent they are defined. Economics computes the grouped category costs for each item recursively as well as summary properties like annualized values and levelized cost. Economic output is determined by a `fixed_output` or overriding `calculate_production(self,parent)` to dynamically calculate changing economics based on factors in the parent. Default costs can be set on any CostModel.Slot attribute, by using default_cost(,) on the class, this will provide a default cost for the slot if no cost is set on the instance. Custom costs can be set on the instance with custom_cost(,). If cost is a CostModel, it will be assigned to the slot if it is not already assigned. @@ -47,7 +53,7 @@ class Parent(System,CostModel) import numpy import collections import pandas - +import collections class CostLog(LoggingMixin): pass @@ -80,9 +86,10 @@ class cost_property(system_property): Categories are a way to report cost categories and multiple can be applied to a cost. Categories are grouped by the Economics system at reported in bulk by term and over the term_length """ - + valild_types = (int, float) cost_categories: list = None term_mode: str = None + num_items: int = None _all_modes: dict = COST_TERM_MODES _all_categories: set = COST_CATEGORIES @@ -98,11 +105,14 @@ def __init__( stochastic=False, mode: str = "initial", category: category_type = None, + num_items: int = None, ): """extends system_property interface with mode & category keywords :param mode: can be one of `initial`,`maintenance`,`always` or a function with signature f(inst,term) as an integer and returning a boolean True if it is to be applied durring that term. """ super().__init__(fget, fset, fdel, doc, desc, label, stochastic) + + self.valild_types = (int, str) #only numerics if isinstance(mode, str): mode = mode.lower() assert ( @@ -116,6 +126,7 @@ def __init__( else: raise ValueError(f"mode: {mode} must be cost term str or callable") + #cost categories if category is not None: if isinstance(category, str): self.cost_categories = category.split(",") @@ -128,6 +139,10 @@ def __init__( else: self.cost_categories = ["misc"] + #number of items override + if num_items is not None: + self.num_items = num_items + def apply_at_term(self, inst, term): if term < 0: raise ValueError(f"negative term!") @@ -146,6 +161,19 @@ def get_func_return(self, func): else: self.return_type = float + + def __get__(self, obj, objtype=None): + if obj is None: + return self # class support + if self.fget is None: + raise AttributeError("unreadable attribute") + + #apply the costmodel with the item multiplier + if self.num_items is not None: + k = self.num_items + else: + k = obj.num_items if isinstance(obj, CostModel) else 1 + return k * self.fget(obj) @forge class CostModel(Configuration, TabulationMixin): @@ -156,9 +184,13 @@ class CostModel(Configuration, TabulationMixin): `sub_items_cost` system_property summarizes the costs of any component in a Slot that has a `CostModel` or for SlotS which CostModel.declare_cost(`slot`,default=numeric|CostModelInst|dict[str,float]) """ + #TODO: remove "default costs" concept and just use cost_properties since thats declarative and doesn't create a "phantom" representation to maintain + #TODO: it might be a good idea to add a "castable" namespace for components so they can all reference a common dictionary. Maybe all problem variables are merged into a single namespace to solve issues of "duplication" _slot_costs: dict # TODO: insantiate per class + #basic attribute interface returns the item cost as `N x cost_per_item`` cost_per_item: float = attrs.field(default=numpy.nan) + num_items: int = attrs.field(default=1) #set to 0 to disable the item cost def __on_init__(self): self.set_default_costs() @@ -227,8 +259,8 @@ def default_cost( cost, CostModel ), "only numeric types or CostModel instances supported" - atrb = cls.slots_attributes()[slot_name] - atypes = atrb.type.accepted + atrb = cls.slots_attributes(attr_type=True)[slot_name] + atypes = atrb.accepted if warn_on_non_costmodel and not any( [issubclass(at, CostModel) for at in atypes] ): @@ -253,8 +285,8 @@ def custom_cost( cost, CostModel ), "only numeric types or CostModel instances supported" - atrb = self.__class__.slots_attributes()[slot_name] - atypes = atrb.type.accepted + atrb = self.__class__.slots_attributes(attr_type=True)[slot_name] + atypes = atrb.accepted if warn_on_non_costmodel and not any( [issubclass(at, CostModel) for at in atypes] ): @@ -276,7 +308,7 @@ def custom_cost( def calculate_item_cost(self) -> float: """override this with a parametric model related to this systems attributes and properties""" - return self.cost_per_item + return self.num_items*self.cost_per_item @system_property def sub_items_cost(self) -> float: @@ -368,7 +400,7 @@ def sub_costs(self, saved: set = None, categories: tuple = None, term=0): # add base class slot values when comp was nonee if comp is None: # print(f'skipping {slot}:{comp}') - comp_cls = self.slots_attributes()[slot].type.accepted + comp_cls = self.slots_attributes(attr_type=True)[slot].accepted for cc in comp_cls: if issubclass(cc, CostModel): if cc._slot_costs: @@ -430,6 +462,8 @@ def all_categories(self): return COST_CATEGORIES + + cost_type = typing.Union[float, int, CostModel, dict] @@ -458,14 +492,47 @@ def gend(deect: dict): else: yield k, v - parent_types = typing.Union[Component, "System"] # TODO: automatically apply economics at problem level if cost_model present, no need for parent econ lookups @forge class Economics(Component): - """Economics is a component that summarizes costs and reports the economics of a system and its components in a recursive format""" + """ + Economics is a component that summarizes costs and reports the economics of a system and its components in a recursive format. + Attributes: + term_length (int): The length of the term for economic calculations. Default is 0. + discount_rate (float): The discount rate applied to economic calculations. Default is 0.0. + fixed_output (float): The fixed output value for the economic model. Default is numpy.nan. + output_type (str): The type of output for the economic model. Default is "generic". + terms_per_year (int): The number of terms per year for economic calculations. Default is 1. + _calc_output (float): Internal variable to store calculated output. + _costs (float): Internal variable to store calculated costs. + _cost_references (dict): Internal dictionary to store cost references. + _cost_categories (dict): Internal dictionary to store cost categories. + _comp_categories (dict): Internal dictionary to store component categories. + _comp_costs (dict): Internal dictionary to store component costs. + parent (parent_types): The parent component or system. + Methods: + __on_init__(): Initializes internal dictionaries for cost and component categories. + update(parent: parent_types): Updates the economic model with the given parent component or system. + calculate_production(parent, term) -> float: Calculates the production output for the given parent and term. Must be overridden. + calculate_costs(parent) -> float: Recursively calculates the costs for the given parent component or system. + sum_cost_references(): Sums the cost references stored in the internal dictionary. + sum_references(refs): Sums the values of the given references. + get_prop(ref): Retrieves the property associated with the given reference. + term_fgen(comp, prop): Generates a function to calculate the term value for the given component and property. + sum_term_fgen(ref_group): Sums the term functions for the given reference group. + internal_references(recache=True, numeric_only=False): Gathers and sets internal references for the economic model. + lifecycle_output() -> dict: Returns lifecycle calculations for the levelized cost of energy (LCOE). + lifecycle_dataframe() -> pandas.DataFrame: Simulates the economics lifecycle and stores the results in a term-based dataframe. + _create_term_eval_functions(): Creates evaluation functions for term-based calculations grouped by categories and components. + _gather_cost_references(parent: "System"): Gathers cost references from the parent component or system. + _extract_cost_references(conf: "CostModel", bse: str): Extracts cost references from the given cost model configuration. + cost_references(): Returns the internal dictionary of cost references. + combine_cost() -> float: Returns the combined cost of the economic model. + output() -> float: Returns the calculated output of the economic model. + """ term_length: int = attrs.field(default=0) discount_rate: float = attrs.field(default=0.0) @@ -502,6 +569,37 @@ def update(self, parent: parent_types): if self._costs is None: self.warning(f"no economic costs!") + self._anything_changed = True + + #TODO: expand this... + @solver_cached + def econ_output(self): + return self.lifecycle_output + + @system_property(label='cost/output') + def levelized_cost(self)->float: + """Price per kwh""" + eco = self.econ_output + return eco['summary.levelized_cost'] + + @system_property + def total_cost(self)->float: + eco = self.econ_output + return eco['summary.total_cost'] + + @system_property + def levelized_output(self)->float: + """ouput per dollar (KW/$)""" + eco = self.econ_output + return eco['summary.levelized_output'] + + @system_property + def term_years(self)->float: + """ouput per dollar (KW/$)""" + eco = self.econ_output + return eco['summary.years'] + + #Calculate Output def calculate_production(self, parent, term) -> float: """must override this function and set economic_output""" return numpy.nansum([0, self.fixed_output]) @@ -510,7 +608,8 @@ def calculate_costs(self, parent) -> float: """recursively accounts for costs in the parent, its children recursively.""" return self.sum_cost_references() - # Reference Utilitly Functions + #Reference Utilitly Functions + #Set Costs over time "flat" through ref trickery... def sum_cost_references(self): cst = 0 for k, v in self._cost_references.items(): @@ -550,6 +649,9 @@ def sum_term_fgen(self, ref_group): # TODO: update internal_references callback to problem def internal_references(self, recache=True, numeric_only=False): """standard component references are""" + + recache=True #override + d = self._gather_references() self._create_term_eval_functions() # Gather all internal economic variables and report costs @@ -586,6 +688,65 @@ def internal_references(self, recache=True, numeric_only=False): return d + def cost_summary(self, annualized=False, do_print=True, ignore_zero=True): + """ + Generate a summary of costs, optionally annualized, and print the summary. + :param annualized: If True, include only annualized costs in the summary. Default is False. + :type annualized: bool + :param do_print: If True, print the summary to the console. Default is True. + :type do_print: bool + :param ignore_zero: If True, ignore costs with a value of zero. Default is True. + :type ignore_zero: bool + :return: A dictionary containing component costs, skipped costs, and summary. + :rtype: dict + """ + + dct = self.lifecycle_output + cols = list(dct.keys()) + + skippd = set() + comp_costs = collections.defaultdict(dict) + summary = {} + costs = {'comps':comp_costs,'skip':skippd,'summary':summary} + + def abriv(val): + if val > 1E6: + return f'{val/1E6:^12.4f}M' + elif val > 1E3: + return f'{val/1E3:^12.2f}k' + return f'{val:^12.2f}' + + for col in cols: + is_ann = '.annualized.' in col + val = dct[col] + + #handle no value case + if val == 0 and ignore_zero: + continue + + if '.cost.' in col and is_ann == annualized: + base,cst = col.split('.cost.') + comp_costs[base][cst]= val + elif col.startswith('summary.'): + summary[col.replace('summary.','')]=val + + total_cost = sum([sum(list(cc.values())) for cc in comp_costs.values()]) + + if do_print: + self.info('_'*80) + for key,val in summary.items(): + self.info(f' {key:<35}| {val:^12.10f}') + self.info('-'*80) + + for base,items in comp_costs.items(): + if (subtot:=sum(list(items.values()))) > 0: + self.info(f' {base:<35}| total ---> {abriv(subtot)} | {subtot*100/total_cost:3.0f}%') + for key,val in sorted(items.items(),lambda kv:[0]): + self.info(f' \t{key:<32}|{abriv(val)} | {val*100/total_cost:^3.0f}%') + self.info('-'*80) + self.info('_'*80) + return costs + @property def lifecycle_output(self) -> dict: """return lifecycle calculations for lcoe""" @@ -609,10 +770,10 @@ def lifecycle_output(self) -> dict: summary["total_cost"] = lc.term_cost.sum() summary["years"] = lc.year.max() + 1 - LC = lc.levalized_cost.sum() - LO = lc.levalized_output.sum() - summary["levalized_cost"] = LC / LO if LO != 0 else numpy.nan - summary["levalized_output"] = LO / LC if LC != 0 else numpy.nan + LC = lc.levelized_cost.sum() + LO = lc.levelized_output.sum() + summary["levelized_cost"] = LC / LO if LO != 0 else numpy.nan + summary["levelized_output"] = LO / LC if LC != 0 else numpy.nan out2 = dict(gend(out)) self._term_output = out2 @@ -641,9 +802,9 @@ def lifecycle_dataframe(self) -> pandas.DataFrame: row["term_cost"] = tc = numpy.nansum( [v(t) for v in self._term_comp_cost.values()] ) - row["levalized_cost"] = tc * (1 + self.discount_rate) ** (-1 * t) + row["levelized_cost"] = tc * (1 + self.discount_rate) ** (-1 * t) row["output"] = output = self.calculate_production(self.parent, t) - row["levalized_output"] = output * (1 + self.discount_rate) ** (-1 * t) + row["levelized_output"] = output * (1 + self.discount_rate) ** (-1 * t) return pandas.DataFrame(out) @@ -700,7 +861,7 @@ def _gather_cost_references(self, parent: "System"): self.debug(f"checking {key} {comp_key} {kbase}") - # Get Costs Directly From the cost model instance + #0. Get Costs Directly From the cost model instance if isinstance(conf, CostModel): comps[key] = conf self.debug(f"adding cost model for {kbase}.{comp_key}") @@ -834,7 +995,7 @@ def _extract_cost_references(self, conf: "CostModel", bse: str): f"{conf} looking up base class costs for {compnm}", lvl=5, ) - comp_cls = conf.slots_attributes()[compnm].type.accepted + comp_cls = conf.slots_attributes(attr_type=True)[compnm].accepted for cc in comp_cls: if issubclass(cc, CostModel): if cc._slot_costs: diff --git a/engforge/engforge_attributes.py b/engforge/engforge_attributes.py index b8445fa..e496180 100644 --- a/engforge/engforge_attributes.py +++ b/engforge/engforge_attributes.py @@ -71,7 +71,7 @@ def collect_inst_attributes(self, **kw): return out @classmethod - def _get_init_attrs_data(cls, subclass_of: type, exclude=False): + def _get_init_attrs_data(cls, subclass_of: type, exclude=False,attr_type=False): choose = issubclass if exclude: choose = lambda ty, type_set: not issubclass(ty, type_set) @@ -80,7 +80,7 @@ def _get_init_attrs_data(cls, subclass_of: type, exclude=False): if "__attrs_attrs__" in cls.__dict__: # Handle Attrs Class for k, v in attrs.fields_dict(cls).items(): if isinstance(v.type, type) and choose(v.type, subclass_of): - attrval[k] = v + attrval[k] = v.type if attr_type else v # else: # Handle Pre-Attrs Class # FIXME: should this run first? @@ -167,9 +167,9 @@ def slot_refs(cls, recache=False): return o @classmethod - def slots_attributes(cls) -> typing.Dict[str, "Attribute"]: + def slots_attributes(cls,attr_type=False) -> typing.Dict[str, "Attribute"]: """Lists all slots attributes for class""" - return cls._get_init_attrs_data(Slot) + return cls._get_init_attrs_data(Slot,attr_type=attr_type) @classmethod def signals_attributes(cls) -> typing.Dict[str, "Attribute"]: diff --git a/engforge/problem_context.py b/engforge/problem_context.py index efc219a..7f9b51c 100644 --- a/engforge/problem_context.py +++ b/engforge/problem_context.py @@ -126,7 +126,7 @@ class ProbLog(LoggingMixin): save_mode="all", x_start=None, save_on_exit=False, - enter_refresh=False, + enter_refresh=True, ) # can be found on session._ or session. root_defined = dict( @@ -209,6 +209,7 @@ class ProblemExec: - _problem_id: uuid for subproblems, or True for top level, None means uninitialized """ + full_update = True #TODO: cant justify setting this to false for performance gains. accuracy comes first. see event based update # TODO: convert this to a system based cache where there is a unique problem for each system instance. On subprobem copy a system and add o dictionary. class_cache = None # ProblemExec is assigned below @@ -244,7 +245,7 @@ class ProblemExec: _converged: bool # Interior Context Options - enter_refresh: bool = False + enter_refresh: bool = True #TODO: allow this off with event system save_on_exit: bool = False save_mode: str = "all" level_name: str = None # target this context with the level name @@ -593,19 +594,14 @@ def get_sesh(self, sesh=None): # Update Methods def refresh_references(self, sesh=None): """refresh the system references""" - sesh = self.sesh + + if sesh is None: + sesh = self.sesh if self.log_level < 5: self.warning(f"refreshing system references") - check_dynamics = sesh.check_dynamics - sesh._num_refs = sesh.system.system_references(numeric_only=True) - sesh._sys_refs = sesh.system.solver_vars( - check_dynamics=check_dynamics, - addable=sesh._num_refs, - **sesh._slv_kw, - ) - sesh.update_methods(sesh=sesh) + sesh.full_refresh(sesh=sesh) sesh.min_refresh(sesh=sesh) def update_methods(self, sesh=None): @@ -625,16 +621,36 @@ def update_dynamics(self, sesh=None): self.info(f"update dynamics") self.system.setup_global_dynamics() + def full_refresh(self,sesh=None): + """a more time consuming but throughout refresh of the system""" + if self.log_level < 5: + self.info(f"full refresh") + + check_dynamics = sesh.check_dynamics + sesh._num_refs = sesh.system.system_references(numeric_only=True,none_ok=True, only_inst=False,ignore_none_comp=False,recache=True) + sesh._sys_refs = sesh.system.solver_vars( + check_dynamics=check_dynamics, + addable=sesh._num_refs, + **sesh._slv_kw, + ) + sesh.update_methods(sesh=sesh) + def min_refresh(self, sesh=None): + """what things need to be refreshed per execution, this is important whenever items are replaced""" + #TODO: replace this function with an event based responsiblity model. sesh = sesh if sesh is not None else self.sesh if self.log_level < 5: self.info(f"min refresh") + if sesh.full_update: + #TODO: dont require this + sesh.full_refresh(sesh=sesh) + # final ref's after update # after updates sesh._all_refs = sesh.system.system_references( - recache=True, check_config=False, ignore_none_comp=False + recache=True, check_config=False, ignore_none_comp=False,none_ok=True, only_inst=False ) # sesh._attr_sys_key_map = sesh.attribute_sys_key_map @@ -642,7 +658,7 @@ def min_refresh(self, sesh=None): sesh.Xref = sesh.all_problem_vars sesh.Yref = sesh.sys_solver_objectives() - cons = {} # TODO: parse additional constraints + cons = {} #TODO: parse additional constraints sesh.constraints = sesh.sys_solver_constraints(cons) @property @@ -676,7 +692,6 @@ def __enter__(self): # transients wont update components/ methods dynamically (or shouldn't) so we can just update the system references once and be done with it for other cases, but that is not necessary unless a component changes or a component has in general a unique reference update system (economics / component-iterators) sesh = self.sesh if not sesh._dxdt is True and self.enter_refresh: - sesh.update_methods(sesh=sesh) sesh.min_refresh(sesh=sesh) elif sesh.dynamics_updated: @@ -806,6 +821,7 @@ def debug_levels(self): raise IllegalArgument(f"no session available") # Multi Context Exiting: + #TODO: rethink this def persist_contexts(self): """convert all contexts to a new storage format""" self.info(f"persisting contexts!") diff --git a/engforge/properties.py b/engforge/properties.py index ed4dcd4..b516182 100644 --- a/engforge/properties.py +++ b/engforge/properties.py @@ -21,6 +21,7 @@ class PropertyLog(LoggingMixin): log = PropertyLog() +_basic_valild_types = (int, str, float) #check for return_type class engforge_prop: """ @@ -28,15 +29,15 @@ class engforge_prop: Use as follows: @engforge_prop - def our_custom_function(self): + def our_custom_function(self) -> return_type: pass """ - + valid_types = _basic_valild_types must_return = False def __init__(self, fget=None, fset=None, fdel=None, *args, **kwargs): """call with the function you want to wrap in a decorator""" - + self.valild_types = (int, str, float) #check for return_type self.fget = fget if fget: self.gname = fget.__name__ @@ -63,7 +64,7 @@ def get_func_return(self, func): """ensures that the function has a return annotation, and that return annotation is in valid sort types""" anno = func.__annotations__ typ = anno.get("return", None) - if not typ in (int, str, float) and self.must_return: + if not typ in self.valid_types and self.must_return: raise Exception( f"system_property input: function {func.__name__} must have valid return annotation of type: {(int,str,float)}" ) @@ -102,6 +103,7 @@ class cache_prop(engforge_prop): def __init__(self, *args, **kwargs): self.allow_set = True + self.valild_types = (int, str, float) #check for return_type super().__init__(*args, **kwargs) def __set__(self, instance, value): @@ -155,7 +157,7 @@ def function(...): < this uses __init__ to assign function @system_property(desc='really nice',label='funky function') def function(...): < this uses __call__ to assign function """ - + self.valild_types = (int, str, float) #check for return_type self.fget = fget if fget: self.get_func_return(fget) diff --git a/engforge/solveable.py b/engforge/solveable.py index 9deb692..561ed2a 100644 --- a/engforge/solveable.py +++ b/engforge/solveable.py @@ -334,8 +334,9 @@ def comp_references(self, ignore_none_comp=True, **kw): #FIXME: by instance recache on iterative component change or other signals """ out = {} - for key, lvl, comp in self.go_through_configurations(parent_level=1, **kw): + for key,lvl,comp in self.go_through_configurations(parent_level=1,**kw): if ignore_none_comp and not isinstance(comp, SolveableMixin): + self.warning(f"ignoring {key} {lvl}|{comp}") continue out[key] = comp return out @@ -530,7 +531,7 @@ def locate(cls, key, fail=True) -> type: args = key.split(".") comp, sub = args[0], ".".join(args[1:]) assert comp in cls.slots_attributes(), f"invalid {comp} in {key}" - comp_cls = cls.slots_attributes()[comp].type.accepted[0] + comp_cls = cls.slots_attributes(attr_type=True)[comp].accepted[0] val = comp_cls.locate(sub, fail=True) elif key in cls.input_fields(): @@ -606,6 +607,8 @@ def system_references(self, recache=False, numeric_only=False, **kw): ): return self._prv_system_references + #TODO: system references are really important and nesting them together complicates the refresh process. Each component should be able to refresh itself and its children on set_state, as well as alert parents to change. Ideally the `Ref` objects could stay the same and no `recache` would need to occur. This would be a huge performance boost and fix a lot of the issues with the current system. + out = self.internal_references(recache, numeric_only=numeric_only) tatr = out["attributes"] tprp = out["properties"] @@ -614,6 +617,8 @@ def system_references(self, recache=False, numeric_only=False, **kw): # component iternals for key, comp in self.comp_references(**kw).items(): + if comp is None: + continue sout = comp.internal_references(recache, numeric_only=numeric_only) satr = sout["attributes"] sprp = sout["properties"] diff --git a/engforge/solver.py b/engforge/solver.py index c192579..9a93d44 100644 --- a/engforge/solver.py +++ b/engforge/solver.py @@ -244,8 +244,6 @@ def solver(self, enter_refresh=True, save_on_exit=True, **kw): """ runs the system solver using the current system state and modifying it. This is the default solver for the system, and it is recommended to add additional options or methods via the execute method. - - :param obj: the objective function to minimize, by default will minimize the sum of the squares of the residuals. Objective function should be a function(system,Xs,Xt) where Xs is the system state and Xt is the system transient state. The objective function will be argmin(X)|(1+custom_objective)*residual_RSS when `add_obj` is True in kw otherwise argmin(X)|custom_objective with constraints on the system as balances instead of first objective being included. :param cons: the constraints to be used in the solver, by default will use the system's constraints will be enabled when True. If a dictionary is passed the solver will use the dictionary as the constraints in addition to system constraints. These can be individually disabled by key=None in the dictionary. diff --git a/engforge/tabulation.py b/engforge/tabulation.py index f38fffc..08012b0 100644 --- a/engforge/tabulation.py +++ b/engforge/tabulation.py @@ -233,9 +233,10 @@ def system_properties_classdef(cls, recache=False): """Combine other parent-classes table properties into this one, in the case of subclassed system_properties""" from engforge.tabulation import TabulationMixin + cls_key =f"_{cls.__name__}_system_properties" # Use a cache for deep recursion - if not recache and hasattr(cls, "_{cls.__name__}_system_properties"): - res = getattr(cls, "_{cls.__name__}_system_properties") + if not recache and hasattr(cls, cls_key): + res = getattr(cls, cls_key) if res is not None: return res @@ -269,7 +270,7 @@ def system_properties_classdef(cls, recache=False): __system_properties[k] = prop log.msg(f"adding system property {mrv.__name__}.{k}") - setattr(cls, "_{cls.__name__}_system_properties", __system_properties) + setattr(cls,cls_key, __system_properties) return __system_properties diff --git a/engforge/test/test_costs.py b/engforge/test/test_costs.py index 3d7cd5b..2399c51 100644 --- a/engforge/test/test_costs.py +++ b/engforge/test/test_costs.py @@ -123,7 +123,7 @@ def test_econ_defaults(self): d = er.data_dict self.assertEqual(78, d["econ.summary.total_cost"]) self.assertEqual(78, d["econ.lifecycle.annualized.term_cost"]) - self.assertEqual(78, d["econ.lifecycle.annualized.levalized_cost"]) + self.assertEqual(78, d["econ.lifecycle.annualized.levelized_cost"]) self.assertEqual(78, d["econ.lifecycle.term_cost"]) def test_recursive_null(self, ANS=75): diff --git a/engforge/test/test_structures.py b/engforge/test/test_structures.py deleted file mode 100644 index 4851881..0000000 --- a/engforge/test/test_structures.py +++ /dev/null @@ -1,221 +0,0 @@ -import unittest -from matplotlib import pylab -from engforge.eng.structure import * -from engforge.eng.geometry import ShapelySection -from sectionproperties.pre.library.steel_sections import i_section - -from numpy import * - - -class test_cantilever(unittest.TestCase): - # tests the first example here - # https://www.engineeringtoolbox.com/cantilever-beams-d_1848.html - - def setUp(self): - self.st = Structure(name="cantilever_beam", add_gravity_force=False) - self.st.add_node("wall", 0, 0, 0) - self.st.add_node("free", 0, 5, 0) - - self.ibeam = i_section(0.3072, 0.1243, 0.0121, 0.008, 0.0089, 4) - beam = ShapelySection(shape=self.ibeam, material=ANSI_4130()) - self.bm = self.st.add_member("mem", "wall", "free", section=beam) - - self.st.def_support( - "wall", - support_DX=True, - support_DY=True, - support_DZ=True, - support_RY=True, - support_RX=True, - support_RZ=True, - ) - self.st.frame.add_node_load("free", "FX", 3000, case="gravity") - - self.st.run(check_statics=True) - - def test_beam(self): - self.subtest_assert_near(self.bm.A, 53.4 / (100**2)) - self.subtest_assert_near(self.bm.Ix, 8196 / (100**4)) - self.subtest_assert_near(self.bm.Iy, 388.8 / (100**4)) - self.subtest_assert_near(self.bm.section_mass, 41.9) - - self.subtest_assert_near(self.bm.max_von_mises(), 27.4e6) - self.subtest_assert_near(float(self.bm.data_dict["min_deflection_y"]), -0.0076) - # self.subtest_assert_near(float(self.bm.data_dict["max_shear_y"]), 3000) - self.subtest_assert_near(float(self.bm.data_dict["max_shear_y"]), 3000) - - df = self.st.node_dataframe.loc["gravity"] - - dfw = df.loc["wall"] - dff = df.loc["free"] - - self.subtest_assert_near(float(dfw["rxfx"]), -3000, msg="wall rxfx") - self.subtest_assert_near(float(dfw["rxmz"]), 15000, msg="wall rxmz") - self.subtest_assert_near(float(dfw["dx"]), 0, msg="wall dx") - - self.subtest_assert_near(float(dff["dx"]), 0.0076, msg="dx") - self.subtest_assert_near(float(dff["rxfx"]), 0, msg="rxfx") - self.subtest_assert_near(float(dff["rxmz"]), 0, msg="rxmz") - - stress_obj = self.bm.get_stress_at(0, "gravity") - # stress_obj.plot_stress_vm() - - def subtest_assert_near(self, value, truth, pct=0.025, **kw): - with self.subTest(**kw): - self.assertAlmostEqual(value, truth, delta=max(abs(truth * pct), abs(pct))) - - -class test_truss(unittest.TestCase): - # Match this example, no beam stresses - # https://engineeringlibrary.org/reference/trusses-air-force-stress-manual - - def setUp(self): - self.st = Structure(name="truss", add_gravity_force=False) - self.st.add_node("A", 0, 0, 0) - self.st.add_node("B", 15, 30 * sqrt(3) / 2, 0) - self.st.add_node("C", 45, 30 * sqrt(3) / 2, 0) - self.st.add_node("D", 75, 30 * sqrt(3) / 2, 0) - self.st.add_node("E", 90, 0, 0) - self.st.add_node("F", 60, 0, 0) - self.st.add_node("G", 30, 0, 0) - - pairs = set() - Lmin = 30 * sqrt(3) / 2 - Lmax = 30.1 - - for n1 in self.st.nodes.values(): - for n2 in self.st.nodes.values(): - L = numpy.sqrt( - (n1.X - n2.X) ** 2.0 + (n1.Y - n2.Y) ** 2.0 + (n1.Z - n2.Z) ** 2.0 - ) - - if ( - L >= Lmin - and L <= Lmax - and (n1.name, n2.name) not in pairs - and (n2.name, n1.name) not in pairs - ): - pairs.add((n1.name, n2.name)) - - elif (n1.name, n2.name) in pairs or ( - n2.name, - n1.name, - ) in pairs: - pass - - self.beam = sections.rectangular_section(1.5, 1.5) - material = ANSI_4130() - self.section = ShapelySection(shape=self.beam, material=material) - - constrained = ("A", "E") - for n1, n2 in pairs: - bkey = f"{n1}_{n2}" - self.bm = self.st.add_member( - bkey, - n1, - n2, - # material=ANSI_4130(), - section=self.section, - # min_mesh_size=0.01, - ) - - # if n1 not in constrained: - # print(f'releasing {bkey}') - # self.st.frame.DefineReleases(bkey, Rzi=True) - - # if n2 not in constrained: - # print(f'releasing {bkey} J') - # self.st.frame.DefineReleases(bkey, Rzj=True) - - self.st.def_support( - "A", - support_DX=True, - support_DY=True, - support_DZ=True, - support_RY=True, - support_RX=True, - support_RZ=True, - ) - self.st.def_support( - "E", - support_DX=False, - support_DY=True, - support_DZ=True, - support_RY=False, - support_RX=False, - support_RZ=True, - ) - # for node in self.st.nodes: - # self.st.frame.Definesupport_(node,support_DZ=True,support_RZ=True) - - self.st.add_node_load("F", "FY", -1000, case="gravity") - self.st.add_node_load("G", "FY", -2000, case="gravity") - - self.st.run(check_statics=True) - # self.st.visulize() - - def test_reactions(self): - df = self.st.node_dataframe.loc["gravity"] - # print(df) - - dfa = df.loc["A"] - dfe = df.loc["E"] - - # print(dfa) - # print(dfe) - - self.subtest_assert_near(float(dfa["rxfy"]), 1667) - self.subtest_assert_near(float(dfe["rxfy"]), 1333) - - self.subtest_member("A", "B", "max_axial", 1925) - self.subtest_member("A", "G", "max_axial", -949) # -926 textbook - self.subtest_member("B", "C", "max_axial", 1925) - self.subtest_member("B", "G", "max_axial", -1925) - self.subtest_member("C", "D", "max_axial", 1541) - self.subtest_member("F", "G", "max_axial", -1734) - self.subtest_member("C", "F", "max_axial", 382) - self.subtest_member("C", "G", "max_axial", -382) - self.subtest_member("C", "G", "max_axial", -378) # 1541 textbook - self.subtest_member("D", "F", "max_axial", -1541) - self.subtest_member("E", "F", "max_axial", -770) - - # Visualization.RenderModel( self.st.frame ) - - def subtest_member(self, nodea, nodeb, result_key, truth, pct=0.025): - with self.subTest(msg=f"test {nodea}->{nodeb}"): - key_1 = f"{nodea}_{nodeb}" - key_2 = f"{nodeb}_{nodea}" - - if key_1 in self.st.beams: - key = key_1 - elif key_2 in self.st.beams: - key = key_2 - else: - raise - - value = self.get_member_result(nodea, nodeb, result_key) - dopasst = abs(value - truth) <= abs(truth) * pct - - if not dopasst: - print(f"fails {key} {result_key}| {value:3.5f} == {truth:3.5f}?") - self.subtest_assert_near(value, truth, pct=pct) - - def subtest_assert_near(self, value, truth, pct=0.025): - with self.subTest(): - self.assertAlmostEqual(value, truth, delta=abs(truth * pct)) - - def get_member_result(self, nodea, nodeb, result_key): - key_1 = f"{nodea}_{nodeb}" - key_2 = f"{nodeb}_{nodea}" - - if key_1 in self.st.beams: - mem = self.st.beams[key_1] - if result_key in mem.data_dict: - return float(mem.data_dict[result_key]) - - elif key_2 in self.st.beams: - mem = self.st.beams[key_2] - if result_key in mem.data_dict: - return float(mem.data_dict[result_key]) - - return numpy.nan # shouod fail, nan is not comparable From 62f2ac7bfbc722b05bee96159c547a687c2c336b Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Sun, 6 Oct 2024 15:38:15 -0400 Subject: [PATCH 07/15] Improve cost summary --- engforge/dataframe.py | 10 +- engforge/eng/costs.py | 259 ++++++++++++++++++++++++------------ engforge/problem_context.py | 2 +- 3 files changed, 180 insertions(+), 91 deletions(-) diff --git a/engforge/dataframe.py b/engforge/dataframe.py index 3db1672..e538a7b 100644 --- a/engforge/dataframe.py +++ b/engforge/dataframe.py @@ -107,8 +107,8 @@ def split_dataframe(df: pandas.DataFrame) -> tuple: class DataframeMixin: dataframe: pandas.DataFrame - _split_dataframe_func = split_dataframe - _determine_split_func = determine_split + _split_dataframe_func = staticmethod(split_dataframe) + _determine_split_func = staticmethod(determine_split) def smart_split_dataframe(self, df=None, split_groups=0, key_f=key_func): """splits dataframe between constant values and variants""" @@ -166,10 +166,8 @@ def dataframe_variants(self): return o def format_columns(self, dataframe: pandas.DataFrame): - #replace(".", "_") - dataframe.rename( - lambda x: x.lower(), axis="columns", inplace=True - ) + # replace(".", "_") + dataframe.rename(lambda x: x.lower(), axis="columns", inplace=True) # Plotting Interface @property diff --git a/engforge/eng/costs.py b/engforge/eng/costs.py index 90adb18..1416f42 100644 --- a/engforge/eng/costs.py +++ b/engforge/eng/costs.py @@ -55,6 +55,7 @@ class Parent(System,CostModel) import pandas import collections + class CostLog(LoggingMixin): pass @@ -73,6 +74,14 @@ class CostLog(LoggingMixin): COST_CATEGORIES = set(("misc",)) +def get_num_from_cost_prop(ref): + """analyzes the reference and returns the number of items""" + if isinstance(ref,(float,int)): + return ref + co = getattr(ref.comp.__class__,ref.key,None) + return co.get_num_items(ref.comp) + + class cost_property(system_property): """A thin wrapper over `system_property` that will be accounted by `Economics` Components and apply term & categorization @@ -86,6 +95,7 @@ class cost_property(system_property): Categories are a way to report cost categories and multiple can be applied to a cost. Categories are grouped by the Economics system at reported in bulk by term and over the term_length """ + valild_types = (int, float) cost_categories: list = None term_mode: str = None @@ -112,7 +122,7 @@ def __init__( """ super().__init__(fget, fset, fdel, doc, desc, label, stochastic) - self.valild_types = (int, str) #only numerics + self.valild_types = (int, str) # only numerics if isinstance(mode, str): mode = mode.lower() assert ( @@ -126,7 +136,7 @@ def __init__( else: raise ValueError(f"mode: {mode} must be cost term str or callable") - #cost categories + # cost categories if category is not None: if isinstance(category, str): self.cost_categories = category.split(",") @@ -139,7 +149,7 @@ def __init__( else: self.cost_categories = ["misc"] - #number of items override + # number of items override if num_items is not None: self.num_items = num_items @@ -161,20 +171,25 @@ def get_func_return(self, func): else: self.return_type = float + def get_num_items(self,obj): + """applies the num_items override or the costmodel default if not set""" + if self.num_items is not None: + k = self.num_items + else: + k = obj.num_items if isinstance(obj, CostModel) else 1 + return k def __get__(self, obj, objtype=None): if obj is None: return self # class support if self.fget is None: raise AttributeError("unreadable attribute") - - #apply the costmodel with the item multiplier - if self.num_items is not None: - k = self.num_items - else: - k = obj.num_items if isinstance(obj, CostModel) else 1 + + # apply the costmodel with the item multiplier + k = self.get_num_items(obj) return k * self.fget(obj) + @forge class CostModel(Configuration, TabulationMixin): """CostModel is a mixin for components or systems that reports its costs through the `cost` system property, which by default sums the `item_cost` and `sub_items_cost`. @@ -184,13 +199,13 @@ class CostModel(Configuration, TabulationMixin): `sub_items_cost` system_property summarizes the costs of any component in a Slot that has a `CostModel` or for SlotS which CostModel.declare_cost(`slot`,default=numeric|CostModelInst|dict[str,float]) """ - #TODO: remove "default costs" concept and just use cost_properties since thats declarative and doesn't create a "phantom" representation to maintain - #TODO: it might be a good idea to add a "castable" namespace for components so they can all reference a common dictionary. Maybe all problem variables are merged into a single namespace to solve issues of "duplication" + # TODO: remove "default costs" concept and just use cost_properties since thats declarative and doesn't create a "phantom" representation to maintain + # TODO: it might be a good idea to add a "castable" namespace for components so they can all reference a common dictionary. Maybe all problem variables are merged into a single namespace to solve issues of "duplication" _slot_costs: dict # TODO: insantiate per class - #basic attribute interface returns the item cost as `N x cost_per_item`` + # basic attribute interface returns the item cost as `N x cost_per_item`` cost_per_item: float = attrs.field(default=numpy.nan) - num_items: int = attrs.field(default=1) #set to 0 to disable the item cost + num_items: int = attrs.field(default=1) # set to 0 to disable the item cost def __on_init__(self): self.set_default_costs() @@ -206,7 +221,9 @@ def update_dflt_costs(self): for k, v in self._slot_costs.items(): # Check if the cost model will be accessed no_comp = k not in current_comps - is_cost = not no_comp and isinstance(current_comps[k], CostModel) + is_cost = not no_comp and isinstance( + current_comps[k], CostModel + ) dflt_is_cost_comp = all( [isinstance(v, CostModel), isinstance(v, Component)] ) @@ -235,7 +252,9 @@ def set_default_costs(self): @classmethod def subcls_compile(cls): - assert not issubclass(cls, ComponentIter), "component iter not supported" + assert not issubclass( + cls, ComponentIter + ), "component iter not supported" log.debug(f"compiling costs {cls}") cls.reset_cls_costs() @@ -254,7 +273,9 @@ def default_cost( assert not isinstance( cost, type ), f"insantiate classes before adding as a cost!" - assert slot_name in cls.slots_attributes(), f"slot {slot_name} doesnt exist" + assert ( + slot_name in cls.slots_attributes() + ), f"slot {slot_name} doesnt exist" assert isinstance(cost, (float, int, dict)) or isinstance( cost, CostModel ), "only numeric types or CostModel instances supported" @@ -264,7 +285,9 @@ def default_cost( if warn_on_non_costmodel and not any( [issubclass(at, CostModel) for at in atypes] ): - log.warning(f"assigning cost to non CostModel based slot {slot_name}") + log.warning( + f"assigning cost to non CostModel based slot {slot_name}" + ) cls._slot_costs[slot_name] = cost @@ -280,7 +303,9 @@ def custom_cost( assert not isinstance( cost, type ), f"insantiate classes before adding as a cost!" - assert slot_name in self.slots_attributes(), f"slot {slot_name} doesnt exist" + assert ( + slot_name in self.slots_attributes() + ), f"slot {slot_name} doesnt exist" assert isinstance(cost, (float, int, dict)) or isinstance( cost, CostModel ), "only numeric types or CostModel instances supported" @@ -290,7 +315,9 @@ def custom_cost( if warn_on_non_costmodel and not any( [issubclass(at, CostModel) for at in atypes] ): - self.warning(f"assigning cost to non CostModel based slot {slot_name}") + self.warning( + f"assigning cost to non CostModel based slot {slot_name}" + ) # convert from classinfo if self._slot_costs is self.__class__._slot_costs: @@ -308,7 +335,7 @@ def custom_cost( def calculate_item_cost(self) -> float: """override this with a parametric model related to this systems attributes and properties""" - return self.num_items*self.cost_per_item + return self.num_items * self.cost_per_item @system_property def sub_items_cost(self) -> float: @@ -342,7 +369,9 @@ def sum_costs(self, saved: set = None, categories: tuple = None, term=0): saved = set((self,)) # item cost included! elif self not in saved: saved.add(self) - itemcst = list(self.dict_itemized_costs(saved, categories, term).values()) + itemcst = list( + self.dict_itemized_costs(saved, categories, term).values() + ) csts = [self.sub_costs(saved, categories, term), numpy.nansum(itemcst)] return numpy.nansum(csts) @@ -351,7 +380,11 @@ def dict_itemized_costs( ) -> dict: ccp = self.class_cost_properties() costs = { - k: obj.__get__(self) if obj.apply_at_term(self, term) == test_val else 0 + k: ( + obj.__get__(self) + if obj.apply_at_term(self, term) == test_val + else 0 + ) for k, obj in ccp.items() if categories is None or any([cc in categories for cc in obj.cost_categories]) @@ -421,7 +454,11 @@ def costs_at_term(self, term: int, test_val=True) -> dict: function returns False at that term""" ccp = self.class_cost_properties() return { - k: obj.__get__(self) if obj.apply_at_term(self, term) == test_val else 0 + k: ( + obj.__get__(self) + if obj.apply_at_term(self, term) == test_val + else 0 + ) for k, obj in ccp.items() } @@ -462,8 +499,6 @@ def all_categories(self): return COST_CATEGORIES - - cost_type = typing.Union[float, int, CostModel, dict] @@ -492,6 +527,7 @@ def gend(deect: dict): else: yield k, v + parent_types = typing.Union[Component, "System"] @@ -571,35 +607,35 @@ def update(self, parent: parent_types): self._anything_changed = True - #TODO: expand this... + # TODO: expand this... @solver_cached def econ_output(self): return self.lifecycle_output - @system_property(label='cost/output') - def levelized_cost(self)->float: + @system_property(label="cost/output") + def levelized_cost(self) -> float: """Price per kwh""" eco = self.econ_output - return eco['summary.levelized_cost'] - + return eco["summary.levelized_cost"] + @system_property - def total_cost(self)->float: + def total_cost(self) -> float: eco = self.econ_output - return eco['summary.total_cost'] + return eco["summary.total_cost"] @system_property - def levelized_output(self)->float: + def levelized_output(self) -> float: """ouput per dollar (KW/$)""" eco = self.econ_output - return eco['summary.levelized_output'] + return eco["summary.levelized_output"] @system_property - def term_years(self)->float: + def term_years(self) -> float: """ouput per dollar (KW/$)""" eco = self.econ_output - return eco['summary.years'] + return eco["summary.years"] - #Calculate Output + # Calculate Output def calculate_production(self, parent, term) -> float: """must override this function and set economic_output""" return numpy.nansum([0, self.fixed_output]) @@ -608,8 +644,8 @@ def calculate_costs(self, parent) -> float: """recursively accounts for costs in the parent, its children recursively.""" return self.sum_cost_references() - #Reference Utilitly Functions - #Set Costs over time "flat" through ref trickery... + # Reference Utilitly Functions + # Set Costs over time "flat" through ref trickery... def sum_cost_references(self): cst = 0 for k, v in self._cost_references.items(): @@ -639,18 +675,22 @@ def get_prop(self, ref): def term_fgen(self, comp, prop): if isinstance(comp, dict): return lambda term: comp[prop] if term == 0 else 0 - return lambda term: prop.__get__(comp) if prop.apply_at_term(comp, term) else 0 + return lambda term: ( + prop.__get__(comp) if prop.apply_at_term(comp, term) else 0 + ) def sum_term_fgen(self, ref_group): - term_funs = [self.term_fgen(ref.comp, self.get_prop(ref)) for ref in ref_group] + term_funs = [ + self.term_fgen(ref.comp, self.get_prop(ref)) for ref in ref_group + ] return lambda term: numpy.nansum([t(term) for t in term_funs]) # Gather & Set References (the magic!) # TODO: update internal_references callback to problem def internal_references(self, recache=True, numeric_only=False): """standard component references are""" - - recache=True #override + + recache = True # override d = self._gather_references() self._create_term_eval_functions() @@ -690,7 +730,7 @@ def internal_references(self, recache=True, numeric_only=False): def cost_summary(self, annualized=False, do_print=True, ignore_zero=True): """ - Generate a summary of costs, optionally annualized, and print the summary. + Generate a summary of costs, optionally annualized, and optionally print the summary. :param annualized: If True, include only annualized costs in the summary. Default is False. :type annualized: bool :param do_print: If True, print the summary to the console. Default is True. @@ -700,52 +740,81 @@ def cost_summary(self, annualized=False, do_print=True, ignore_zero=True): :return: A dictionary containing component costs, skipped costs, and summary. :rtype: dict """ - + dct = self.lifecycle_output cols = list(dct.keys()) skippd = set() comp_costs = collections.defaultdict(dict) + comp_nums = collections.defaultdict(dict) summary = {} - costs = {'comps':comp_costs,'skip':skippd,'summary':summary} + costs = { + "comps": comp_costs, + "skip": skippd, + "summary": summary, + } def abriv(val): - if val > 1E6: - return f'{val/1E6:^12.4f}M' - elif val > 1E3: - return f'{val/1E3:^12.2f}k' - return f'{val:^12.2f}' - + if val > 1e6: + return f"{val/1E6:>12.4f} M" + elif val > 1e3: + return f"{val/1E3:>12.2f} k" + return f"{val:>12.2f}" + for col in cols: - is_ann = '.annualized.' in col + is_ann = ".annualized." in col val = dct[col] - - #handle no value case + + # handle no value case if val == 0 and ignore_zero: continue - if '.cost.' in col and is_ann == annualized: - base,cst = col.split('.cost.') - comp_costs[base][cst]= val - elif col.startswith('summary.'): - summary[col.replace('summary.','')]=val - + if ".cost." in col and is_ann == annualized: + base, cst = col.split(".cost.") + ckey = f"{base.replace('lifecycle.','')}.cost.{cst}" + #print(ckey,str(self._comp_costs.keys())) + comp_costs[base][cst] = val + comp_nums[base][cst] = get_num_from_cost_prop(self._comp_costs[ckey]) + elif col.startswith("summary."): + summary[col.replace("summary.", "")] = val + total_cost = sum([sum(list(cc.values())) for cc in comp_costs.values()]) - + + # provide consistent format + hdr = "{key:<32}|\t{value:12.10f}" + fmt = "{key:<32}|\t{fmt:<24} | {total:^12} | {pct:3.0f}%" + title = f'COST SUMMARY: {self.parent.identity}' if do_print: + self.info('#'*80) + self.info(f'{title:^80}') self.info('_'*80) + #summary items for key,val in summary.items(): - self.info(f' {key:<35}| {val:^12.10f}') - self.info('-'*80) - - for base,items in comp_costs.items(): + self.info(hdr.format(key=key,value=val)) + itcst = '{val:>24}'.format(val='TOTAL----->') + self.info('='*80) + self.info(fmt.format(key='COMBINED',fmt=itcst,total=abriv(total_cost),pct=100)) + self.info('-'*80) + #itemization + sgroups = lambda kv:sum(list(kv[-1].values())) + for base,items in sorted(comp_costs.items(),key=sgroups,reverse=True): if (subtot:=sum(list(items.values()))) > 0: - self.info(f' {base:<35}| total ---> {abriv(subtot)} | {subtot*100/total_cost:3.0f}%') - for key,val in sorted(items.items(),lambda kv:[0]): - self.info(f' \t{key:<32}|{abriv(val)} | {val*100/total_cost:^3.0f}%') - self.info('-'*80) - self.info('_'*80) - return costs + #self.info(f' {base:<35}| total ---> {abriv(subtot)} | {subtot*100/total_cost:3.0f}%') + pct = subtot*100/total_cost + itcst = '{val:>24}'.format(val='TOTAL----->') + #todo; add number of items for cost comp + self.info(fmt.format(key=base,fmt=itcst,total=abriv(subtot),pct=pct)) + #Sort costs by value + for key,val in sorted(items.items(),key=lambda kv:kv[-1],reverse=True): + #self.info(f' \t{key:<32}|{abriv(val)} | {val*100/total_cost:^3.0f}%') + tot = abriv(val) + pct = val*100/total_cost + num = comp_nums[base][key] + itcst = f'{abriv(val/num):^18} x {num:3.0f}' if num != 0 else '0' + self.info(fmt.format(key='-'+key,fmt=itcst,total=tot,pct=pct)) + self.info('-'*80) #section break + self.info('#'*80) + return costs @property def lifecycle_output(self) -> dict: @@ -804,7 +873,9 @@ def lifecycle_dataframe(self) -> pandas.DataFrame: ) row["levelized_cost"] = tc * (1 + self.discount_rate) ** (-1 * t) row["output"] = output = self.calculate_production(self.parent, t) - row["levelized_output"] = output * (1 + self.discount_rate) ** (-1 * t) + row["levelized_output"] = output * (1 + self.discount_rate) ** ( + -1 * t + ) return pandas.DataFrame(out) @@ -836,11 +907,14 @@ def _gather_cost_references(self, parent: "System"): comp_set = set() # reset data + # groupings of components, categories, and pairs of components and categories self._cost_categories = collections.defaultdict(list) self._comp_categories = collections.defaultdict(list) self._comp_costs = dict() - for key, level, conf in parent.go_through_configurations(check_config=False): + for key, level, conf in parent.go_through_configurations( + check_config=False + ): # skip self if conf is self: continue @@ -861,7 +935,7 @@ def _gather_cost_references(self, parent: "System"): self.debug(f"checking {key} {comp_key} {kbase}") - #0. Get Costs Directly From the cost model instance + # 0. Get Costs Directly From the cost model instance if isinstance(conf, CostModel): comps[key] = conf self.debug(f"adding cost model for {kbase}.{comp_key}") @@ -881,7 +955,9 @@ def _gather_cost_references(self, parent: "System"): compcanidate = child._slot_costs[comp_key] if isinstance(compcanidate, CostModel): self.debug(f"dflt child costmodel {kbase}.{comp_key}") - self._extract_cost_references(compcanidate, bse + "cost.") + self._extract_cost_references( + compcanidate, bse + "cost." + ) else: _key = bse + "cost.item_cost" self.debug(f"dflt child cost for {kbase}.{comp_key}") @@ -895,7 +971,9 @@ def _gather_cost_references(self, parent: "System"): cc = "unit" self._comp_costs[_key] = ref self._cost_categories["category." + cc].append(ref) - self._comp_categories[bse + "category." + cc].append(ref) + self._comp_categories[bse + "category." + cc].append( + ref + ) # 2. try looking at the parent elif ( @@ -960,6 +1038,7 @@ def _extract_cost_references(self, conf: "CostModel", bse: str): lvl=5, ) # add slot costs with current items (skip class defaults) + # TODO: remove defaults costs for slot_name, slot_value in conf._slot_costs.items(): # Skip items that are internal components if slot_name in comps_act: @@ -987,25 +1066,35 @@ def _extract_cost_references(self, conf: "CostModel", bse: str): elif _key in CST: self.debug(f"skipping key {_key}") - # add base class slot values when comp was none - for compnm, comp in conf.internal_configurations(False, none_ok=True).items(): + # add base class slot values when comp was none (recursively) + for compnm, comp in conf.internal_configurations( + False, none_ok=True + ).items(): if comp is None: if self.log_level < 5: self.msg( f"{conf} looking up base class costs for {compnm}", lvl=5, ) - comp_cls = conf.slots_attributes(attr_type=True)[compnm].accepted + comp_cls = conf.slots_attributes(attr_type=True)[ + compnm + ].accepted for cc in comp_cls: if issubclass(cc, CostModel): if cc._slot_costs: if self.log_level < 5: - self.msg(f"{conf} looking up base slot cost for {cc}") + self.msg( + f"{conf} looking up base slot cost for {cc}" + ) for k, v in cc._slot_costs.items(): - _key = bse + compnm + "." + k + ".cost.item_cost" + _key = ( + bse + compnm + "." + k + ".cost.item_cost" + ) if _key in CST: if self.log_level < 10: - self.debug(f"{conf} skipping dflt key {_key}") + self.debug( + f"{conf} skipping dflt key {_key}" + ) # break #skip if already added continue @@ -1028,7 +1117,9 @@ def _extract_cost_references(self, conf: "CostModel", bse: str): cc = "unit" self._comp_costs[_key] = ref - self._cost_categories["category." + cc].append(ref) + self._cost_categories[ + "category." + cc + ].append(ref) self._comp_categories[ bse + "category." + cc ].append(ref) diff --git a/engforge/problem_context.py b/engforge/problem_context.py index 7f9b51c..1805b62 100644 --- a/engforge/problem_context.py +++ b/engforge/problem_context.py @@ -245,7 +245,7 @@ class ProblemExec: _converged: bool # Interior Context Options - enter_refresh: bool = True #TODO: allow this off with event system + enter_refresh: bool = True #TODO: allow this off (or lower impact) with event system save_on_exit: bool = False save_mode: str = "all" level_name: str = None # target this context with the level name From cec6c506f399ebdc89eec0e33c3d7026a859a5ec Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Sun, 13 Oct 2024 13:05:04 -0400 Subject: [PATCH 08/15] Improved printing & Diagnostics: - print all problem vars with filter for keys and comps - did you mean error on run input --- engforge.egg-info/PKG-INFO | 2 +- engforge/dataframe.py | 4 ++- engforge/eng/costs.py | 3 +- engforge/eng/geometry.py | 2 +- engforge/problem_context.py | 53 +++++++++++++++++++++++++++++++----- engforge/solveable.py | 6 ++++ engforge/solver.py | 2 +- engforge/system_reference.py | 25 ++++++++++++----- engforge/tabulation.py | 3 +- setup.py | 2 +- 10 files changed, 81 insertions(+), 21 deletions(-) diff --git a/engforge.egg-info/PKG-INFO b/engforge.egg-info/PKG-INFO index 694e267..2ad2f47 100644 --- a/engforge.egg-info/PKG-INFO +++ b/engforge.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: engforge -Version: 0.0.9 +Version: 0.1.0 Summary: The Engineer's Framework Home-page: https://github.com/SoundsSerious/engforge Author: Kevin russell diff --git a/engforge/dataframe.py b/engforge/dataframe.py index e538a7b..d70735c 100644 --- a/engforge/dataframe.py +++ b/engforge/dataframe.py @@ -32,9 +32,11 @@ class DataFrameLog(LoggingMixin): # Dataframe interrogation functions def is_uniform(s: pandas.Series): a = s.to_numpy() # s.values (pandas<0.24) - if (a[0] == a).all(): + + if numpy.all(a == a[0]): return True try: + # if not numpy.isfinite(a).any(): return True except: diff --git a/engforge/eng/costs.py b/engforge/eng/costs.py index 1416f42..e3e3bd7 100644 --- a/engforge/eng/costs.py +++ b/engforge/eng/costs.py @@ -349,6 +349,7 @@ def item_cost(self) -> float: @system_property def combine_cost(self) -> float: + """the sum of all cost properties""" return self.sum_costs() @system_property @@ -782,7 +783,7 @@ def abriv(val): # provide consistent format hdr = "{key:<32}|\t{value:12.10f}" - fmt = "{key:<32}|\t{fmt:<24} | {total:^12} | {pct:3.0f}%" + fmt = "{key:<32}|\t{fmt:<24} | {total:^12} | {pct:3.3f}%" title = f'COST SUMMARY: {self.parent.identity}' if do_print: self.info('#'*80) diff --git a/engforge/eng/geometry.py b/engforge/eng/geometry.py index 98b9de7..55ab469 100644 --- a/engforge/eng/geometry.py +++ b/engforge/eng/geometry.py @@ -622,7 +622,7 @@ def mesh_section(self): self.calculate_bounds() def calculate_bounds(self): - self.info(f"calculating shape bounds!") + self.debug(f"calculating shape bounds!") xcg, ycg = self._geo.calculate_centroid() minx, maxx, miny, maxy = self._geo.calculate_extents() self.y_bounds = (miny - ycg, maxy - ycg) diff --git a/engforge/problem_context.py b/engforge/problem_context.py index 1805b62..2869a17 100644 --- a/engforge/problem_context.py +++ b/engforge/problem_context.py @@ -661,6 +661,38 @@ def min_refresh(self, sesh=None): cons = {} #TODO: parse additional constraints sesh.constraints = sesh.sys_solver_constraints(cons) + def print_all_info(self,keys:str=None,comps:str=None): + """ + Print all the information of each component's dictionary. + Parameters: + key_sch (str, optional): A pattern to match dictionary keys. Only keys matching this pattern will be included in the output. + comps (list, optional): A list of component sys names to filter. Only information of these components will be printed. + Returns: None (except stdout :) + """ + from pprint import pprint + keys = keys.split(',') + comps = (comps+',').split(',') #always top level + print(f'CONTEXT: {self}') + + mtch = lambda key,ptrns: any([fnmatch.fnmatch(key.lower(),ptn.lower()) for ptn in ptrns]) + + #check your comps + itrs = self.all_comps.copy() + itrs[''] = Ref(self.system,'',True,False) + + #check your comps + for cn,comp in itrs.items(): + if comps is not None and not mtch(cn,comps): + continue + + dct = comp.value().as_dict + if keys: #filter keys + dct = {k:v for k,v in dct.items() if mtch(k,keys)} + if dct: + print(f'INFO: {cn if cn else ""}') + pprint(dct) + print('-'*80) + @property def check_dynamics(self): sesh = self.sesh @@ -1086,7 +1118,7 @@ def integral_rate(self, t, x, dt, Xss=None, Yobj=None, **kw): self.info( f'exiting solver {t} {ss_out["Xans"]} {ss_out["Xstart"]}' ) - pbx.set_ref_values(ss_out["Xans"]) + pbx.set_ref_values(ss_out["Xans"],scope='intgrl') pbx.exit_to_level("ss_slvr", False) else: self.warning( @@ -1191,7 +1223,7 @@ def handle_solution(self, answer, Xref, Yref, output): # Output Results Xa = {p: answer.x[i] for i, p in enumerate(vars)} output["Xans"] = Xa - Ref.refset_input(Xref, Xa) + Ref.refset_input(Xref, Xa,scope='solvd') Yout = {p: yit.value(yit.comp, self) for p, yit in Yref.items()} output["Yobj"] = Yout @@ -1680,13 +1712,13 @@ def get_ref_values(self, refs=None): refs = sesh.all_system_references return Ref.refset_get(refs, sys=self.system, prob=self) - def set_ref_values(self, values, refs=None): + def set_ref_values(self, values, refs=None,scope='sref'): """returns the values of the refs""" # TODO: add checks for the refs if refs is None: sesh = self.sesh refs = sesh.all_comps_and_vars - return Ref.refset_input(refs, values) + return Ref.refset_input(refs, values,scope=scope) def change_sys_var(self, key, value, refs=None, doset=True, attr_key_map=None): """use this function to change the value of a system var and update the start state, multiple uses in the same context will not change the record preserving the start value @@ -1731,7 +1763,7 @@ def revert_to_start(self): rs = list(self.record_state.values()) self.debug(f"reverting to start: {xs} -> {rs}") # TODO: STRICT MODE Fail for refset_input - Ref.refset_input(sesh.all_comps_and_vars, self.x_start, fail=False) + Ref.refset_input(sesh.all_comps_and_vars, self.x_start, fail=False,scope='rvtst') def activate_temp_state(self, new_state=None): # TODO: determine when components change, and update refs accordingly! @@ -1740,11 +1772,11 @@ def activate_temp_state(self, new_state=None): if new_state: if self.log_level < 3: self.debug(f"new-state: {self.temp_state}") - Ref.refset_input(sesh.all_comps_and_vars, new_state, fail=False) + Ref.refset_input(sesh.all_comps_and_vars, new_state, fail=False,scope='ntemp') elif self.temp_state: if self.log_level < 3: self.debug(f"act-state: {self.temp_state}") - Ref.refset_input(sesh.all_comps_and_vars, self.temp_state, fail=False) + Ref.refset_input(sesh.all_comps_and_vars, self.temp_state, fail=False,scope='atemp') elif self.log_level < 3: self.debug(f"no-state: {new_state}") @@ -1994,6 +2026,13 @@ def is_active(self): """checks if the context has been entered and not exited""" return self.entered and not self.exited + @classmethod + def cls_is_active(cls): + """checks if the cache has a session""" + if cls.class_cache and hasattr(cls.class_cache, "session"): + return True + return False + @property def solveable(self): """checks the system's references to determine if its solveabl""" diff --git a/engforge/solveable.py b/engforge/solveable.py index 561ed2a..05fff7e 100644 --- a/engforge/solveable.py +++ b/engforge/solveable.py @@ -352,6 +352,7 @@ def _iterate_input_matrix( force_solve=False, return_results=False, method_kw: dict = None, + print_kw:dict=None, **kwargs, ): """applies a permutation of input vars for vars. runs the system instance by applying input to the system and its slot-components, ensuring that the targeted attributes actualy exist. @@ -361,6 +362,8 @@ def _iterate_input_matrix( :param sequence: a list of dictionaries that should be run in order per the outer-product of kwargs :param eval_kw: a dictionary of keyword arguments to pass to the eval function of each component by their name and a set of keyword args. Use this to set values in the component that are not inputs to the system. No iteration occurs upon these values, they are static and irrevertable :param sys_kw: a dictionary of keyword arguments to pass to the eval function of each system by their name and a set of keyword args. Use this to set values in the component that are not inputs to the system. No iteration occurs upon these values, they are static and irrevertable + :param print_kw: a dictionary of keyword arguments to pass to the print_all_info function of the current context + :param kwargs: inputs are run on a product basis asusming they correspond to actual scoped vars (system.var or system.slot.var) @@ -416,6 +419,9 @@ def _iterate_input_matrix( # Run The Method with inputs provisioned out = method(icur, eval_kw, sys_kw, cb=cb, **method_kw) + if print_kw and hasattr(self, "last_context") and self.last_context: + self.last_context.print_all_info(**print_kw) + if return_results: # store the output output[max(output) + 1 if output else 0] = out diff --git a/engforge/solver.py b/engforge/solver.py index 9a93d44..33057fa 100644 --- a/engforge/solver.py +++ b/engforge/solver.py @@ -284,7 +284,7 @@ def solver(self, enter_refresh=True, save_on_exit=True, **kw): # depending on the solver success, failure or no solution, we can exit the solver if has_ans and out["ans"] and out["ans"].success: # this is where you want to be! <<< - pbx.set_ref_values(out["Xans"]) + pbx.set_ref_values(out["Xans"],scope='solvr') pbx.exit_to_level("sys_slvr", False) elif has_ans and out["ans"] is None: diff --git a/engforge/system_reference.py b/engforge/system_reference.py index ec17780..70079cd 100644 --- a/engforge/system_reference.py +++ b/engforge/system_reference.py @@ -13,7 +13,7 @@ from contextlib import contextmanager from engforge.properties import * import copy - +from difflib import get_close_matches class RefLog(LoggingMixin): pass @@ -22,32 +22,39 @@ class RefLog(LoggingMixin): log = RefLog() -def refset_input(refs, delta_dict, chk=True, fail=True, warn=True): +def refset_input(refs, delta_dict, chk=True, fail=True, warn=True,scope='ref'): """change a set of refs with a dictionary of values. If chk is True k will be checked for membership in refs""" + keys = set(refs.keys()) for k, v in delta_dict.items(): if isinstance(k, Ref): k.set_value(v) continue memb = k in refs + #if a match or not checking go ahead. if not chk or memb: refs[k].set_value(v) + + #TODO: handle setting non-ref values such as dictionaries + elif fail and chk and not memb: - raise KeyError(f"key {k} not in refs {refs.keys()}") + close = get_close_matches(k, keys) + raise KeyError(f"{scope}| key {k} not in refs. did you mean {close}?") elif warn and chk and not memb: - log.warning(f"key {k} not in refs {refs.keys()}") + close = get_close_matches(k, keys) + log.warning(f"{scope}| key {k} not in refs. did you mean {close}") def refset_get(refs, *args, **kw): out = {} + scope = kw.get('scope','ref') for k in refs: try: # print(k,refs[k]) - out[k] = refs[k].value(refs[k].comp, **kw) except Exception as e: rf = refs[k] - log.error(e, f"issue with ref: {rf}|{rf.key}|{rf.comp}") + log.error(e, f"{scope}| issue with ref: {rf}|{rf.key}|{rf.comp}") return out @@ -196,6 +203,7 @@ def set(self, comp, key, use_call=True, allow_set=True, eval_f=None): self.hxd = str(hex(id(self)))[-6:] + #TODO: update with change (pyee?) self.setup_calls() def setup_calls(self): @@ -215,7 +223,10 @@ def setup_calls(self): self._value_eval = lambda *a, **kw: self.key(*a, **kw) else: # do not cross reference vars! - if self.use_dict: + #TODO: allo for comp/key changes with events + if self.key == '': #return the component + p = lambda *a, **kw: self.comp + elif self.use_dict: p = lambda *a, **kw: self.comp.get(self.key) elif self.key in self.comp.__dict__: p = lambda *a, **kw: self.comp.__dict__[self.key] diff --git a/engforge/tabulation.py b/engforge/tabulation.py index 08012b0..b22fe97 100644 --- a/engforge/tabulation.py +++ b/engforge/tabulation.py @@ -268,7 +268,8 @@ def system_properties_classdef(cls, recache=False): prop = getattr(cls, k, None) if prop and isinstance(prop, system_property): __system_properties[k] = prop - log.msg(f"adding system property {mrv.__name__}.{k}") + if log.log_level <= 3: + log.msg(f"adding system property {mrv.__name__}.{k}") setattr(cls,cls_key, __system_properties) diff --git a/setup.py b/setup.py index a58fc6f..f1e61f2 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup import setuptools -__version__ = "0.0.9" +__version__ = "0.1.0" def parse_requirements(filename): From 170548cf91cd9be06f0a2c39af1eefdc93ce0061 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Mon, 14 Oct 2024 03:03:56 -0400 Subject: [PATCH 09/15] fix duplicate structures costs --- engforge/eng/costs.py | 2 +- engforge/eng/structure.py | 15 +++++++++++++-- engforge/eng/structure_beams.py | 5 +++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/engforge/eng/costs.py b/engforge/eng/costs.py index e3e3bd7..3e2854c 100644 --- a/engforge/eng/costs.py +++ b/engforge/eng/costs.py @@ -468,7 +468,7 @@ def class_cost_properties(cls) -> dict: """returns cost_property objects from this class & subclasses""" return { k: v - for k, v in cls.system_properties_classdef().items() + for k, v in cls.system_properties_classdef(True).items() if isinstance(v, cost_property) } diff --git a/engforge/eng/structure.py b/engforge/eng/structure.py index 440bc3b..1c4c513 100644 --- a/engforge/eng/structure.py +++ b/engforge/eng/structure.py @@ -835,6 +835,7 @@ def add_member_with( return beam # OUTPUT + def beam_dataframe(self, univ_parms: list = None, add_columns: list = None): """creates a dataframe entry for each beam and combo :param univ_parms: keys represent the dataframe parm, and the values represent the lookup value @@ -923,8 +924,18 @@ def structure_cost_panels(self) -> float: """sum of all panels cost""" return sum([sum(self.costs["quads"].values())]) - @cost_property(category="mfg,material,structure") - def structure_cost(self): + @cost_property(category="mfg,material,panels") + def panels_cost(self): + return self.structure_cost_panels + + @cost_property(category="mfg,material,beams") + def beam_cost(self) -> float: + """sum of all beams cost""" + return sum([bm.cost for bm in self.beams.values()]) + + @system_property + def structure_cost(self) -> float: + """sum of all beams and quad cost""" return self.structure_cost_beams + self.structure_cost_panels @solver_cached diff --git a/engforge/eng/structure_beams.py b/engforge/eng/structure_beams.py index fa5f411..c50e818 100644 --- a/engforge/eng/structure_beams.py +++ b/engforge/eng/structure_beams.py @@ -55,7 +55,7 @@ def rotation_matrix_from_vectors(vec1, vec2): @forge -class Beam(Component, CostModel): +class Beam(Component): """Beam is a wrapper for emergent useful properties of the structure""" # parent structure, will be in its _beams @@ -302,7 +302,8 @@ def section_mass(self) -> float: def mass(self) -> float: return self.material.density * self.Vol - @cost_property(category="mfg,material,beams") + #@system_property(category="mfg,material,beams") + @system_property def cost(self) -> float: return self.mass * self.material.cost_per_kg From 2713d7aeb66bde1c9acea60cccfa2e54b670fead Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Mon, 14 Oct 2024 19:07:56 -0400 Subject: [PATCH 10/15] accept update + cost update when both apply - update logging @ 15 --- engforge/solveable.py | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/engforge/solveable.py b/engforge/solveable.py index 05fff7e..967f08b 100644 --- a/engforge/solveable.py +++ b/engforge/solveable.py @@ -36,12 +36,12 @@ class SolvableLog(LoggingMixin): def _update_func(comp, eval_kw): def updt(*args, **kw): eval_kw.update(kw) - if log.log_level <= 5: - log.msg(f"update| {comp.name} ", lvl=5) + if log.log_level <= 12: + log.info(f"update| {comp.name} ")#=5) return comp.update(comp.parent, *args, **eval_kw) - if log.log_level <= 5: - log.msg(f"create method| {comp.name}| {eval_kw}") + if log.log_level <= 12: + log.info(f"create method| {comp.name}| {eval_kw}") updt.__name__ = f"{comp.name}_update" return updt @@ -51,8 +51,8 @@ def updt(*args, **kw): eval_kw.update(kw) return comp.post_update(comp.parent, *args, **eval_kw) - if log.log_level <= 5: - log.msg(f"create post method| {comp.name}| {eval_kw}") + if log.log_level <= 12: + log.info(f"create post method| {comp.name}| {eval_kw}") updt.__name__ = f"{comp.name}_post_update" return updt @@ -63,24 +63,25 @@ def _cost_update(comp): if isinstance(comp, Economics): def updt(*args, **kw): - if log.log_level <= 8: - log.debug(f"update economics {comp.name} | {comp.term_length} ") + if log.log_level <= 12: + log.info(f"update economics {comp.name} | {comp.term_length} ") comp.system_properties_classdef(True) comp.update(comp.parent, *args, **kw) - if log.log_level <= 8: - log.debug(f"economics update cb {comp.name} | {comp.term_length} ") + if log.log_level <= 12: + log.info(f"economics update cb {comp.name} | {comp.term_length} ") updt.__name__ = f"{comp.name}_econ_update" else: def updt(*args, **kw): - if log.log_level <= 7: - log.msg(f"update costs {comp.name} ", lvl=5) + if log.log_level <= 12: + log.info(f"update costs {comp.name} ")#=5) comp.system_properties_classdef(True) + #comp.update(comp.parent, *args, **kw) #called as update without cm return comp.update_dflt_costs() - if log.log_level <= 7: - log.debug(f"cost update cb {comp.name} ") + if log.log_level <= 12: + log.info(f"cost update cb {comp.name} ") updt.__name__ = f"{comp.name}_cost_update" return updt @@ -123,13 +124,13 @@ class SolveableMixin(AttributedBaseMixin): #'Configuration' # TODO: pass the problem vs the parent component, then locate this component in the problem and update any references def update(self, parent, *args, **kwargs): """Kwargs comes from eval_kw in solver""" - if log.log_level <= 5: - log.debug(f"void updating {self.__class__.__name__}.{self}") + if log.log_level <= 12: + log.info(f"void updating {self.__class__.__name__}.{self}") def post_update(self, parent, *args, **kwargs): """Kwargs comes from eval_kw in solver""" - if log.log_level <= 5: - log.debug(f"void post-updating {self.__class__.__name__}.{self}") + if log.log_level <= 12: + log.info(f"void post-updating {self.__class__.__name__}.{self}") def collect_update_refs(self, eval_kw=None, ignore=None): """checks all methods and creates ref's to execute them later""" @@ -146,6 +147,7 @@ def collect_update_refs(self, eval_kw=None, ignore=None): return for key, comp in self.internal_configurations(False).items(): + #for key,lvl,comp in self.go_through_configurations(check_config=False): if ignore is not None and comp in ignore: continue @@ -164,7 +166,7 @@ def collect_update_refs(self, eval_kw=None, ignore=None): ref = Ref(comp, _cost_update(comp)) updt_refs[key + "._cost_model_"] = ref - elif comp.__class__.update != SolveableMixin.update: + if comp.__class__.update != SolveableMixin.update: ref = Ref(comp, _update_func(comp, ekw)) updt_refs[key] = ref @@ -186,6 +188,7 @@ def collect_post_update_refs(self, eval_kw=None, ignore=None): return for key, comp in self.internal_configurations(False).items(): + #for key,lvl,comp in self.go_through_configurations(check_config=False): if ignore is not None and comp in ignore: continue From 1a3c1714508e7e829bb426ea04019ebc03efdea6 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Sun, 20 Oct 2024 21:49:15 -0400 Subject: [PATCH 11/15] Update Methods Include Top Comp - widen Option inputs --- engforge/datastores/data.py | 26 +++++++++++++++++++++----- engforge/solveable.py | 13 +++++++++++++ engforge/typing.py | 3 ++- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/engforge/datastores/data.py b/engforge/datastores/data.py index 07a69fc..fab6a14 100644 --- a/engforge/datastores/data.py +++ b/engforge/datastores/data.py @@ -195,10 +195,16 @@ class DiskCacheStore(LoggingMixin, metaclass=SingletonMeta): _current_keys = None expire_threshold = 60.0 - retries = 3 + retries = 1 sleep_time = 0.1 - def __init__(self, **kwargs): + proj_dir: str = None #avoid using implicit determinination + cache_path: str = None #override for implicit path, a recommended practice + + def __init__(self,root_path=None, **kwargs): + if root_path is not None: + self.cache_path = root_path + if kwargs: self.cache_init_kwargs = kwargs else: @@ -206,13 +212,23 @@ def __init__(self, **kwargs): self.info(f"Created DiskCacheStore In {self.cache_root}") self.cache + @property + def proj_root(self): + if self.proj_dir is not None: + return self.proj_dir + return client_path(skip_wsl=False) + @property def cache_root(self): # TODO: CHECK CACHE IS NOT SYNCED TO DROPBOX + if self.cache_path is not None: + return self.cache_path + if self.alt_path is not None: - return os.path.join(client_path(skip_wsl=False), "cache", self.alt_path) + return os.path.join(self.proj_root, "cache", self.alt_path) + return os.path.join( - client_path(skip_wsl=False), + self.proj_root, "cache", "{}".format(type(self).__name__).lower(), ) @@ -247,7 +263,7 @@ def set(self, key=None, data=None, retry=True, ttl=None, **kwargs): time.sleep(self.sleep_time * (self.retries - ttl)) return self.set(key=key, data=data, retry=True, ttl=ttl) else: - self.error(e, "Issue Getting Item From Cache") + self.error(e, "Issue Setting Item In Cache") # @ray_cache def get(self, key=None, on_missing=None, retry=True, ttl=None): diff --git a/engforge/solveable.py b/engforge/solveable.py index 967f08b..87b187a 100644 --- a/engforge/solveable.py +++ b/engforge/solveable.py @@ -145,6 +145,15 @@ def collect_update_refs(self, eval_kw=None, ignore=None): ignore = set() elif self in ignore: return + + key= 'top' + if self.__class__.update != SolveableMixin.update: + ref = Ref(self, _update_func(self, eval_kw if eval_kw else {})) + updt_refs[key] = ref + + if isinstance(self, (CostModel, Economics)): + ref = Ref(self, _cost_update(self)) + updt_refs[key + "._cost_model_"] = ref for key, comp in self.internal_configurations(False).items(): #for key,lvl,comp in self.go_through_configurations(check_config=False): @@ -186,6 +195,10 @@ def collect_post_update_refs(self, eval_kw=None, ignore=None): ignore = set() elif self in ignore: return + + if self.__class__.post_update != SolveableMixin.post_update: + ref = Ref(self, _post_update_func(self, eval_kw if eval_kw else {})) + updt_refs['top'] = ref for key, comp in self.internal_configurations(False).items(): #for key,lvl,comp in self.go_through_configurations(check_config=False): diff --git a/engforge/typing.py b/engforge/typing.py index 51fc816..e634f62 100644 --- a/engforge/typing.py +++ b/engforge/typing.py @@ -53,7 +53,8 @@ def Options(*choices, **kwargs): :param kwargs: keyword args passed to attrs field""" assert choices, f"must have some choices!" assert "type" not in kwargs, "options type set is str" - assert set([type(c) for c in choices]) == set((str,)), "choices must be str" + valids = set((str,int,float,bool,type(None))) + assert set([type(c) for c in choices]).issubset(valids), f"choices must be in {valids}" assert "on_setattr" not in kwargs validators = [attrs.validators.in_(choices)] From 7049df86ab6be65f5d1fcd8925d3dc2bcc038118 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Wed, 23 Oct 2024 17:39:37 -0400 Subject: [PATCH 12/15] Added End Term mode - redefined cost at term with econ input - accomodate this func redef with sane defaults (term=0 is initial cost) - allow net-negative cost sections to print in summary - printing of output + num-items and terms / ratinfo --- engforge/eng/costs.py | 175 ++++++++++++++++++++++++++++-------------- requirements.txt | 1 + 2 files changed, 120 insertions(+), 56 deletions(-) diff --git a/engforge/eng/costs.py b/engforge/eng/costs.py index 3e2854c..0969f90 100644 --- a/engforge/eng/costs.py +++ b/engforge/eng/costs.py @@ -65,9 +65,14 @@ class CostLog(LoggingMixin): # Cost Term Modes are a quick lookup for cost term support global COST_TERM_MODES, COST_CATEGORIES COST_TERM_MODES = { - "initial": lambda inst, term: True if term < 1 else False, - "maintenance": lambda inst, term: True if term >= 1 else False, - "always": lambda inst, term: True, + "initial": lambda inst, term, econ: True if term < 1 else False, + "maintenance": lambda inst, term, econ: True if term >= 1 else False, + "always": lambda inst, term, econ: True, + "end": lambda inst, term, econ: ( + True + if hasattr(econ, "term_length") and term == econ.term_length + else False + ), } category_type = typing.Union[str, list] @@ -76,9 +81,9 @@ class CostLog(LoggingMixin): def get_num_from_cost_prop(ref): """analyzes the reference and returns the number of items""" - if isinstance(ref,(float,int)): + if isinstance(ref, (float, int)): return ref - co = getattr(ref.comp.__class__,ref.key,None) + co = getattr(ref.comp.__class__, ref.key, None) return co.get_num_items(ref.comp) @@ -118,7 +123,7 @@ def __init__( num_items: int = None, ): """extends system_property interface with mode & category keywords - :param mode: can be one of `initial`,`maintenance`,`always` or a function with signature f(inst,term) as an integer and returning a boolean True if it is to be applied durring that term. + :param mode: can be one of `initial`,`maintenance`,`always` or a function with signature f(inst,term,econ) as an integer and returning a boolean True if it is to be applied durring that term. """ super().__init__(fget, fset, fdel, doc, desc, label, stochastic) @@ -153,10 +158,10 @@ def __init__( if num_items is not None: self.num_items = num_items - def apply_at_term(self, inst, term): + def apply_at_term(self, inst, term, econ=None): if term < 0: raise ValueError(f"negative term!") - if self.__class__._all_modes[self.term_mode](inst, term): + if self.__class__._all_modes[self.term_mode](inst, term, econ): return True return False @@ -171,14 +176,14 @@ def get_func_return(self, func): else: self.return_type = float - def get_num_items(self,obj): + def get_num_items(self, obj): """applies the num_items override or the costmodel default if not set""" if self.num_items is not None: k = self.num_items else: k = obj.num_items if isinstance(obj, CostModel) else 1 - return k - + return k + def __get__(self, obj, objtype=None): if obj is None: return self # class support @@ -364,26 +369,35 @@ def future_costs(self) -> float: initial_costs = self.costs_at_term(0, False) return numpy.nansum(list(initial_costs.values())) - def sum_costs(self, saved: set = None, categories: tuple = None, term=0): + def sum_costs( + self, saved: set = None, categories: tuple = None, term=0, econ=None + ): """sums costs of cost_property's in this item that are present at term=0, and by category if define as input""" if saved is None: saved = set((self,)) # item cost included! elif self not in saved: saved.add(self) itemcst = list( - self.dict_itemized_costs(saved, categories, term).values() + self.dict_itemized_costs( + saved, categories, term, econ=econ + ).values() ) csts = [self.sub_costs(saved, categories, term), numpy.nansum(itemcst)] return numpy.nansum(csts) def dict_itemized_costs( - self, saved: set = None, categories: tuple = None, term=0, test_val=True + self, + saved: set = None, + categories: tuple = None, + term=0, + test_val=True, + econ=None, ) -> dict: ccp = self.class_cost_properties() costs = { k: ( obj.__get__(self) - if obj.apply_at_term(self, term) == test_val + if obj.apply_at_term(self, term, econ) == test_val else 0 ) for k, obj in ccp.items() @@ -392,7 +406,9 @@ def dict_itemized_costs( } return costs - def sub_costs(self, saved: set = None, categories: tuple = None, term=0): + def sub_costs( + self, saved: set = None, categories: tuple = None, term=0, econ=None + ): """gets items from CostModel's defined in a Slot attribute or in a slot default, tolerrant to nan's in cost definitions""" if saved is None: saved = set() @@ -410,7 +426,7 @@ def sub_costs(self, saved: set = None, categories: tuple = None, term=0): saved.add(comp) if isinstance(comp, CostModel): - sub = comp.sum_costs(saved, categories, term) + sub = comp.sum_costs(saved, categories, term, econ=econ) log.debug( f"{self.identity} adding: {comp.identity if comp else comp}: {sub}+{sub_tot}" ) @@ -450,14 +466,17 @@ def sub_costs(self, saved: set = None, categories: tuple = None, term=0): return sub_tot # Cost Term & Category Reporting - def costs_at_term(self, term: int, test_val=True) -> dict: + def costs_at_term(self, term: int, test_val=True, econ=None) -> dict: """returns a dictionary of all costs at term i, with zero if the mode - function returns False at that term""" + function returns False at that term + + :param econ: the economics component to apply "end" term mode + """ ccp = self.class_cost_properties() return { k: ( obj.__get__(self) - if obj.apply_at_term(self, term) == test_val + if obj.apply_at_term(self, term, econ) == test_val else 0 ) for k, obj in ccp.items() @@ -487,10 +506,10 @@ def cost_categories(self): base[cc] += obj.__get__(self) return base - def cost_categories_at_term(self, term: int): + def cost_categories_at_term(self, term: int, econ=None): base = {cc: 0 for cc in self.all_categories()} for k, obj in self.class_cost_properties().items(): - if obj.apply_at_term(self, term): + if obj.apply_at_term(self, term, econ): for cc in obj.cost_categories: base[cc] += obj.__get__(self) return base @@ -677,7 +696,7 @@ def term_fgen(self, comp, prop): if isinstance(comp, dict): return lambda term: comp[prop] if term == 0 else 0 return lambda term: ( - prop.__get__(comp) if prop.apply_at_term(comp, term) else 0 + prop.__get__(comp) if prop.apply_at_term(comp, term, self) else 0 ) def sum_term_fgen(self, ref_group): @@ -756,9 +775,10 @@ def cost_summary(self, annualized=False, do_print=True, ignore_zero=True): } def abriv(val): - if val > 1e6: + evall = abs(val) + if evall > 1e6: return f"{val/1E6:>12.4f} M" - elif val > 1e3: + elif evall > 1e3: return f"{val/1E3:>12.2f} k" return f"{val:>12.2f}" @@ -772,49 +792,92 @@ def abriv(val): if ".cost." in col and is_ann == annualized: base, cst = col.split(".cost.") - ckey = f"{base.replace('lifecycle.','')}.cost.{cst}" - #print(ckey,str(self._comp_costs.keys())) + if base == "lifecycle": + ckey = f"cost.{cst}" # you're at the top bby + else: + ckey = f"{base.replace('lifecycle.','')}.cost.{cst}" + # print(ckey,str(self._comp_costs.keys())) comp_costs[base][cst] = val - comp_nums[base][cst] = get_num_from_cost_prop(self._comp_costs[ckey]) + comp_nums[base][cst] = get_num_from_cost_prop( + self._comp_costs[ckey] + ) elif col.startswith("summary."): summary[col.replace("summary.", "")] = val + else: + self.msg(f"skipping: {col}") total_cost = sum([sum(list(cc.values())) for cc in comp_costs.values()]) # provide consistent format hdr = "{key:<32}|\t{value:12.10f}" fmt = "{key:<32}|\t{fmt:<24} | {total:^12} | {pct:3.3f}%" - title = f'COST SUMMARY: {self.parent.identity}' + title = f"COST SUMMARY: {self.parent.identity}" if do_print: - self.info('#'*80) - self.info(f'{title:^80}') - self.info('_'*80) - #summary items - for key,val in summary.items(): - self.info(hdr.format(key=key,value=val)) - itcst = '{val:>24}'.format(val='TOTAL----->') - self.info('='*80) - self.info(fmt.format(key='COMBINED',fmt=itcst,total=abriv(total_cost),pct=100)) - self.info('-'*80) - #itemization - sgroups = lambda kv:sum(list(kv[-1].values())) - for base,items in sorted(comp_costs.items(),key=sgroups,reverse=True): - if (subtot:=sum(list(items.values()))) > 0: - #self.info(f' {base:<35}| total ---> {abriv(subtot)} | {subtot*100/total_cost:3.0f}%') - pct = subtot*100/total_cost - itcst = '{val:>24}'.format(val='TOTAL----->') - #todo; add number of items for cost comp - self.info(fmt.format(key=base,fmt=itcst,total=abriv(subtot),pct=pct)) - #Sort costs by value - for key,val in sorted(items.items(),key=lambda kv:kv[-1],reverse=True): - #self.info(f' \t{key:<32}|{abriv(val)} | {val*100/total_cost:^3.0f}%') + self.info("#" * 80) + self.info(f"{title:^80}") + self.info("_" * 80) + # possible core values + if NI := getattr(self, "num_items", None): # num items + self.info(hdr.format(key="num_items", value=NI)) + if TI := getattr(self, "term_length", None): + self.info(hdr.format(key="term_length", value=TI)) + if DR := getattr(self, "discount_rate", None): + self.info(hdr.format(key="discount_rate", value=DR)) + if DR := getattr(self, "output", None): + self.info(hdr.format(key="output", value=DR)) + # summary items + for key, val in summary.items(): + self.info(hdr.format(key=key, value=val)) + itcst = "{val:>24}".format(val="TOTAL----->") + self.info("=" * 80) + self.info( + fmt.format( + key="COMBINED", fmt=itcst, total=abriv(total_cost), pct=100 + ) + ) + self.info("-" * 80) + # itemization + sgroups = lambda kv: sum(list(kv[-1].values())) + for base, items in sorted( + comp_costs.items(), key=sgroups, reverse=True + ): + # skip if all zeros (allow for net negative costs) + if (subtot := sum([abs(v) for v in items.values()])) > 0: + # self.info(f' {base:<35}| total ---> {abriv(subtot)} | {subtot*100/total_cost:3.0f}%') + pct = subtot * 100 / total_cost + itcst = "{val:>24}".format(val="TOTAL----->") + # todo; add number of items for cost comp + adj_base = base.replace("lifecycle.", "") + self.info( + fmt.format( + key=adj_base, + fmt=itcst, + total=abriv(subtot), + pct=pct, + ) + ) + # Sort costs by value + for key, val in sorted( + items.items(), key=lambda kv: kv[-1], reverse=True + ): + if val == 0 or numpy.isnan(val): + continue # skip zero costs + # self.info(f' \t{key:<32}|{abriv(val)} | {val*100/total_cost:^3.0f}%') tot = abriv(val) - pct = val*100/total_cost + pct = val * 100 / total_cost num = comp_nums[base][key] - itcst = f'{abriv(val/num):^18} x {num:3.0f}' if num != 0 else '0' - self.info(fmt.format(key='-'+key,fmt=itcst,total=tot,pct=pct)) - self.info('-'*80) #section break - self.info('#'*80) + itcst = ( + f"{abriv(val/num):^18} x {num:3.0f}" + if num != 0 + else "0" + ) + self.info( + fmt.format( + key="-" + key, fmt=itcst, total=tot, pct=pct + ) + ) + self.info("-" * 80) # section break + self.info("#" * 80) return costs @property diff --git a/requirements.txt b/requirements.txt index 89727e8..838e8a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ jupyter pandas scikit-learn~=1.3.2 seaborn +tabulate #Engineering sectionproperties~=3.1.2 From 1e81f1afde3be68c150d53e215780cff1819db0a Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Mon, 28 Oct 2024 15:06:00 -0400 Subject: [PATCH 13/15] Add Econ Plots - added end-period correciton to start zero - add cost categorization plots and breakdowns --- engforge/eng/costs.py | 243 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 240 insertions(+), 3 deletions(-) diff --git a/engforge/eng/costs.py b/engforge/eng/costs.py index 0969f90..e8d13a3 100644 --- a/engforge/eng/costs.py +++ b/engforge/eng/costs.py @@ -54,7 +54,7 @@ class Parent(System,CostModel) import collections import pandas import collections - +import re class CostLog(LoggingMixin): pass @@ -70,7 +70,7 @@ class CostLog(LoggingMixin): "always": lambda inst, term, econ: True, "end": lambda inst, term, econ: ( True - if hasattr(econ, "term_length") and term == econ.term_length + if hasattr(econ, "term_length") and term == econ.term_length - 1 else False ), } @@ -723,6 +723,7 @@ def internal_references(self, recache=True, numeric_only=False): if self._cost_references: props.update(**self._cost_references) + #lookup ref from the cost categories dictionary, recreate every time if self._cost_categories: for key, refs in self._cost_categories.items(): props[key] = Ref( @@ -893,7 +894,7 @@ def lifecycle_output(self) -> dict: for c in lc.columns: if "category" not in c and "cost" not in c: continue - tot = lc[c].sum() + tot = lc[c].sum()#lifecycle cost if "category" in c: c_ = c.replace("category.", "") lifecat[c_] = tot @@ -1210,6 +1211,242 @@ def output(self) -> float: return 0 return self._calc_output + @property + def cost_category_store(self): + D = collections.defaultdict(list) + Acat = set() + for catkey,cdict in self._cost_categories.items(): + for ci in cdict: + cprop = getattr(ci.comp.__class__,ci.key) + ccat = set(cprop.cost_categories.copy()) + Acat = Acat.union(ccat) + D[f'{ci.comp.classname}|{ci.key:>36}'] = ccat + return D,Acat + + def create_cost_graph(self,plot=True): + """creates a graph of the cost model using network X and display it""" + import collections + import networkx as nx + + D = collections.defaultdict(dict) + for catkey,cdict in self._cost_categories.items(): + for ci in cdict: + D[catkey][(ci.comp.classname,ci.key)] = ci + + G = nx.Graph() + + for d,dk in D.items(): + print(d.upper()) + cat = d.replace('category.','') + G.add_node(cat,category=cat) + for kk,r in dk.items(): + cmp = kk[0] + edge = kk[1] + if cmp not in G.nodes: + G.add_node(cmp,component=cmp) + G.add_edge(cmp,cat,cost=edge) + #print(kk) + + #pos = nx.nx_agraph.graphviz_layout(G) + #nx.draw(G, pos=pos) + #nx.draw(G,with_labels=True) + + if plot: + categories = nx.get_node_attributes(G, 'category').keys() + components = nx.get_node_attributes(G, 'component').keys() + + cm = [] + for nd in G: + if nd in categories: + cm.append('cyan') + else: + cm.append('pink') + + pos = nx.spring_layout(G,k=0.2, iterations=20,scale=1) + nx.draw(G,node_color=cm,with_labels=True,pos=pos,arrows=True,font_size=10,font_color='0.09',font_weight='bold',node_size=200) + + return G + + def cost_matrix(self): + D,Cats = self.cost_category_store + X = list(sorted(Cats)) + C = list(sorted(D.keys())) + M = [] + for k in C: + cats = D[k] + M.append([True if x in cats else numpy.nan for x in X]) + + Mx = numpy.array(M) + X = numpy.array(X) + C = numpy.array(C) + return Mx,X,C + + def create_cost_category_table(self): + """creates a table of costs and categories""" + Mx,X,C = self.cost_matrix() + + fig,ax = subplots(figsize=(12,12)) + + Mc = numpy.nansum(Mx,axis=0) + x = numpy.argsort(Mc) + + Xs = X[x] + ax.imshow(Mx[:,x]) + ax.set_yticklabels(C,fontdict={'family':'monospace','size':8}) + ax.set_yticks(numpy.arange(len(C))) + ax.set_xticklabels(Xs,fontdict={'family':'monospace','size':8}) + ax.set_xticks(numpy.arange(len(Xs))) + ax.grid(which='major',linestyle=':',color='k',zorder=0) + xticks(rotation=90) + fig.tight_layout() + + def determine_exclusive_cost_categories(self,include_categories=None,ignore_categories:set=None,min_groups:int=2,max_group_size=None,min_score=0.95,include_item_cost=False): + """looks at all possible combinations of cost categories, scoring them based on coverage of costs, and not allowing any double accounting of costs. This is an NP-complete problem and will take a long time for large numbers of items. You can add ignore_categories to ignore certain categories""" + import itertools + + Mx,X,C = self.cost_matrix() + + bad = [] + solutions = [] + inx = {k:i for i,k in enumerate(X)} + + + + assert include_categories is None or set(X).issuperset(include_categories), 'include_categories must be subset of cost categories' + + #ignore categories + if ignore_categories: + X = [x for x in X if x not in ignore_categories] + + if include_categories: + #dont include them in pair since they are added to the group explicitly + X = [x for x in X if x not in include_categories] + + if not include_item_cost: + C = [c for c in C if 'item_cost' not in c] + + Num_Costs = len(C) + goal_score = Num_Costs * min_score + NumCats = len(X)//2 + GroupSize = NumCats if max_group_size is None else max_group_size + for ni in range(min_groups,GroupSize): + print(f'level {ni}/{GroupSize}| {len(solutions)} answers') + for cgs in itertools.combinations(X,ni): + val = None + + #make the set with included if needed + scg = set(cgs) + if include_categories: + scg = scg.union(include_categories) + + #skip bad groups + if any([b.issubset(scg) for b in bad]): + #print(f'skipping {cgs}') + #sys.stdout.write('.') + continue + + good = True #innocent till guilty + for cg in cgs: + xi = Mx[:,inx[cg]].copy() + xi[np.isnan(xi)] = 0 + if val is None: + val = xi + else: + val = val + xi + #determine if any overlap (only pair level) + if np.nanmax(val) > 1: + print(f'bad {cgs}') + bad.append(scg) + good = False + break + + score = np.nansum(val) + if good and score > goal_score: + print(f'found good: {scg}') + solutions.append({'grp':scg,'score':score,'gsize':ni}) + + return solutions + + def cost_categories_from_df(self,df): + categories = set() + for val in df.columns: + m = re.match(re.compile('economics\.lifecycle\.category\.(?s:[a-z]*)$'),val) + if m: + categories.add(val) + return categories + + def plot_cost_categories(self,df,group,cmap='tab20c',make_title=None,ax=None): + categories = self.cost_categories_from_df(df) + from matplotlib import cm + #if grps: + #assert len(grps) == len(y_vars), 'groups and y_vars must be same length' + #assert all([g in categories for g in grps]), 'all groups must be in categories' + #TODO: project costs onto y_vars + #TODO: ensure groups and y_vars are same length + + + color = cm.get_cmap(cmap) + styles = {c.replace('economics.lifecycle.category.',''):{'color': color(i/ len(categories))} for i,c in enumerate(categories)} + + if make_title is None: + def make_title(row): + return f'{row["name"]}x{row["num_items"]} @{"floating" if row["ldepth"]>50 else "fixed"}' + + #for j,grp in enumerate(groups): + figgen = False + if ax is None: + figgen = True + fig,ax = subplots(figsize=(12,8)) + else: + fig = ax.get_figure() + + titles = [] + xticks = [] + data = {} + i = 0 + + + for inx,row in df.iterrows(): + i += 1 + tc = row['economics.summary.total_cost'] + cat_costs = {k.replace('economics.lifecycle.category.',''):row[k] for k in categories} + #print(i,cat_costs) + + spec_costs = {k:v for k,v in cat_costs.items() if k in group} + pos_costs = {k:v for k,v in spec_costs.items() if v>=0} + neg_costs = {k:v for k,v in spec_costs.items() if k not in pos_costs} + neg_amt = sum(list(neg_costs.values())) + pos_amt = sum(list(pos_costs.values())) + + data[i] = spec_costs.copy() + + com = {'x':i,'width':0.5,'linewidth':0} + cur = neg_amt + for k,v in neg_costs.items(): + opt = {} if i != 1 else {'label':k} + ax.bar(height=abs(v),bottom=cur,**com,**styles[k],**opt) + cur += abs(v) + for k,v in pos_costs.items(): + opt = {} if i != 1 else {'label':k} + ax.bar(height=abs(v),bottom=cur,**com,**styles[k],**opt) + cur += abs(v) + xticks.append(com['x']) + titles.append(make_title(row)) + + #Format the chart + ax.legend(loc='upper right') + ax.set_xlim([0,i+max(2,0.2*i)]) + ax.set_xticks(xticks) + + ax.set_xticklabels(titles,rotation=90) + ylim = ax.get_ylim() + ylim = ylim[0]-0.05*abs(ylim[0]),ylim[1]+0.05*abs(ylim[1]) + ax.set_yticks(numpy.linspace(*ylim,50),minor=True) + ax.grid(which='major',linestyle='--',color='k',zorder=0) + ax.grid(which='minor',linestyle=':',color='k',zorder=0) + if figgen: fig.tight_layout() + return {'fig':fig,'ax':ax,'data':data} + # TODO: add costs for iterable components (wide/narrow modes) # if isinstance(conf,ComponentIter): From 3776fa192c2811af2bc1c5ae040bb1a816b525bb Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Mon, 28 Oct 2024 18:09:40 -0400 Subject: [PATCH 14/15] Add Black Formatting --- docs/conf.py | 8 +- engforge/_testing_components.py | 68 +++++-- engforge/analysis.py | 4 +- engforge/attr_plotting.py | 36 +++- engforge/attr_signals.py | 4 +- engforge/attr_slots.py | 4 +- engforge/attr_solver.py | 8 +- engforge/attributes.py | 18 +- engforge/common.py | 4 +- engforge/component_collections.py | 6 +- engforge/components.py | 5 +- engforge/configuration.py | 46 +++-- engforge/datastores/data.py | 29 ++- engforge/datastores/gdocs.py | 267 +++++++++++++++++++------- engforge/datastores/reporting.py | 95 ++++++--- engforge/datastores/secrets.py | 35 +++- engforge/dynamics.py | 38 +++- engforge/eng/costs.py | 242 +++++++++++++---------- engforge/eng/fluid_material.py | 4 +- engforge/eng/geometry.py | 47 +++-- engforge/eng/prediction.py | 21 +- engforge/eng/structure.py | 63 ++++-- engforge/eng/structure_beams.py | 31 ++- engforge/eng/thermodynamics.py | 5 +- engforge/engforge_attributes.py | 39 ++-- engforge/env_var.py | 12 +- engforge/logging.py | 20 +- engforge/patterns.py | 4 +- engforge/problem_context.py | 201 +++++++++++++------ engforge/properties.py | 22 ++- engforge/solveable.py | 96 ++++++--- engforge/solver.py | 14 +- engforge/solver_utils.py | 16 +- engforge/system.py | 4 +- engforge/system_reference.py | 23 ++- engforge/tabulation.py | 41 ++-- engforge/test/_pre_test_structures.py | 16 +- engforge/test/report_testing.py | 8 +- engforge/test/test_comp_iter.py | 4 +- engforge/test/test_costs.py | 3 +- engforge/test/test_dynamics_spaces.py | 4 +- engforge/test/test_four_bar.py | 8 +- engforge/test/test_performance.py | 4 +- engforge/test/test_pipes.py | 32 ++- engforge/test/test_problem.py | 20 +- engforge/test/test_solver.py | 12 +- engforge/test/test_tabulation.py | 16 +- engforge/typing.py | 6 +- examples/air_filter.py | 4 +- examples/spring_mass.py | 4 +- test/test_problem.py | 20 +- 51 files changed, 1220 insertions(+), 521 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index fbc5966..199c0f5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,11 +47,11 @@ False # Remove 'view source code' from top of page (for html, not python) ) autodoc_inherit_docstrings = True # If no docstring, inherit from base class -set_type_checking_flag = True # Enable 'expensive' imports for sphinx_autodoc_typehints -# nbsphinx_allow_errors = True # Continue through Jupyter errors -autodoc_typehints = ( - "description" # Sphinx-native method. Not as good as sphinx_autodoc_typehints +set_type_checking_flag = ( + True # Enable 'expensive' imports for sphinx_autodoc_typehints ) +# nbsphinx_allow_errors = True # Continue through Jupyter errors +autodoc_typehints = "description" # Sphinx-native method. Not as good as sphinx_autodoc_typehints add_module_names = False # Remove namespaces from class/method signatures html_theme = "sphinx_rtd_theme" diff --git a/engforge/_testing_components.py b/engforge/_testing_components.py index 4a966db..4a9d469 100644 --- a/engforge/_testing_components.py +++ b/engforge/_testing_components.py @@ -95,15 +95,23 @@ class SpaceMixin(SolveableInterface): lenP = Solver.con_ineq("edge_margin", combos="ineq_length", active=False) # Objectives - size_goal = Solver.objective("goal", combos="prop_goal", kind="min", active=False) + size_goal = Solver.objective( + "goal", combos="prop_goal", kind="min", active=False + ) - size = Solver.objective("volume", combos="obj_size", kind="max", active=False) + size = Solver.objective( + "volume", combos="obj_size", kind="max", active=False + ) sizeF = Solver.objective( wrap_f(Fun_size), combos="obj_size", kind="max", active=False ) - eff = Solver.objective("cost_to_volume", combos="obj_eff", kind="min", active=False) - effF = Solver.objective(wrap_f(Fun_eff), combos="obj_eff", kind="min", active=False) + eff = Solver.objective( + "cost_to_volume", combos="obj_eff", kind="min", active=False + ) + effF = Solver.objective( + wrap_f(Fun_eff), combos="obj_eff", kind="min", active=False + ) @system_property def combine_length(self) -> float: @@ -176,9 +184,13 @@ class CubeSystem(System, SpaceMixin): goal_vol_frac: float = 0.5 - sys_budget = Solver.con_ineq("total_budget", "system_cost", combos="total_budget") + sys_budget = Solver.con_ineq( + "total_budget", "system_cost", combos="total_budget" + ) - sys_length = Solver.con_ineq("total_length", "system_length", combos="total_length") + sys_length = Solver.con_ineq( + "total_length", "system_length", combos="total_length" + ) volfrac = Solver.con_eq( "goal_vol_frac", "vol_frac", combos="vol_frac_eq", active=False @@ -319,7 +331,8 @@ class DynamicSystem(System): def spring_accel(self) -> float: # print(self.comp.v,self.comp.x,self.comp.a) return ( - -self.comp.v * self.comp.b - (self.comp.x - self.comp.x0) * self.comp.K + -self.comp.v * self.comp.b + - (self.comp.x - self.comp.x0) * self.comp.K ) / self.comp.M @system_property @@ -333,7 +346,9 @@ def delta_a(self) -> float: def create_state_matrix(self, *args, **kwargs) -> np.ndarray: """creates the state matrix for the system""" - return np.array([[0, 1.0], [-self.K / self.Mass, -1 * self.Damp / self.Mass]]) + return np.array( + [[0, 1.0], [-self.K / self.Mass, -1 * self.Damp / self.Mass]] + ) def create_state_constants(self, *args, **kwargs) -> np.ndarray: """creates the input matrix for the system, called B""" @@ -341,7 +356,9 @@ def create_state_constants(self, *args, **kwargs) -> np.ndarray: def update_state(self, *args, **kwargs) -> np.ndarray: """creates the state matrix for the system""" - return np.array([[0, 1.0], [-self.K / self.Mass, -1 * self.Damp / self.Mass]]) + return np.array( + [[0, 1.0], [-self.K / self.Mass, -1 * self.Damp / self.Mass]] + ) def update_state_constants(self, *args, **kwargs) -> np.ndarray: """creates the input matrix for the system, called B""" @@ -449,7 +466,9 @@ class Airfilter(System): pr_eq = Solver.constraint_equality("sum_dP", 0, combos="flow") - flow_curve = Plot.define("throttle", "w", kind="lineplot", title="Flow Curve") + flow_curve = Plot.define( + "throttle", "w", kind="lineplot", title="Flow Curve" + ) @system_property def dP_parasitic(self) -> float: @@ -530,7 +549,9 @@ class SliderCrank(System, CostModel): rg_slv.add_var_constraint(0.0125, "min") rg_slv = Solver.declare_var("Lo", combos="design") - rg_slv.add_var_constraint(lambda s, p: s.Rc * s.Lo_factor, "min", combos="design") + rg_slv.add_var_constraint( + lambda s, p: s.Rc * s.Lo_factor, "min", combos="design" + ) offy_slv = Solver.declare_var("y_offset", combos="design") offx_slv = Solver.declare_var("x_offset", combos="design") @@ -620,7 +641,8 @@ def gamma(self) -> float: def motion_curve(self) -> np.ndarray: x = ( self.Rc * np.cos(theta) - + (self.Lo**2 - (self.Rc * np.sin(theta) - self.y_offset) ** 2) ** 0.5 + + (self.Lo**2 - (self.Rc * np.sin(theta) - self.y_offset) ** 2) + ** 0.5 ) return x @@ -679,7 +701,12 @@ def ds_goal(self) -> float: @system_property def mass_main_gear(self) -> float: - return np.pi * self.gear_material.density * self.Ro**2 * self.gear_thickness + return ( + np.pi + * self.gear_material.density + * self.Ro**2 + * self.gear_thickness + ) @system_property def Imain_gear(self) -> float: @@ -687,7 +714,12 @@ def Imain_gear(self) -> float: @system_property def mass_pwr_gear(self) -> float: - return np.pi * self.gear_material.density * self.Rg**2 * self.gear_thickness + return ( + np.pi + * self.gear_material.density + * self.Rg**2 + * self.gear_thickness + ) @system_property def Ipwr_gear(self) -> float: @@ -777,7 +809,9 @@ def cost_maintenance(self): def cost_tax(self): return 1 - @cost_property(mode=quarterly, category="opex,tax", label="quarterly wage tax") + @cost_property( + mode=quarterly, category="opex,tax", label="quarterly wage tax" + ) def cost_wage_tax(self): return 5 * 3 @@ -852,7 +886,9 @@ def calculate_production(self, parent, term): @forge class FanSystem(System, CostModel): - base = Slot.define(Component) #FIXME: not showing in "econ" (due to base default cost?) + base = Slot.define( + Component + ) # FIXME: not showing in "econ" (due to base default cost?) fan = Slot.define(Fan) motor = Slot.define(Motor) diff --git a/engforge/analysis.py b/engforge/analysis.py index 0014b5f..bc7b707 100644 --- a/engforge/analysis.py +++ b/engforge/analysis.py @@ -64,7 +64,9 @@ def uploaded(self): def run(self, *args, **kwargs): """Analysis.run() passes inputs to the assigned system and saves data via the system.run(cb=callback), once complete `Analysis.post_process()` is run also being passed input arguments, then plots & reports are made""" - self.info(f"running analysis {self.identity} with input {args} {kwargs}") + self.info( + f"running analysis {self.identity} with input {args} {kwargs}" + ) cb = lambda *args, **kw: self.system.last_context.save_data(force=True) out = self.system.run(*args, **kwargs, cb=cb) self.post_process(*args, **kwargs) diff --git a/engforge/attr_plotting.py b/engforge/attr_plotting.py index 8762cdd..1206083 100644 --- a/engforge/attr_plotting.py +++ b/engforge/attr_plotting.py @@ -257,7 +257,9 @@ def __init__(self, system: "System", plot_cls: "Plot"): self.system = system _sys_refs = self.system.system_references() - sys_refs = {k: v for atr, grp in _sys_refs.items() for k, v in grp.items()} + sys_refs = { + k: v for atr, grp in _sys_refs.items() for k, v in grp.items() + } diff = set() varss = set() @@ -276,11 +278,15 @@ def __init__(self, system: "System", plot_cls: "Plot"): if self.system.log_level < 10: log.debug(f"system references: {sys_refs}") if diff: - log.warning(f"has diff {diff}| found: {varss}| possible: {sys_refs}") + log.warning( + f"has diff {diff}| found: {varss}| possible: {sys_refs}" + ) if diff: # raise KeyError(f"has system diff: {diff} found: {vars}| from: {sys_ref}") - log.warning(f"has system diff: {diff} found: {vars}| from: {sys_refs}") + log.warning( + f"has system diff: {diff} found: {vars}| from: {sys_refs}" + ) self.refs = {k: sys_refs[k] for k in varss} @@ -317,7 +323,9 @@ def __call__(self, **override_kw): log.debug(f"overriding vars {override_kw}") args.update(**override_kw) - log.info(f"plotting {self.system.identity}| {self.identity} with {args}") + log.info( + f"plotting {self.system.identity}| {self.identity} with {args}" + ) fig = ax = f(data=self.system.dataframe, **args, **extra) return self.process_fig(fig, title) @@ -402,10 +410,14 @@ def validate_plot_args(cls, system: "System"): diff.add(vers) if log.log_level <= 10: - log.debug(f"{cls.__name__} has vars: {attr_keys} and bad input: {diff}") + log.debug( + f"{cls.__name__} has vars: {attr_keys} and bad input: {diff}" + ) if diff: - log.warning(f"bad plot vars: {diff} do not exist in system: {valid}") + log.warning( + f"bad plot vars: {diff} do not exist in system: {valid}" + ) # TODO: fix time being defined on components # raise KeyError( # f"bad plot vars: {diff} do not exist in system: {valid}" @@ -468,7 +480,9 @@ def __call__(self, **override_kw): log.debug(f"overriding vars {override_kw}") args.update(**override_kw) - log.info(f"plotting {self.system.identity}| {self.identity} with {args}") + log.info( + f"plotting {self.system.identity}| {self.identity} with {args}" + ) # PLOTTING # Make the axes and plot @@ -572,7 +586,9 @@ class Trace(PlotBase): always = ("x", "y", "y2") @classmethod - def define(cls, x="time", y: trace_type = None, y2=None, kind="line", **kwargs): + def define( + cls, x="time", y: trace_type = None, y2=None, kind="line", **kwargs + ): """Defines a plot that will be matplotlib, with validation happening as much as possible in the define method #Plot Choice @@ -748,7 +764,9 @@ def define( """ # Validate Plot - assert _type in PLOT_KINDS, f"type {_type} must be in {PLOT_KINDS.keys()}" + assert ( + _type in PLOT_KINDS + ), f"type {_type} must be in {PLOT_KINDS.keys()}" kinds = PLOT_KINDS[_type] assert kind in kinds, f"plot kind {kind} not in {kinds}" diff --git a/engforge/attr_signals.py b/engforge/attr_signals.py index 62820a9..44cf6ea 100644 --- a/engforge/attr_signals.py +++ b/engforge/attr_signals.py @@ -37,7 +37,9 @@ def apply(self): """sets `target` from `source`""" val = self.source.value() if self.system.log_level < 10: - self.system.msg(f"Signal| applying {self.source}|{val} to {self.target}") + self.system.msg( + f"Signal| applying {self.source}|{val} to {self.target}" + ) self.target.set_value(val) @property diff --git a/engforge/attr_slots.py b/engforge/attr_slots.py index 05f10b5..8278fa5 100644 --- a/engforge/attr_slots.py +++ b/engforge/attr_slots.py @@ -190,7 +190,9 @@ def make_factory(cls, **kwargs): log.debug(f"slot factory: {accepted},{cls.dflt_kw},{cls.default_ok}") if cls.dflt_kw: - return attrs.Factory(cls.make_accepted(accepted, **cls.dflt_kw), False) + return attrs.Factory( + cls.make_accepted(accepted, **cls.dflt_kw), False + ) elif cls.default_ok: return attrs.Factory(accepted, False) else: diff --git a/engforge/attr_solver.py b/engforge/attr_solver.py index 4da79f7..f24f47f 100644 --- a/engforge/attr_solver.py +++ b/engforge/attr_solver.py @@ -212,7 +212,9 @@ def configure_for_system(cls, name, config_class, cb=None, **kwargs): :returns: [optional] a dictionary of options to be used in the make_attribute method """ pre_name = cls.name # random attr name - super(Solver, cls).configure_for_system(name, config_class, cb, **kwargs) + super(Solver, cls).configure_for_system( + name, config_class, cb, **kwargs + ) # change name of constraint var if if cls.slvtype == "var": @@ -399,7 +401,9 @@ def add_var_constraint(cls, value, kind="min", **kwargs): var = cls.var assert var is not None, "must provide var on non-var solvers" - assert cls.slvtype == "var", "only Solver.declare_var can have constraints" + assert ( + cls.slvtype == "var" + ), "only Solver.declare_var can have constraints" assert kind in ("min", "max") combo_dflt = "default,lim" diff --git a/engforge/attributes.py b/engforge/attributes.py index 259cbda..30ea302 100644 --- a/engforge/attributes.py +++ b/engforge/attributes.py @@ -25,7 +25,9 @@ class AttributeInstance: # TODO: universal slots method # __slots__ = ["system", "class_attr"] - def __init__(self, class_attr: "CLASS_ATTR", system: "System", **kwargs) -> None: + def __init__( + self, class_attr: "CLASS_ATTR", system: "System", **kwargs + ) -> None: self.class_attr = class_attr self.system = system self.compile(**kwargs) @@ -186,14 +188,14 @@ def define(cls, **kwargs): def _setup_cls(cls, name, new_dict, **kwargs): # randomize name for specifics reasons uid = str(uuid.uuid4()) - #base info + # base info name = name + "_" + uid.replace("-", "")[0:16] new_dict["uuid"] = uid new_dict["default_options"] = cls.default_options.copy() new_dict["template_class"] = False new_dict["name"] = name - - #create a new type of attribute + + # create a new type of attribute new_slot = type(name, (cls,), new_dict) new_slot.default_options["default"] = new_slot.make_factory() new_slot.default_options["validator"] = new_slot.configure_instance @@ -235,7 +237,9 @@ def collect_cls(cls, system) -> dict: """collects all the attributes for a system""" if not isinstance(system, type): system = system.__class__ - return {k: at.type for k, at in system._get_init_attrs_data(cls).items()} + return { + k: at.type for k, at in system._get_init_attrs_data(cls).items() + } @classmethod def collect_attr_inst(cls, system, handle_inst=True) -> dict: @@ -244,7 +248,9 @@ def collect_attr_inst(cls, system, handle_inst=True) -> dict: out = {} for k, v in cattr.items(): inst = getattr(system, k) - if (inst is None and getattr(cls.instance_class, "none_ok", False)) or ( + if ( + inst is None and getattr(cls.instance_class, "none_ok", False) + ) or ( cls.instance_class is not None and not isinstance(inst, cls.instance_class) ): diff --git a/engforge/common.py b/engforge/common.py index 3168d17..9213087 100644 --- a/engforge/common.py +++ b/engforge/common.py @@ -58,7 +58,9 @@ def get_size(obj, seen=None): size += sum([get_size(k, seen) for k in obj.keys()]) elif hasattr(obj, "__dict__"): size += get_size(obj.__dict__, seen) - elif hasattr(obj, "__iter__") and not isinstance(obj, (str, bytes, bytearray)): + elif hasattr(obj, "__iter__") and not isinstance( + obj, (str, bytes, bytearray) + ): size += sum([get_size(i, seen) for i in obj]) return size diff --git a/engforge/component_collections.py b/engforge/component_collections.py index ff83bfa..61d902e 100644 --- a/engforge/component_collections.py +++ b/engforge/component_collections.py @@ -28,7 +28,11 @@ def check_comp_type(instance, attr, value): """ensures the input component type is a Component""" from engforge.eng.costs import CostModel - if not instance.wide and isinstance(value, type) and issubclass(value, CostModel): + if ( + not instance.wide + and isinstance(value, type) + and issubclass(value, CostModel) + ): raise TypeError(f"Cost Mixin Not Supported As Iter Type! {value}") if isinstance(value, type) and issubclass(value, Component): diff --git a/engforge/components.py b/engforge/components.py index e1908cd..2016867 100644 --- a/engforge/components.py +++ b/engforge/components.py @@ -30,7 +30,10 @@ def last_context(self): """get the last context run, or the parent's""" if hasattr(self, "_last_context"): # cleanup parent context - if hasattr(self, "_parent_context") and not self.parent.last_context: + if ( + hasattr(self, "_parent_context") + and not self.parent.last_context + ): del self._last_context del self._parent_context else: diff --git a/engforge/configuration.py b/engforge/configuration.py index 0113819..f3be4a4 100644 --- a/engforge/configuration.py +++ b/engforge/configuration.py @@ -56,7 +56,9 @@ def name_generator(instance): """a name generator for the instance""" base = str(instance.__class__.__name__).lower() + "-" if instance.__class__._use_random_name: - out = base + randomname.get_name(adj=NAME_ADJ.secret, noun=NAME_NOUN.secret) + out = base + randomname.get_name( + adj=NAME_ADJ.secret, noun=NAME_NOUN.secret + ) else: out = base log.debug(f"generated name: {out}") @@ -164,7 +166,9 @@ def property_changed(instance, variable, value): if not session and instance._anything_changed: # Bypass Check since we've already flagged for an update if log.log_level <= 2: - log.debug(f"already property changed {instance}{variable.name} {value}") + log.debug( + f"already property changed {instance}{variable.name} {value}" + ) return value # elif session: @@ -175,7 +179,9 @@ def property_changed(instance, variable, value): # session.change_sys_var(variable,value,doset=False) attrs = attr.fields(instance.__class__) # check identity of variable cur = getattr(instance, variable.name) - is_different = value != cur if isinstance(value, (int, float, str)) else True + is_different = ( + value != cur if isinstance(value, (int, float, str)) else True + ) is_var = variable in attrs chgnw = instance._anything_changed @@ -183,7 +189,9 @@ def property_changed(instance, variable, value): log.debug( f"checking property changed {instance}{variable.name} {value}|invar: {is_var}| nteqval: {is_different}" ) - print(f"checking property changed {instance}{variable.name} {value}|invar: {is_var}| nteqval: {is_different}") + print( + f"checking property changed {instance}{variable.name} {value}|invar: {is_var}| nteqval: {is_different}" + ) # Check if should be updated if not chgnw and is_var and is_different: @@ -239,7 +247,9 @@ def signals_slots_handler( # Fields for t in fields: if t.name in PROTECTED_NAMES: - raise Exception(f"cannot use {t.name} as a field name, its protected") + raise Exception( + f"cannot use {t.name} as a field name, its protected" + ) if t.type is None: log.warning(f"{cls.__name__}.{t.name} has no type") @@ -318,7 +328,7 @@ def signals_slots_handler( cls_properties = cls.system_properties_classdef(True) else: cls_properties = {} - + cls_dict = cls.__dict__.copy() cls.__anony_store = {} # print(f'tab found!! {cls_properties.keys()}') @@ -409,7 +419,9 @@ def internal_configurations( if not use_dict: # slots obj = {k: obj.get(k, None) for k in slots} - return {k: v for k, v in obj.items() if chk(k, v) and not k.startswith("_")} + return { + k: v for k, v in obj.items() if chk(k, v) and not k.startswith("_") + } def copy_config_at_state( self, level=None, levels_deep: int = -1, changed: dict = None, **kw @@ -449,7 +461,9 @@ def copy_config_at_state( if config in changed: ccomp = changed[config] else: - ccomp = config.copy_config_at_state(level + 1, levels_deep, changed) + ccomp = config.copy_config_at_state( + level + 1, levels_deep, changed + ) kwcomps[key] = ccomp # Finally make the new system with changed internals @@ -462,7 +476,7 @@ def copy_config_at_state( new_sys.system_references(recache=True) # update the parents - #TODO: use pyee to broadcast change + # TODO: use pyee to broadcast change if hasattr(self, "parent"): if self.parent in changed: new_sys.parent = changed[self.parent] @@ -484,7 +498,7 @@ def go_through_configurations( :return: level,config""" from engforge.configuration import Configuration - #TODO: instead of a recursive loop a global map per problem context should be used, with a static map of slots, updating with every change per note in system_references. This function could be a part of that but each system shouldn't be responsible for it. + # TODO: instead of a recursive loop a global map per problem context should be used, with a static map of slots, updating with every change per note in system_references. This function could be a part of that but each system shouldn't be responsible for it. should_yield_level = lambda level: all( [ @@ -553,7 +567,7 @@ def __attrs_post_init__(self): self.warning( f"Component {compnm} already has a parent {comp.parent} #copying, and assigning to {self}" ) - #setattr(self, compnm, attrs.evolve(comp, parent=self)) + # setattr(self, compnm, attrs.evolve(comp, parent=self)) else: comp.parent = self @@ -561,7 +575,9 @@ def __attrs_post_init__(self): # subclass instance instance init causes conflicts in structures self.__on_init__() - runs = set((self.__class__.__on_init__,)) # keep track of unique functions + runs = set( + (self.__class__.__on_init__,) + ) # keep track of unique functions if self._subclass_init: try: for comp in self.__class__.mro(): @@ -622,7 +638,11 @@ def filename(self): .title() ) filename = "".join( - [c for c in fil if c.isalpha() or c.isdigit() or c == "_" or c == "-"] + [ + c + for c in fil + if c.isalpha() or c.isdigit() or c == "_" or c == "-" + ] ).rstrip() return filename diff --git a/engforge/datastores/data.py b/engforge/datastores/data.py index fab6a14..bcae433 100644 --- a/engforge/datastores/data.py +++ b/engforge/datastores/data.py @@ -198,10 +198,10 @@ class DiskCacheStore(LoggingMixin, metaclass=SingletonMeta): retries = 1 sleep_time = 0.1 - proj_dir: str = None #avoid using implicit determinination - cache_path: str = None #override for implicit path, a recommended practice + proj_dir: str = None # avoid using implicit determinination + cache_path: str = None # override for implicit path, a recommended practice - def __init__(self,root_path=None, **kwargs): + def __init__(self, root_path=None, **kwargs): if root_path is not None: self.cache_path = root_path @@ -226,7 +226,7 @@ def cache_root(self): if self.alt_path is not None: return os.path.join(self.proj_root, "cache", self.alt_path) - + return os.path.join( self.proj_root, "cache", @@ -289,7 +289,9 @@ def get(self, key=None, on_missing=None, retry=True, ttl=None): ttl -= 1 if ttl > 0: time.sleep(self.sleep_time * (self.retries - ttl)) - return self.get(key=key, on_missing=on_missing, retry=True, ttl=ttl) + return self.get( + key=key, on_missing=on_missing, retry=True, ttl=ttl + ) else: self.error(e, "Issue Getting Item From Cache") @@ -297,7 +299,10 @@ def expire(self): """wrapper for diskcache expire method that only permits expiration on a certain interval :return: bool, True if expired called""" now = time.time() - if self.last_expire is None or now - self.last_expire > self.expire_threshold: + if ( + self.last_expire is None + or now - self.last_expire > self.expire_threshold + ): self.cache.expire() self.last_expire = now return True @@ -336,7 +341,9 @@ class DBConnection(LoggingMixin, metaclass=InputSingletonMeta): # TODO: Make Threadsafe W/ ThreadPoolExecutor! # we love postgres! - _connection_template = "postgresql://{user}:{passd}@{host}:{port}/{database}" + _connection_template = ( + "postgresql://{user}:{passd}@{host}:{port}/{database}" + ) pool_size = 20 max_overflow = 0 @@ -359,7 +366,9 @@ class DBConnection(LoggingMixin, metaclass=InputSingletonMeta): connect_args = {"connect_timeout": 5} - def __init__(self, database_name=None, host=None, user=None, passd=None, **kwargs): + def __init__( + self, database_name=None, host=None, user=None, passd=None, **kwargs + ): """On the Singleton DBconnection.instance(): __init__(*args,**kwargs) will get called, technically you could do it this way but won't be thread safe, or a single instance :param database_name: the name for the database inside the db server @@ -434,7 +443,9 @@ def configure(self): # self.scopefunc = functools.partial(context.get, "uuid") - self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=True) + self.session_factory = sessionmaker( + bind=self.engine, expire_on_commit=True + ) self.Session = scoped_session(self.session_factory) @contextmanager diff --git a/engforge/datastores/gdocs.py b/engforge/datastores/gdocs.py index fd3242c..f22147b 100644 --- a/engforge/datastores/gdocs.py +++ b/engforge/datastores/gdocs.py @@ -177,8 +177,10 @@ def is_folder(self): if self.is_drivefile: if any( [ - self.item["mimeType"] == "application/vnd.google-apps.folder", - self.item["mimeType"] == "application/vnd.google-apps.shortcut", + self.item["mimeType"] + == "application/vnd.google-apps.folder", + self.item["mimeType"] + == "application/vnd.google-apps.shortcut", ] ): return True @@ -189,8 +191,10 @@ def is_file(self): if self.is_drivefile: if not any( [ - self.item["mimeType"] == "application/vnd.google-apps.folder", - self.item["mimeType"] == "application/vnd.google-apps.shortcut", + self.item["mimeType"] + == "application/vnd.google-apps.folder", + self.item["mimeType"] + == "application/vnd.google-apps.shortcut", ] ): return True @@ -212,7 +216,9 @@ def is_protected(self): return True # Protected Folder Name if self.title in self.drive.protected_filenames and self.is_folder: - self.debug(f'folder has a protected filename {self.item["title"]}') + self.debug( + f'folder has a protected filename {self.item["title"]}' + ) return True # File Id is protected elif self.id in self.drive.protected_ids: @@ -285,7 +291,9 @@ def absolute_paths(self): npaths = self.best_nodeid_paths if npaths: return [ - os.path.join(*[self.drive.item_nodes[itdd].title for itdd in path_list]) + os.path.join( + *[self.drive.item_nodes[itdd].title for itdd in path_list] + ) for path_list in npaths ] @@ -294,7 +302,9 @@ def best_nodeid_path(self): try: with self.filesystem as fs: return list( - nx.shortest_path(fs, source=self.drive.sync_root_id, target=self.id) + nx.shortest_path( + fs, source=self.drive.sync_root_id, target=self.id + ) ) except Exception as e: self.error(e) @@ -314,7 +324,9 @@ def absolute_path(self): parents = self.parents if parents: - self._absolute_path = os.path.join(parents[0].absolute_path, self.title) + self._absolute_path = os.path.join( + parents[0].absolute_path, self.title + ) # self.debug(f'caching file abspath for node {self._absolute_path}') else: self._absolute_path = self.title @@ -346,7 +358,9 @@ def ensure_http(self): def delete(self): self.ensure_http() if not self.is_protected and not self.drive.dry_run: - with self.drive.rate_limit_manager(self.delete, 2, gfileMeta=self.item): + with self.drive.rate_limit_manager( + self.delete, 2, gfileMeta=self.item + ): self.info(f"deleting: {self.title}") self.item.Trash() self.drive.sleep() @@ -470,7 +484,8 @@ def absolute_path(self): timestamp_divider = 24 * 3600 # a day in seconds days_since_2020 = ( - lambda item: (item.createdDate.timestamp() - default_timestamp) / timestamp_divider + lambda item: (item.createdDate.timestamp() - default_timestamp) + / timestamp_divider ) content_size = lambda item: len(item.contents) @@ -745,15 +760,19 @@ def authoirze_google_integrations(self, retry=True, ttl=3): self.sleep() self.gauth.auth_method = "service" - self.gauth.credentials = ServiceAccountCredentials.from_json_keyfile_name( - self.creds_file, scope + self.gauth.credentials = ( + ServiceAccountCredentials.from_json_keyfile_name( + self.creds_file, scope + ) ) self.gauth.Authorize() self.sleep() # Do Sheets Authentication - self.gsheets = pygsheets.authorize(service_account_file=self.creds_file) + self.gsheets = pygsheets.authorize( + service_account_file=self.creds_file + ) self.sleep() self.gdrive = GoogleDrive(self.gauth) @@ -792,11 +811,13 @@ def reset(self): def sync_path(self, path): """Sync path likes absolute paths to google drive relative""" - assert os.path.commonpath([self.filepath_root, path]) == os.path.commonpath( - [self.filepath_root] - ) + assert os.path.commonpath( + [self.filepath_root, path] + ) == os.path.commonpath([self.filepath_root]) - self.debug(f"finding relative path from {path} and {self.filepath_root}") + self.debug( + f"finding relative path from {path} and {self.filepath_root}" + ) rel_root = os.path.relpath(path, self.filepath_root) # self.debug(f'getting gdrive path {self.full_sync_root} and {rel_root}') @@ -877,7 +898,9 @@ class token: if protect: self.protected_ids.add(parent_id) - if pool is None: # and self.use_threadpool #Always use threads for this! + if ( + pool is None + ): # and self.use_threadpool #Always use threads for this! if top: self.debug("GET DRIVE INFO USING THREADS!!!") pool = ThreadPoolExecutor(max_workers=self.num_threads) @@ -931,7 +954,9 @@ class token: ) # TODO: handle duplicates - if item.is_folder: # if it had contents it was already cached!! + if ( + item.is_folder + ): # if it had contents it was already cached!! # Either do not stop or ensure that the item path is in the target if ( stop_when_found is None @@ -1005,7 +1030,9 @@ def rate_limit_manager(self, retry_function, multiplier, *args, **kwargs): except ConnectionError as err: self.info("connection reset, retrying auth") - self.hit_rate_limit(30.0 + 30.0 * random.random(), multiplier=multiplier) + self.hit_rate_limit( + 30.0 + 30.0 * random.random(), multiplier=multiplier + ) self.authoirze_google_integrations() self.sleep(10.0 + 10.0 * random.random()) return retry_function(*args, **kwargs) @@ -1104,7 +1131,9 @@ def initalize_google_drive_root(self): if "client" in sfol.lower(): for clientname in engforge_clients(): if clientname.lower() != "archive": - self.get_or_create_folder(clientname, parent_id=fol["id"]) + self.get_or_create_folder( + clientname, parent_id=fol["id"] + ) # Sync Methods def generate_sync_filepath_pairs(self, skip_existing=True): @@ -1120,7 +1149,9 @@ def generate_sync_filepath_pairs(self, skip_existing=True): parent_id = self.target_folder_id self.cache_directory(parent_id) - for i, (dirpath, dirnames, dirfiles) in enumerate(os.walk(self.filepath_root)): + for i, (dirpath, dirnames, dirfiles) in enumerate( + os.walk(self.filepath_root) + ): # we will modify dirnames in place to eliminate options for walking self.debug("looping through directory {}".format(dirpath)) # Handle File Sync Ignores @@ -1144,7 +1175,8 @@ def generate_sync_filepath_pairs(self, skip_existing=True): if any( [ - os.path.commonpath([dirpath, spath]) == os.path.commonpath([spath]) + os.path.commonpath([dirpath, spath]) + == os.path.commonpath([spath]) for spath in skipped_paths ] ): @@ -1222,7 +1254,9 @@ def sync(self, force=False): self.thread_time_multiplier = 1.0 # ensure its normal # #Conventional way make folders if they don't exist submitted_set = set() - with ThreadPoolExecutor(max_workers=self.num_threads) as pool: + with ThreadPoolExecutor( + max_workers=self.num_threads + ) as pool: for lpath, gpath in self.generate_sync_filepath_pairs( skip_existing=not force ): @@ -1248,7 +1282,9 @@ def sync(self, force=False): ) else: if gpath in self.item_paths: - self.debug("found existing {}".format(gpath)) + self.debug( + "found existing {}".format(gpath) + ) self.thread_time_multiplier = 1.0 @@ -1266,7 +1302,9 @@ def sync_path_pair_single(self, filepath, syncpath): if syncpath in self.item_paths: self.debug(f"updating {filepath} -> {syncpath}") fid = self.get_gpath_id(syncpath) - self.upload_or_update_file(par_id, file_path=filepath, file_id=fid) + self.upload_or_update_file( + par_id, file_path=filepath, file_id=fid + ) self.cache_directory(par_id) else: self.debug(f"uploading {filepath} -> {syncpath}") @@ -1281,7 +1319,9 @@ def sync_path_pair_thread(self, filepath, syncpath, parent_id=None): if syncpath in self.item_paths: self.debug(f"updating {filepath} -> {syncpath}") fid = self.get_gpath_id(syncpath) - self.upload_or_update_file(parent_id, file_path=filepath, file_id=fid) + self.upload_or_update_file( + parent_id, file_path=filepath, file_id=fid + ) self.cache_directory(parent_id) else: self.debug(f"uploading {filepath} -> {syncpath}") @@ -1310,13 +1350,13 @@ def create_file(self, input_args, file_path=None, content=None): with self.rate_limit_manager( self.create_file, 2, input_args, file_path, content ): - if not self.dry_run and any((file_path is not None, content is not None)): + if not self.dry_run and any( + (file_path is not None, content is not None) + ): action = "creating" if "id" not in input_args else "updating" if content is not None: - gfile_path = ( - file_path # Direct conversion, we have to input correctly - ) + gfile_path = file_path # Direct conversion, we have to input correctly if isinstance(content, (str, unicode)): self.debug( f"{action} file with content string {input_args} -> {gfile_path}" @@ -1334,11 +1374,15 @@ def create_file(self, input_args, file_path=None, content=None): file_name = os.path.basename(file_path) gfile_path = self.sync_path(file_path) - self.debug(f"{action} file with args {input_args} -> {gfile_path}") + self.debug( + f"{action} file with args {input_args} -> {gfile_path}" + ) file.SetContentFile(file_path) self.sleep() - file.Upload(param={"supportsTeamDrives": True}) # Upload the file. + file.Upload( + param={"supportsTeamDrives": True} + ) # Upload the file. self.sleep() file.FetchMetadata(fields="permissions,labels,mimeType") self.sleep() @@ -1376,7 +1420,9 @@ def upload_or_update_file( if file_id is not None: # Override takes precident self.debug( - "updating id: {} {}->{}".format(file_id, parent_id, gfile_path) + "updating id: {} {}->{}".format( + file_id, parent_id, gfile_path + ) ) fil = self.create_file( @@ -1412,7 +1458,9 @@ def upload_or_update_file( if file_id is not None: self.debug( - "updating file w/ content id:{} par:{}".format(file_id, parent_id) + "updating file w/ content id:{} par:{}".format( + file_id, parent_id + ) ) fil = self.create_file( {"id": file_id, "parents": [{"id": parent_id}]}, @@ -1425,7 +1473,9 @@ def upload_or_update_file( file_name = os.path.basename(file_path) self.debug( - "creating file w/ content {}->{}".format(parent_id, file_path) + "creating file w/ content {}->{}".format( + parent_id, file_path + ) ) fil = self.create_file( {"title": file_name, "parents": [{"id": parent_id}]}, @@ -1453,7 +1503,9 @@ def create_folder(self, input_args, upload=True): self.create_folder, 2, input_args, upload, gfileMeta=file ): if upload and not self.dry_run: - file.Upload(param={"supportsTeamDrives": True}) # Upload the file. + file.Upload( + param={"supportsTeamDrives": True} + ) # Upload the file. self.sleep() self.debug(f"uploaded {input_args}") file.FetchMetadata(fields="permissions,labels,mimeType") @@ -1509,7 +1561,9 @@ def get_or_create_folder(self, folder_name, parent_id=None, **kwargs): protect_name = not folder_name in self.protected_filenames - self.debug("found folder in parent {}: {}".format(parent_id, parent_folders)) + self.debug( + "found folder in parent {}: {}".format(parent_id, parent_folders) + ) if folder_name not in parent_folders: # Create It self.debug(f"creating {parent_id}->{folder_name}") @@ -1523,7 +1577,9 @@ def get_or_create_folder(self, folder_name, parent_id=None, **kwargs): return self.cache_item(fol) else: # Grab It - self.debug("found folder {} in parent {}".format(folder_name, parent_id)) + self.debug( + "found folder {} in parent {}".format(folder_name, parent_id) + ) return parent_items[folder_name] # Duplicate Handiling @@ -1569,7 +1625,9 @@ def group_duplicate_pairs(self, duplicate_paths_identified): duplicates_fil_sets = {} # path: [it1,it2...] if duplicate_paths_identified: - self.info(f"duplicate paths identified {len(duplicate_paths_identified)}") + self.info( + f"duplicate paths identified {len(duplicate_paths_identified)}" + ) files, folders = {}, {} for mtch_id, mtch in duplicate_paths_identified.items(): @@ -1596,7 +1654,9 @@ def group_duplicate_pairs(self, duplicate_paths_identified): for mtch_id, mtch in files.items() if not any( [ - self.path_contains(fol.absolute_path, mtch.absolute_path) + self.path_contains( + fol.absolute_path, mtch.absolute_path + ) for fid, fol in folders.items() ] ) @@ -1633,12 +1693,16 @@ def group_duplicate_pairs(self, duplicate_paths_identified): else: oked_paths.add(fil.absolute_path) - self.info(f"duplicate files paths to fix: {len(duplicate_file_paths)}") + self.info( + f"duplicate files paths to fix: {len(duplicate_file_paths)}" + ) check_fol_fil_intersection = { path: mtch for path, mtch in duplicates_fil_sets.items() - if any([self.path_contains(folpath, path) for folpath in dup_paths]) + if any( + [self.path_contains(folpath, path) for folpath in dup_paths] + ) } assert len(check_fol_fil_intersection) == 0 @@ -1659,7 +1723,9 @@ def remove_duplicates(self): # Proceed with work self.process_duplicates(duplicates_fols, duplicates_fil_sets) - def process_duplicates(self, duplicate_folder_groups, duplicate_file_groups): + def process_duplicates( + self, duplicate_folder_groups, duplicate_file_groups + ): """evals a set of duplicates and adjusts the fileystem as nessicary, the files and folders input should have no interaction""" self.delete_duplicate_items(duplicate_file_groups) @@ -1725,7 +1791,9 @@ def score_folders_suggest_action(items): ] # Sort by highest score - order_items = sorted(group, key=lambda itd: itd["score"], reverse=True) + order_items = sorted( + group, key=lambda itd: itd["score"], reverse=True + ) keep = order_items[0] # keep oldest folder remove = order_items[1:] out = {"keep": keep, "remove": remove} @@ -1807,7 +1875,10 @@ def ensure_g_path_get_id(self, gpath): else: self.debug(f"path doesnt exist, create it {current_pos}") if parent_id is not None and not any( - [fol.startswith(".") for fol in current_pos.split(os.sep)] + [ + fol.startswith(".") + for fol in current_pos.split(os.sep) + ] ): fol = self.get_or_create_folder(sub, parent_id) if fol is not None: @@ -1819,7 +1890,9 @@ def ensure_g_path_get_id(self, gpath): if parent_id is not None: return parent_id - elif not any([fol.startswith(".") for fol in current_pos.split(os.sep)]): + elif not any( + [fol.startswith(".") for fol in current_pos.split(os.sep)] + ): self.warning(f"Failed To get Gpath {gpath} Trying again ") return self.ensure_g_path_get_id(gpath) @@ -1833,10 +1906,14 @@ def get_gpath_id(self, gpath): matches = list(fids[paths == gpath]) if len(matches) > 1: - self.warning(f"gpathid found matches {matches} for {gpath}, using first") + self.warning( + f"gpathid found matches {matches} for {gpath}, using first" + ) # TODO: Handle this case! nodes = [self.item_nodes[mtch] for mtch in matches] - snodes = sorted(nodes, key=lambda it: days_since_2020(it), reverse=True) + snodes = sorted( + nodes, key=lambda it: days_since_2020(it), reverse=True + ) if snodes[0].is_folder: return snodes[0].id # This returns the oldest folder @@ -1910,7 +1987,11 @@ def file_nodes(self): @property def file_cache(self): with self.filesystem as fs: - return {node.id: node.absolute_path for node in fs.nodes() if node.is_file} + return { + node.id: node.absolute_path + for node in fs.nodes() + if node.is_file + } @property def file_paths(self): @@ -1926,7 +2007,9 @@ def folder_nodes(self): def folder_cache(self): with self.filesystem as fs: return { - node.id: node.absolute_path for node in fs.nodes() if node.is_folder + node.id: node.absolute_path + for node in fs.nodes() + if node.is_folder } @property @@ -1959,7 +2042,10 @@ def removeNode(self, node): self.protected_ids.remove(node.id) def cache_item(self, item_meta): - if "teamDriveId" in item_meta and item_meta["teamDriveId"] == self.sync_root_id: + if ( + "teamDriveId" in item_meta + and item_meta["teamDriveId"] == self.sync_root_id + ): if item_meta["id"] not in self.item_nodes: node = FileNode(self, item_meta) self.addFileNode(node) @@ -2006,7 +2092,9 @@ def search_items(self, q="", parent_id=None, **kwargs): existing_contents = {} success = False - with self.rate_limit_manager(self.search_items, 2, q, parent_id, **kwargs): + with self.rate_limit_manager( + self.search_items, 2, q, parent_id, **kwargs + ): self.sleep() for output in self.gdrive.ListFile(input_args): for file in output: @@ -2055,7 +2143,9 @@ def all_in_folder(self, folder_id=None): self.debug(f"searching {folder_id} for anything") output = list( - self.search_items(f"'{folder_id}' in parents and trashed=false", folder_id) + self.search_items( + f"'{folder_id}' in parents and trashed=false", folder_id + ) ) for file in output: yield file @@ -2078,7 +2168,9 @@ def hit_rate_limit(self, sleep_time=5, multiplier=2): self.max_sleep_time, ) else: - self._sleep_time = min(self._sleep_time * multiplier, self.max_sleep_time) + self._sleep_time = min( + self._sleep_time * multiplier, self.max_sleep_time + ) dynamicsleep = sleep_time * 0.5 + random.random() * sleep_time self.warning( @@ -2165,7 +2257,10 @@ def context(self, filepath_root, sync_root): old_syncroot = self.sync_root try: - if self.filepath_root == filepath_root and self.sync_root == sync_root: + if ( + self.filepath_root == filepath_root + and self.sync_root == sync_root + ): self.debug(f"Alread in context {filepath_root} -> {sync_root}") yield self # no work to do @@ -2177,7 +2272,9 @@ def context(self, filepath_root, sync_root): yield self except Exception as e: - self.error(e, f"Issue In Drive Context {filepath_root} -> {sync_root}") + self.error( + e, f"Issue In Drive Context {filepath_root} -> {sync_root}" + ) finally: self.filepath_root = old_filepath @@ -2204,7 +2301,9 @@ def filepath_root(self, filepath_root): self.filepath_inferred = False self.debug(f"using verified filepath {self._filepath_root}") else: - raise SourceFolderNotFound(f"could not find filepath {filepath_root}") + raise SourceFolderNotFound( + f"could not find filepath {filepath_root}" + ) elif not self.explict_input_only and self.guess_sync_path: # Infer It if in_client_dir(): @@ -2215,14 +2314,17 @@ def filepath_root(self, filepath_root): self.filepath_inferred = True self.debug(f"using infered client path: {self._filepath_root}") elif ( - self._shared_drive == self.default_shared_drive and in_dropbox_dir() + self._shared_drive == self.default_shared_drive + and in_dropbox_dir() ): # OTTERSYNC if in_wsl(): self._filepath_root = client_path(skip_wsl=False) else: self._filepath_root = client_path(skip_wsl=False) self.filepath_inferred = True - self.debug(f"using infered engforge path: {self._filepath_root}") + self.debug( + f"using infered engforge path: {self._filepath_root}" + ) if do_reinitalize: self.reset_target_id() @@ -2249,7 +2351,9 @@ def shared_drive(self, shared_drive): # 1) Look for credentials in service account creds if "shared_drive" in creds_info: self._shared_drive = creds_info["shared_drive"] - self.debug(f"using service account shared drive: {self._shared_drive}") + self.debug( + f"using service account shared drive: {self._shared_drive}" + ) # 2) Look for enviornmental variable elif "CLIENT_GDRIVE_PATH" in os.environ and os.environ[ @@ -2272,7 +2376,9 @@ def shared_drive(self, shared_drive): drive_canidate = drive_canidate.replace("shared:", "") if drive_canidate in self.shared_drives: self._shared_drive = f"shared:{drive_canidate}" - self.debug(f"using env-var shared drive: {self._shared_drive}") + self.debug( + f"using env-var shared drive: {self._shared_drive}" + ) else: raise SharedDriveNotFound( f"env var shared drive not found {gpath}->{drive_canidate}" @@ -2338,15 +2444,22 @@ def sync_root(self, sync_root): creds_info = self.credentials_info # 1) Look for credentials in service account creds - if "shared_sync_path" in creds_info and creds_info["shared_sync_path"]: + if ( + "shared_sync_path" in creds_info + and creds_info["shared_sync_path"] + ): self._sync_root = creds_info["shared_sync_path"] - self.debug(f"using service account sync path: {self._sync_root}") + self.debug( + f"using service account sync path: {self._sync_root}" + ) # 2) Look for enviornmental variable elif ( "CLIENT_GDRIVE_PATH" in os.environ and os.environ["CLIENT_GDRIVE_PATH"] - and os.environ["CLIENT_GDRIVE_PATH"].startswith(self.shared_drive) + and os.environ["CLIENT_GDRIVE_PATH"].startswith( + self.shared_drive + ) and os.environ["CLIENT_GDRIVE_PATH"] != self.shared_drive ): gpath = os.environ["CLIENT_GDRIVE_PATH"] @@ -2363,7 +2476,9 @@ def sync_root(self, sync_root): else: drive_canidate = "" # root else: - drive_canidate = gpath # one could only assume this is correct + drive_canidate = ( + gpath # one could only assume this is correct + ) self._sync_root = drive_canidate self.debug(f"using env-var sync path: {self._sync_root}") @@ -2380,8 +2495,12 @@ def sync_root(self, sync_root): ): # in a client directory, so map to ClientFolders or Relative Dir if self.shared_drive == self.default_shared_drive: # Map to ClientFolders/Client/matching/path... - relpath = os.path.relpath(guessfilepath, engforge_projects()) - self._sync_root = os.path.join(self.default_sync_path, relpath) + relpath = os.path.relpath( + guessfilepath, engforge_projects() + ) + self._sync_root = os.path.join( + self.default_sync_path, relpath + ) else: # Map to /matching/path... since client drive assumed root self._sync_root = os.path.relpath( @@ -2391,7 +2510,9 @@ def sync_root(self, sync_root): f"client infered sync path from given filepath : {self._sync_root}" ) - elif in_dropbox_dir(guessfilepath): # we're not in a client folder + elif in_dropbox_dir( + guessfilepath + ): # we're not in a client folder if self.shared_drive == self.default_shared_drive: # Map to Dropbox/whatever/path self._sync_root = os.path.relpath( @@ -2531,7 +2652,9 @@ def main_cli(): parser = argparse.ArgumentParser("Otter Drive Sync From Folder") - parser.add_argument("--shared-drive", "-D", default=None, help="shared drive name") + parser.add_argument( + "--shared-drive", "-D", default=None, help="shared drive name" + ) parser.add_argument( "--syncpath", "-S", @@ -2557,7 +2680,9 @@ def main_cli(): action="store_true", help="dry run, dont do any work! (WIP)", ) - parser.add_argument("--sync", action="store_true", help="sync the directories!") + parser.add_argument( + "--sync", action="store_true", help="sync the directories!" + ) parser.add_argument( "--remove-duplicates", action="store_true", help="remove any duplicates" ) diff --git a/engforge/datastores/reporting.py b/engforge/datastores/reporting.py index 23da8bb..1aa4108 100644 --- a/engforge/datastores/reporting.py +++ b/engforge/datastores/reporting.py @@ -213,7 +213,9 @@ def __init__(self, result_id_in, **kwargs): elif isinstance(val, (float, int)): # dynamic mapping if not numpy.isnan(val): - atobj = self.attr_class(result_id_in, key=key, value=val) + atobj = self.attr_class( + result_id_in, key=key, value=val + ) self.attr_store[key] = atobj # self[key] = atobj #dont use orm for upload! @@ -244,7 +246,10 @@ def all_fields(cls): if cls._mapped_component is None: return [] all_table_fields = set( - [attr.lower() for attr in cls._mapped_component.cls_all_property_labels()] + [ + attr.lower() + for attr in cls._mapped_component.cls_all_property_labels() + ] ) all_attr_fields = set( [ @@ -337,7 +342,9 @@ def __on_init__(self): self.db.ensure_database_exists(create_meta=False) self.base.metadata.bind = self.db.engine - self.base.metadata.reflect(self.db.engine, autoload=True, keep_existing=True) + self.base.metadata.reflect( + self.db.engine, autoload=True, keep_existing=True + ) self._component_cls_table_mapping = {} self.initalize() @@ -390,7 +397,9 @@ def filter_by_validators(self, attr_obj): [gtype in attr_obj.validator.type for gtype in (str, float, int)] ): return True - if any([gtype is attr_obj.validator.type for gtype in (str, float, int)]): + if any( + [gtype is attr_obj.validator.type for gtype in (str, float, int)] + ): return True return False @@ -399,7 +408,10 @@ def validator_to_column(self, attr_obj): [gtype in attr_obj.validator.type for gtype in (float, int)] ): return Column(Numeric, default=attr_obj.default, nullable=True) - if type(attr_obj.validator.type) is tuple and str in attr_obj.validator.type: + if ( + type(attr_obj.validator.type) is tuple + and str in attr_obj.validator.type + ): return Column( String(DEFAULT_STRING_LENGTH), default=attr_obj.default, @@ -431,9 +443,15 @@ def dict_attr( "__abstract__": False, } - if results_table is not None and not is_analysis: # analysis won't be mapped - default_attr["__tablename__"] = f"{results_table}_{table_abrv}{name}" - root_obj, _ = self.mapped_tables[results_table] # analysis won't be mapped + if ( + results_table is not None and not is_analysis + ): # analysis won't be mapped + default_attr["__tablename__"] = ( + f"{results_table}_{table_abrv}{name}" + ) + root_obj, _ = self.mapped_tables[ + results_table + ] # analysis won't be mapped backref_name = f"db_comp_attr_{name}" default_attr["result_id"] = Column( Integer, ForeignKey(f"{results_table}.id"), primary_key=True @@ -474,7 +492,9 @@ def default_attr(self, comp_cls, results_table=None): tbl_name = f"{results_table}_{table_abrv}{name}" backref_name = f"db_comp_{name}" root_obj, _ = self.mapped_tables[results_table] - default_attr["__tablename__"] = f"{results_table}_{table_abrv}{name}" + default_attr["__tablename__"] = ( + f"{results_table}_{table_abrv}{name}" + ) default_attr["result_id"] = Column( Integer, ForeignKey(f"{results_table}.id"), primary_key=True ) @@ -483,7 +503,9 @@ def default_attr(self, comp_cls, results_table=None): ) else: # its an analysis - default_attr["created"] = Column(DateTime, server_default=func.now()) + default_attr["created"] = Column( + DateTime, server_default=func.now() + ) default_attr["active"] = Column(Boolean(), server_default="t") default_attr["run_id"] = Column( String(36) @@ -495,7 +517,9 @@ def default_attr(self, comp_cls, results_table=None): # These use keys of system_property or attr and lower() so no spaces or capitals def all_possible_component_fields(self, cls): - all_table_fields = set([attr.lower() for attr in cls.cls_all_property_keys()]) + all_table_fields = set( + [attr.lower() for attr in cls.cls_all_property_keys()] + ) all_attr_fields = set( [attr.lower() for attr in cls.cls_all_attrs_fields().keys()] ) @@ -519,10 +543,14 @@ def component_attr(self, comp_cls, results_table=None): } ) - component_attr.update(self.default_attr(comp_cls, results_table=results_table)) + component_attr.update( + self.default_attr(comp_cls, results_table=results_table) + ) return component_attr - def db_component_dict(self, comp_cls, results_table=None, is_analysis=False): + def db_component_dict( + self, comp_cls, results_table=None, is_analysis=False + ): """returns the nessicary table types to make for reflection of input component class this method represents the dynamic creation of SQLA typing and attributes via __dict__ @@ -628,7 +656,9 @@ def check_or_create_db_type(self, tablename, type_tuple): return cls_db def map_component(self, dbcomponent, component): - self.debug(f"mapping {dbcomponent},{component} -> {dbcomponent.__tablename__}") + self.debug( + f"mapping {dbcomponent},{component} -> {dbcomponent.__tablename__}" + ) # add to internal mapping self._component_cls_table_mapping[dbcomponent.__tablename__] = ( dbcomponent, @@ -645,16 +675,22 @@ def map_component(self, dbcomponent, component): if isinstance(component, Analysis): component = component.__class__ if dbcomponent.__tablename__ not in atables: - self.info(f"adding analysis record { dbcomponent.__tablename__}") + self.info( + f"adding analysis record { dbcomponent.__tablename__}" + ) with self.db.session_scope() as sesh: rec = AnalysisRegistry(dbcomponent, component) sesh.add(rec) - elif isinstance(component, Component) or issubclass(component, Component): + elif isinstance(component, Component) or issubclass( + component, Component + ): if isinstance(component, Component): component = component.__class__ if dbcomponent.__tablename__ not in ctables: - self.info(f"adding comonent record { dbcomponent.__tablename__}") + self.info( + f"adding comonent record { dbcomponent.__tablename__}" + ) with self.db.session_scope() as sesh: rec = ComponentRegistry(dbcomponent, component) sesh.add(rec) @@ -733,7 +769,9 @@ def get_analysis_class(self, analysis): elif issubclass(analysis, Analysis): analysis_cls = analysis else: - self.warning("ensure-analysis: analysis not mapped, using direct input") + self.warning( + "ensure-analysis: analysis not mapped, using direct input" + ) analysis_cls = analysis return analysis_cls @@ -782,7 +820,9 @@ def gen(table): if lvl > 0: for comp in comps: cmp = comp["conf"] - data_gens[self.mapped_classes[cmp.__class__]] = gen(cmp.TABLE) + data_gens[self.mapped_classes[cmp.__class__]] = gen( + cmp.TABLE + ) # with self.db.scoped_session() as sesh: any_succeeded = True @@ -832,7 +872,9 @@ def gen(table): others.append(cmp_result) else: - any_succeeded = True # here's ur fricken evidence ur honor + any_succeeded = ( + True # here's ur fricken evidence ur honor + ) # the scoped session rolls back anything created this time :) if not any_succeeded: @@ -842,14 +884,14 @@ def gen(table): add_list = ( main_attrs + others - + flatten([list(item.attr_store.values()) for item in others]) + + flatten( + [list(item.attr_store.values()) for item in others] + ) ) sesh.add_all(add_list) - inx += ( - 1 # index += 1 is done at end of analysis so we should model that - ) + inx += 1 # index += 1 is done at end of analysis so we should model that except AvoidDuplicateAbortUpload: pass # this is fine @@ -901,7 +943,10 @@ def db_columns(cls): @classmethod def all_fields(cls): all_table_fields = set( - [attr.lower() for attr in cls._mapped_component.cls_all_property_labels()] + [ + attr.lower() + for attr in cls._mapped_component.cls_all_property_labels() + ] ) all_attr_fields = set( [ diff --git a/engforge/datastores/secrets.py b/engforge/datastores/secrets.py index 73d52fb..2a11c23 100644 --- a/engforge/datastores/secrets.py +++ b/engforge/datastores/secrets.py @@ -61,7 +61,9 @@ class Secrets(Configuration, metaclass=InputSingletonMeta): credential_key = attr.ib() # What to refer to this credential as credential_location = attr.ib() # Where to find the credential file - _aws_kms: pysecret.AWSSecret = None # a storage for the aws key managment object + _aws_kms: pysecret.AWSSecret = ( + None # a storage for the aws key managment object + ) public_up_level = True # makes the public cred record one level up (creds are stored protected). Yes its hacky but we have deadlines @@ -317,7 +319,9 @@ def local_credential_files(self): paths = [ os.path.join(root, acc) - for acc in itertools.accumulate(rel_paths, func=os.path.join, initial=root) + for acc in itertools.accumulate( + rel_paths, func=os.path.join, initial=root + ) ] creds_paths = {} @@ -385,7 +389,9 @@ def local_credential_packages(self): def decredential_file(filepath): for cred in self.creds_folders: if filepath.endswith(cred): - return str(pathlib.Path(filepath).parent) # Can't be dtwo types + return str( + pathlib.Path(filepath).parent + ) # Can't be dtwo types raw_dict = { decredential_file(key): creds @@ -397,10 +403,16 @@ def decredential_file(filepath): # assert all(list(map( ))) paths_contains_creds = lambda path: [ - cred for key, creds in raw_dict.items() for cred in creds if key in path + cred + for key, creds in raw_dict.items() + for cred in creds + if key in path ] - output = {key: paths_contains_creds(key) for key, creds in raw_dict.items()} + output = { + key: paths_contains_creds(key) + for key, creds in raw_dict.items() + } stage_packages[stage] = output return stage_packages @@ -489,7 +501,10 @@ def identity(self): else: self._ident = f"secrets-{self.stage_name}" - if self.stored_client_name and not self.stored_client_name in self._ident: + if ( + self.stored_client_name + and not self.stored_client_name in self._ident + ): self._log = None # lazy cycle log name return self._ident @@ -504,7 +519,9 @@ def sync(self): if "CLIENT_GDRIVE_SYNC" in os.environ: self.info("got CLIENT_GDRIVE_SYNC") - CLIENT_GDRIVE_SYNC = self.bool_from(os.environ["CLIENT_GDRIVE_SYNC"]) + CLIENT_GDRIVE_SYNC = self.bool_from( + os.environ["CLIENT_GDRIVE_SYNC"] + ) if "CLIENT_GMAIL" in os.environ: self.info("got CLIENT_GMAIL") @@ -539,7 +556,9 @@ def sync(self): and "SLACK_WEBHOOK_NOTIFICATION" in os.environ ): self.info("getting slack webhook") - self.SLACK_WEBHOOK_NOTIFICATION = os.environ["SLACK_WEBHOOK_NOTIFICATION"] + self.SLACK_WEBHOOK_NOTIFICATION = os.environ[ + "SLACK_WEBHOOK_NOTIFICATION" + ] # TODO: Add CLI Method diff --git a/engforge/dynamics.py b/engforge/dynamics.py index be56a4c..c7f73b3 100644 --- a/engforge/dynamics.py +++ b/engforge/dynamics.py @@ -50,7 +50,9 @@ class INDEX_MAP: oppo = {str: int, int: str} def __init__(self, datas: list): - self.data = [data if not data.startswith(".") else data[1:] for data in datas] + self.data = [ + data if not data.startswith(".") else data[1:] for data in datas + ] self.index = {} def get(self, key): @@ -72,17 +74,25 @@ def __call__(self, key): @staticmethod def indify(arr, *args): - return [arr[arg] if isinstance(arg, int) else arr.index(arg) for arg in args] + return [ + arr[arg] if isinstance(arg, int) else arr.index(arg) for arg in args + ] def remap_indexes_to(self, new_index, *args, invert=False, old_data=None): if old_data is None: old_data = self.data opt1 = {arg: self.indify(old_data, arg)[0] for arg in args} opt2 = { - arg: self.indify(old_data, val)[0] if (not isinstance(val, str)) else val + arg: ( + self.indify(old_data, val)[0] + if (not isinstance(val, str)) + else val + ) for arg, val in opt1.items() } - oop1 = {arg: self.indify(new_index, val)[0] for arg, val in opt2.items()} + oop1 = { + arg: self.indify(new_index, val)[0] for arg, val in opt2.items() + } oop2 = { arg: ( self.indify(new_index, val)[0] @@ -178,11 +188,15 @@ def dynamic_output_size(self): @property def dynamic_state(self) -> np.array: - return np.array([getattr(self, var, np.nan) for var in self.dynamic_state_vars]) + return np.array( + [getattr(self, var, np.nan) for var in self.dynamic_state_vars] + ) @property def dynamic_input(self) -> np.array: - return np.array([getattr(self, var, np.nan) for var in self.dynamic_input_vars]) + return np.array( + [getattr(self, var, np.nan) for var in self.dynamic_input_vars] + ) @property def dynamic_output(self) -> np.array: @@ -196,11 +210,15 @@ def create_state_matrix(self, **kwargs) -> np.ndarray: def create_input_matrix(self, **kwargs) -> np.ndarray: """creates the input matrix for the system, called B""" - return np.zeros((self.dynamic_state_size, max(self.dynamic_input_size, 1))) + return np.zeros( + (self.dynamic_state_size, max(self.dynamic_input_size, 1)) + ) def create_output_matrix(self, **kwargs) -> np.ndarray: """creates the input matrix for the system, called C""" - return np.zeros((max(self.dynamic_output_size, 1), self.dynamic_state_size)) + return np.zeros( + (max(self.dynamic_output_size, 1), self.dynamic_state_size) + ) def create_feedthrough_matrix(self, **kwargs) -> np.ndarray: """creates the input matrix for the system, called D""" @@ -513,7 +531,9 @@ def ref_dXdt(self, name: str): accss.__name__ = f"ref_dXdt_{name}" return Ref(self, accss) - def determine_nearest_stationary_state(self, t=0, X=None, U=None) -> np.ndarray: + def determine_nearest_stationary_state( + self, t=0, X=None, U=None + ) -> np.ndarray: """determine the nearest stationary state""" if X is None: diff --git a/engforge/eng/costs.py b/engforge/eng/costs.py index e8d13a3..442ec10 100644 --- a/engforge/eng/costs.py +++ b/engforge/eng/costs.py @@ -56,6 +56,7 @@ class Parent(System,CostModel) import collections import re + class CostLog(LoggingMixin): pass @@ -723,7 +724,7 @@ def internal_references(self, recache=True, numeric_only=False): if self._cost_references: props.update(**self._cost_references) - #lookup ref from the cost categories dictionary, recreate every time + # lookup ref from the cost categories dictionary, recreate every time if self._cost_categories: for key, refs in self._cost_categories.items(): props[key] = Ref( @@ -894,7 +895,7 @@ def lifecycle_output(self) -> dict: for c in lc.columns: if "category" not in c and "cost" not in c: continue - tot = lc[c].sum()#lifecycle cost + tot = lc[c].sum() # lifecycle cost if "category" in c: c_ = c.replace("category.", "") lifecat[c_] = tot @@ -1215,60 +1216,70 @@ def output(self) -> float: def cost_category_store(self): D = collections.defaultdict(list) Acat = set() - for catkey,cdict in self._cost_categories.items(): + for catkey, cdict in self._cost_categories.items(): for ci in cdict: - cprop = getattr(ci.comp.__class__,ci.key) + cprop = getattr(ci.comp.__class__, ci.key) ccat = set(cprop.cost_categories.copy()) Acat = Acat.union(ccat) - D[f'{ci.comp.classname}|{ci.key:>36}'] = ccat - return D,Acat - - def create_cost_graph(self,plot=True): + D[f"{ci.comp.classname}|{ci.key:>36}"] = ccat + return D, Acat + + def create_cost_graph(self, plot=True): """creates a graph of the cost model using network X and display it""" import collections import networkx as nx D = collections.defaultdict(dict) - for catkey,cdict in self._cost_categories.items(): + for catkey, cdict in self._cost_categories.items(): for ci in cdict: - D[catkey][(ci.comp.classname,ci.key)] = ci + D[catkey][(ci.comp.classname, ci.key)] = ci G = nx.Graph() - for d,dk in D.items(): + for d, dk in D.items(): print(d.upper()) - cat = d.replace('category.','') - G.add_node(cat,category=cat) - for kk,r in dk.items(): + cat = d.replace("category.", "") + G.add_node(cat, category=cat) + for kk, r in dk.items(): cmp = kk[0] edge = kk[1] - if cmp not in G.nodes: - G.add_node(cmp,component=cmp) - G.add_edge(cmp,cat,cost=edge) - #print(kk) + if cmp not in G.nodes: + G.add_node(cmp, component=cmp) + G.add_edge(cmp, cat, cost=edge) + # print(kk) - #pos = nx.nx_agraph.graphviz_layout(G) - #nx.draw(G, pos=pos) - #nx.draw(G,with_labels=True) + # pos = nx.nx_agraph.graphviz_layout(G) + # nx.draw(G, pos=pos) + # nx.draw(G,with_labels=True) if plot: - categories = nx.get_node_attributes(G, 'category').keys() - components = nx.get_node_attributes(G, 'component').keys() + categories = nx.get_node_attributes(G, "category").keys() + components = nx.get_node_attributes(G, "component").keys() cm = [] for nd in G: if nd in categories: - cm.append('cyan') + cm.append("cyan") else: - cm.append('pink') - - pos = nx.spring_layout(G,k=0.2, iterations=20,scale=1) - nx.draw(G,node_color=cm,with_labels=True,pos=pos,arrows=True,font_size=10,font_color='0.09',font_weight='bold',node_size=200) + cm.append("pink") + + pos = nx.spring_layout(G, k=0.2, iterations=20, scale=1) + nx.draw( + G, + node_color=cm, + with_labels=True, + pos=pos, + arrows=True, + font_size=10, + font_color="0.09", + font_weight="bold", + node_size=200, + ) return G - + def cost_matrix(self): - D,Cats = self.cost_category_store + D, Cats = self.cost_category_store X = list(sorted(Cats)) C = list(sorted(D.keys())) M = [] @@ -1279,124 +1290,142 @@ def cost_matrix(self): Mx = numpy.array(M) X = numpy.array(X) C = numpy.array(C) - return Mx,X,C - + return Mx, X, C + def create_cost_category_table(self): """creates a table of costs and categories""" - Mx,X,C = self.cost_matrix() + Mx, X, C = self.cost_matrix() - fig,ax = subplots(figsize=(12,12)) + fig, ax = subplots(figsize=(12, 12)) - Mc = numpy.nansum(Mx,axis=0) + Mc = numpy.nansum(Mx, axis=0) x = numpy.argsort(Mc) Xs = X[x] - ax.imshow(Mx[:,x]) - ax.set_yticklabels(C,fontdict={'family':'monospace','size':8}) + ax.imshow(Mx[:, x]) + ax.set_yticklabels(C, fontdict={"family": "monospace", "size": 8}) ax.set_yticks(numpy.arange(len(C))) - ax.set_xticklabels(Xs,fontdict={'family':'monospace','size':8}) + ax.set_xticklabels(Xs, fontdict={"family": "monospace", "size": 8}) ax.set_xticks(numpy.arange(len(Xs))) - ax.grid(which='major',linestyle=':',color='k',zorder=0) + ax.grid(which="major", linestyle=":", color="k", zorder=0) xticks(rotation=90) fig.tight_layout() - def determine_exclusive_cost_categories(self,include_categories=None,ignore_categories:set=None,min_groups:int=2,max_group_size=None,min_score=0.95,include_item_cost=False): + def determine_exclusive_cost_categories( + self, + include_categories=None, + ignore_categories: set = None, + min_groups: int = 2, + max_group_size=None, + min_score=0.95, + include_item_cost=False, + ): """looks at all possible combinations of cost categories, scoring them based on coverage of costs, and not allowing any double accounting of costs. This is an NP-complete problem and will take a long time for large numbers of items. You can add ignore_categories to ignore certain categories""" import itertools - Mx,X,C = self.cost_matrix() + Mx, X, C = self.cost_matrix() bad = [] solutions = [] - inx = {k:i for i,k in enumerate(X)} + inx = {k: i for i, k in enumerate(X)} + assert include_categories is None or set(X).issuperset( + include_categories + ), "include_categories must be subset of cost categories" - - assert include_categories is None or set(X).issuperset(include_categories), 'include_categories must be subset of cost categories' - - #ignore categories + # ignore categories if ignore_categories: X = [x for x in X if x not in ignore_categories] if include_categories: - #dont include them in pair since they are added to the group explicitly + # dont include them in pair since they are added to the group explicitly X = [x for x in X if x not in include_categories] if not include_item_cost: - C = [c for c in C if 'item_cost' not in c] + C = [c for c in C if "item_cost" not in c] Num_Costs = len(C) goal_score = Num_Costs * min_score - NumCats = len(X)//2 + NumCats = len(X) // 2 GroupSize = NumCats if max_group_size is None else max_group_size - for ni in range(min_groups,GroupSize): - print(f'level {ni}/{GroupSize}| {len(solutions)} answers') - for cgs in itertools.combinations(X,ni): + for ni in range(min_groups, GroupSize): + print(f"level {ni}/{GroupSize}| {len(solutions)} answers") + for cgs in itertools.combinations(X, ni): val = None - #make the set with included if needed + # make the set with included if needed scg = set(cgs) if include_categories: scg = scg.union(include_categories) - #skip bad groups + # skip bad groups if any([b.issubset(scg) for b in bad]): - #print(f'skipping {cgs}') - #sys.stdout.write('.') + # print(f'skipping {cgs}') + # sys.stdout.write('.') continue - good = True #innocent till guilty + good = True # innocent till guilty for cg in cgs: - xi = Mx[:,inx[cg]].copy() - xi[np.isnan(xi)] = 0 + xi = Mx[:, inx[cg]].copy() + xi[np.isnan(xi)] = 0 if val is None: val = xi else: val = val + xi - #determine if any overlap (only pair level) + # determine if any overlap (only pair level) if np.nanmax(val) > 1: - print(f'bad {cgs}') + print(f"bad {cgs}") bad.append(scg) good = False break score = np.nansum(val) if good and score > goal_score: - print(f'found good: {scg}') - solutions.append({'grp':scg,'score':score,'gsize':ni}) + print(f"found good: {scg}") + solutions.append({"grp": scg, "score": score, "gsize": ni}) - return solutions + return solutions - def cost_categories_from_df(self,df): + def cost_categories_from_df(self, df): categories = set() for val in df.columns: - m = re.match(re.compile('economics\.lifecycle\.category\.(?s:[a-z]*)$'),val) + m = re.match( + re.compile("economics\.lifecycle\.category\.(?s:[a-z]*)$"), val + ) if m: categories.add(val) - return categories + return categories - def plot_cost_categories(self,df,group,cmap='tab20c',make_title=None,ax=None): + def plot_cost_categories( + self, df, group, cmap="tab20c", make_title=None, ax=None + ): categories = self.cost_categories_from_df(df) from matplotlib import cm - #if grps: - #assert len(grps) == len(y_vars), 'groups and y_vars must be same length' - #assert all([g in categories for g in grps]), 'all groups must be in categories' - #TODO: project costs onto y_vars - #TODO: ensure groups and y_vars are same length + # if grps: + # assert len(grps) == len(y_vars), 'groups and y_vars must be same length' + # assert all([g in categories for g in grps]), 'all groups must be in categories' + # TODO: project costs onto y_vars + # TODO: ensure groups and y_vars are same length color = cm.get_cmap(cmap) - styles = {c.replace('economics.lifecycle.category.',''):{'color': color(i/ len(categories))} for i,c in enumerate(categories)} + styles = { + c.replace("economics.lifecycle.category.", ""): { + "color": color(i / len(categories)) + } + for i, c in enumerate(categories) + } if make_title is None: + def make_title(row): return f'{row["name"]}x{row["num_items"]} @{"floating" if row["ldepth"]>50 else "fixed"}' - #for j,grp in enumerate(groups): + # for j,grp in enumerate(groups): figgen = False if ax is None: figgen = True - fig,ax = subplots(figsize=(12,8)) + fig, ax = subplots(figsize=(12, 8)) else: fig = ax.get_figure() @@ -1405,48 +1434,53 @@ def make_title(row): data = {} i = 0 - - for inx,row in df.iterrows(): + for inx, row in df.iterrows(): i += 1 - tc = row['economics.summary.total_cost'] - cat_costs = {k.replace('economics.lifecycle.category.',''):row[k] for k in categories} - #print(i,cat_costs) - - spec_costs = {k:v for k,v in cat_costs.items() if k in group} - pos_costs = {k:v for k,v in spec_costs.items() if v>=0} - neg_costs = {k:v for k,v in spec_costs.items() if k not in pos_costs} + tc = row["economics.summary.total_cost"] + cat_costs = { + k.replace("economics.lifecycle.category.", ""): row[k] + for k in categories + } + # print(i,cat_costs) + + spec_costs = {k: v for k, v in cat_costs.items() if k in group} + pos_costs = {k: v for k, v in spec_costs.items() if v >= 0} + neg_costs = { + k: v for k, v in spec_costs.items() if k not in pos_costs + } neg_amt = sum(list(neg_costs.values())) pos_amt = sum(list(pos_costs.values())) data[i] = spec_costs.copy() - com = {'x':i,'width':0.5,'linewidth':0} + com = {"x": i, "width": 0.5, "linewidth": 0} cur = neg_amt - for k,v in neg_costs.items(): - opt = {} if i != 1 else {'label':k} - ax.bar(height=abs(v),bottom=cur,**com,**styles[k],**opt) + for k, v in neg_costs.items(): + opt = {} if i != 1 else {"label": k} + ax.bar(height=abs(v), bottom=cur, **com, **styles[k], **opt) cur += abs(v) - for k,v in pos_costs.items(): - opt = {} if i != 1 else {'label':k} - ax.bar(height=abs(v),bottom=cur,**com,**styles[k],**opt) + for k, v in pos_costs.items(): + opt = {} if i != 1 else {"label": k} + ax.bar(height=abs(v), bottom=cur, **com, **styles[k], **opt) cur += abs(v) - xticks.append(com['x']) + xticks.append(com["x"]) titles.append(make_title(row)) - #Format the chart - ax.legend(loc='upper right') - ax.set_xlim([0,i+max(2,0.2*i)]) + # Format the chart + ax.legend(loc="upper right") + ax.set_xlim([0, i + max(2, 0.2 * i)]) ax.set_xticks(xticks) - ax.set_xticklabels(titles,rotation=90) + ax.set_xticklabels(titles, rotation=90) ylim = ax.get_ylim() - ylim = ylim[0]-0.05*abs(ylim[0]),ylim[1]+0.05*abs(ylim[1]) - ax.set_yticks(numpy.linspace(*ylim,50),minor=True) - ax.grid(which='major',linestyle='--',color='k',zorder=0) - ax.grid(which='minor',linestyle=':',color='k',zorder=0) - if figgen: fig.tight_layout() - return {'fig':fig,'ax':ax,'data':data} - + ylim = ylim[0] - 0.05 * abs(ylim[0]), ylim[1] + 0.05 * abs(ylim[1]) + ax.set_yticks(numpy.linspace(*ylim, 50), minor=True) + ax.grid(which="major", linestyle="--", color="k", zorder=0) + ax.grid(which="minor", linestyle=":", color="k", zorder=0) + if figgen: + fig.tight_layout() + return {"fig": fig, "ax": ax, "data": data} + # TODO: add costs for iterable components (wide/narrow modes) # if isinstance(conf,ComponentIter): diff --git a/engforge/eng/fluid_material.py b/engforge/eng/fluid_material.py index 2adbd8f..6bfb954 100644 --- a/engforge/eng/fluid_material.py +++ b/engforge/eng/fluid_material.py @@ -254,7 +254,9 @@ def material(self) -> str: @classmethod def setup(cls): try: - CoolProp.apply_simple_mixing_rule(cls.material, cls.material2, "linear") + CoolProp.apply_simple_mixing_rule( + cls.material, cls.material2, "linear" + ) except Exception as e: pass # self.error(e,'issue setting mixing rule, but continuting.') diff --git a/engforge/eng/geometry.py b/engforge/eng/geometry.py index 55ab469..45e268b 100644 --- a/engforge/eng/geometry.py +++ b/engforge/eng/geometry.py @@ -36,7 +36,7 @@ # generic cross sections from # https://mechanicalbase.com/area-moment-of-inertia-calculator-of-certain-cross-sectional-shapes/ -_temp = os.path.join(os.path.expanduser('~'),'.forge_tmp') +_temp = os.path.join(os.path.expanduser("~"), ".forge_tmp") temp_path = os.path.join(_temp, "shapely_sections") section_cache = EnvVariable( @@ -45,7 +45,7 @@ desc="directory to cache section properties", ) if "FORGE_CACHE" not in os.environ and not os.path.exists(temp_path): - os.makedirs(temp_path,0o741,exist_ok=True) + os.makedirs(temp_path, 0o741, exist_ok=True) section_cache.info(f"loading section from {section_cache.secret}") @@ -371,7 +371,9 @@ class ShapelySection(Profile2D): # Mesh sizing coarse: bool = attrs.field(default=False) mesh_extent_decimation = attrs.field(default=100) - min_mesh_angle: float = attrs.field(default=20) # below 20.7 garunteed to work + min_mesh_angle: float = attrs.field( + default=20 + ) # below 20.7 garunteed to work min_mesh_size: float = attrs.field(default=1e-5) # multiply by min goal_elements: float = attrs.field(default=1000) # multiply by min _mesh_size: float = attrs.field(default=attrs.Factory(get_mesh_size, True)) @@ -497,7 +499,9 @@ def from_cache(cls, hash_id): def prediction_weights(self, df, window, initial_weight=10): weights = numpy.ones(min(len(df), window)) weights[0] = initial_weight**2 # zero value is important! - weights[: getattr(self, "N_base", 100)] = initial_weight # then base values + weights[: getattr(self, "N_base", 100)] = ( + initial_weight # then base values + ) if hasattr(self, "N_pareto"): weights[: getattr(self, "N_pareto")] = ( initial_weight**0.5 @@ -505,7 +509,9 @@ def prediction_weights(self, df, window, initial_weight=10): # Dont emphasise fit above max margin dm = (df.fail_frac - self.max_margin).to_numpy() penalize_inx = dm > 0 - weights[penalize_inx] = np.maximum(1.0 / ((1.0 + dm[penalize_inx])), 0.1) + weights[penalize_inx] = np.maximum( + 1.0 / ((1.0 + dm[penalize_inx])), 0.1 + ) return weights def _subsample_data(self, X, y, window, weights): @@ -613,7 +619,9 @@ def mesh_section(self): self._A = self._sec.get_area() if self.material: - self._Ixx, self._Iyy, self._Ixy = self._sec.get_eic(e_ref=self.material) + self._Ixx, self._Iyy, self._Ixy = self._sec.get_eic( + e_ref=self.material + ) self._J = self._sec.get_ej() else: self._Ixx, self._Iyy, self._Ixy = self._sec.get_ic() @@ -662,7 +670,9 @@ def plot_mesh(self): def plot_mesh(self): return self._sec.plot_centroids() - def calculate_stress(self, n=0, vx=0, vy=0, mxx=0, myy=0, mzz=0, **kw) -> float: + def calculate_stress( + self, n=0, vx=0, vy=0, mxx=0, myy=0, mzz=0, **kw + ) -> float: return calculate_stress( self, n=n, vx=vx, vy=vy, mxx=mxx, myy=myy, mzz=mzz, **kw ) @@ -694,7 +704,9 @@ def estimate_stress( do_calc = not self.prediction or not self._fitted or under_size if do_calc or force_calc: if self._do_print and self.prediction: - print(f"calc till {len(self.prediction_records)} <= {min_est_records}") + print( + f"calc till {len(self.prediction_records)} <= {min_est_records}" + ) stress = calculate_stress( self, n=n, vx=vx, vy=vy, mxx=mxx, myy=myy, mzz=mzz, value=value ) @@ -719,7 +731,9 @@ def estimate_stress( # calculate stress if close to failure within saftey margin err = 1 - val mrg = self.fail_frac_criteria(calc_margin=calc_margin) - do_calc = abs(err) <= mrg or all([calc_every, (Nrec % calc_every) == 0]) + do_calc = abs(err) <= mrg or all( + [calc_every, (Nrec % calc_every) == 0] + ) oob = val <= self.max_margin and self.check_out_of_domain(data, 0.1) if self._do_print: self.info( @@ -864,7 +878,9 @@ def solve_fail(self, fail_parm, base_kw, guess=None, tol=1e-4, mult=1): return 1e6 # Determine Outer Bound Of Failures - def determine_failure_front(self, pareto_inx=[0.5, 0.1], pareto_front=False): + def determine_failure_front( + self, pareto_inx=[0.5, 0.1], pareto_front=False + ): self.info( f"determining failure front for cross section, with pareto inx: {pareto_inx}" ) @@ -933,7 +949,8 @@ def basis_expand( q = max(wt, 1) if normalize else 1 inxs = [self._prediction_parms.index(p) for p in parms] base_kw = { - p: w * self._basis[i] / q for p, w, i in zip(parms, weight, inxs) + p: w * self._basis[i] / q + for p, w, i in zip(parms, weight, inxs) } # print(base_kw) @@ -966,7 +983,9 @@ def train_until_valid(self, print_interval=50, max_iter=1000, est=False): self.calculate_stress(**inp) if i % print_interval == 0: - self.info(f"training... current error: {self._training_history[-1]}") + self.info( + f"training... current error: {self._training_history[-1]}" + ) i += 1 if i >= max_iter: self.info(f"training... max iterations reached") @@ -1016,7 +1035,9 @@ def find_radii(targetth): poss2 = int((poss + 1) % Imax) x_ = np.array([r[poss], r[poss2]]) y_ = np.array([inx[poss], inx[poss2]]) - itarget = (0 - x_[0]) * (y_[1] - y_[0]) / (x_[1] - x_[0]) + y_[0] + itarget = (0 - x_[0]) * (y_[1] - y_[0]) / (x_[1] - x_[0]) + y_[ + 0 + ] r_ = np.interp(itarget, inx2, R) out.add(round(r_, precision)) # print(itarget,r_) diff --git a/engforge/eng/prediction.py b/engforge/eng/prediction.py index 27b6846..c848df5 100644 --- a/engforge/eng/prediction.py +++ b/engforge/eng/prediction.py @@ -85,7 +85,9 @@ def add_prediction_record( and std != 0 and not near_zero ): - g = 3 * min((abs(avg) - std) / abs(std), 1) # negative when std > avg + g = 3 * min( + (abs(avg) - std) / abs(std), 1 + ) # negative when std > avg a = min(abs(dev) / std / J, 1) # prob accept should be based on difference from average prob_deny = np.e**g @@ -93,7 +95,9 @@ def add_prediction_record( std_choice = std avg_choice = avg dev_choice = dev - choice = random.choices([True, False], [prob_accept, prob_deny])[0] + choice = random.choices( + [True, False], [prob_accept, prob_deny] + )[0] cur_stat["avg"] = avg + dev / N cur_stat["var"] = var + (dev**2 - var) / N @@ -149,7 +153,9 @@ def check_out_of_domain(self, record, extra_margin=1, target_items=1000): std = var**0.5 std_err = std / len(self.prediction_records) # check out of bounds - g = 3 * min((abs(avg) - std) / abs(std), 1) # negative when std > avg + g = 3 * min( + (abs(avg) - std) / abs(std), 1 + ) # negative when std > avg a = min(abs(dev) / std / J, 1) near_zero = abs(rec_val) / max(abs(avg), 1) <= 1e-3 prob_deny = np.e**g @@ -302,7 +308,9 @@ def observe_and_predict(self, row): model = mod_dict["mod"] if parm not in row: continue - X = pandas.DataFrame([{parm: row[parm] for parm in self._prediction_parms}]) + X = pandas.DataFrame( + [{parm: row[parm] for parm in self._prediction_parms}] + ) y = row[parm] x = abs(model.predict(X)) if y == 0: @@ -356,7 +364,10 @@ def check_and_retrain(self, records, min_rec=None): model = mod_dict["mod"] N = mod_dict["N"] # - if Nrec > N * self._re_train_frac or (Nrec - N) > self._re_train_maxiter: + if ( + Nrec > N * self._re_train_frac + or (Nrec - N) > self._re_train_maxiter + ): self.train_compare(df) return diff --git a/engforge/eng/structure.py b/engforge/eng/structure.py index 1c4c513..3ad40ce 100644 --- a/engforge/eng/structure.py +++ b/engforge/eng/structure.py @@ -61,7 +61,9 @@ class StructureLog(LoggingMixin): k: v for k, v in filter( lambda kv: ( - issubclass(kv[1], geometry.Geometry) if type(kv[1]) is type else False + issubclass(kv[1], geometry.Geometry) + if type(kv[1]) is type + else False ), sections.__dict__.items(), ) @@ -325,7 +327,9 @@ def create_structure(self): pass # Execution - def execute(self, combos: list = None, save=True, record=True, *args, **kwargs): + def execute( + self, combos: list = None, save=True, record=True, *args, **kwargs + ): """wrapper allowing saving of data by load combo""" # the string input case, with csv support @@ -348,7 +352,9 @@ def execute(self, combos: list = None, save=True, record=True, *args, **kwargs): # run the analysis # self.index += 1 - combo_possible = self.struct_pre_execute(combo) # can change combo + combo_possible = self.struct_pre_execute( + combo + ) # can change combo if combo_possible: self.current_combo = combo_possible else: @@ -420,7 +426,9 @@ def struct_root_failure( res = target - ff if sols is not None: - sols.append({"x": x, "ff": ff, "obj": res, "kw": bkw, "mf": mf, "bf": bf}) + sols.append( + {"x": x, "ff": ff, "obj": res, "kw": bkw, "mf": mf, "bf": bf} + ) self.info(f"ran: {bkw} -> {ff:5.4f} | {res:5.4f}") return res @@ -746,7 +754,9 @@ def add_member(self, name, node1, node2, section, material=None, **kwargs): elif hasattr(section, "material"): material = section.material else: - raise ValueError("material not defined as input or from default sources!") + raise ValueError( + "material not defined as input or from default sources!" + ) uid = material.unique_id if uid not in self._materials: @@ -756,7 +766,9 @@ def add_member(self, name, node1, node2, section, material=None, **kwargs): ) self._materials[uid] = material - beam_attrs = {k: v for k, v in kwargs.items() if k in Beam.input_attrs()} + beam_attrs = { + k: v for k, v in kwargs.items() if k in Beam.input_attrs() + } kwargs = {k: v for k, v in kwargs.items() if k not in beam_attrs} B = beam = Beam( @@ -780,7 +792,9 @@ def add_member(self, name, node1, node2, section, material=None, **kwargs): ) if self.add_gravity_force: - beam.apply_gravity_force(z_dir=self.gravity_dir, z_mag=self.gravity_scalar) + beam.apply_gravity_force( + z_dir=self.gravity_dir, z_mag=self.gravity_scalar + ) return beam @@ -818,7 +832,9 @@ def add_member_with( self.beams[name] = beam if self.add_gravity_force: - beam.apply_gravity_force(z_dir=self.gravity_dir, z_mag=self.gravity_scalar) + beam.apply_gravity_force( + z_dir=self.gravity_dir, z_mag=self.gravity_scalar + ) self.frame.add_member( name, @@ -932,7 +948,7 @@ def panels_cost(self): def beam_cost(self) -> float: """sum of all beams cost""" return sum([bm.cost for bm in self.beams.values()]) - + @system_property def structure_cost(self) -> float: """sum of all beams and quad cost""" @@ -1431,7 +1447,9 @@ def current_failures( summary["mesh_stable"] = mesh_success # update failures - mesh_failures_count = len([v for k, v in mesh_failures.items() if v > 0.99]) + mesh_failures_count = len( + [v for k, v in mesh_failures.items() if v > 0.99] + ) max_fail_frac = max( [0] @@ -1694,7 +1712,10 @@ def run_failure_sections( d_["stress_results"] = sss = s.get_stress() d_["stress_vm_max"] = max([max(ss["sig_vm"]) for ss in sss]) d_["fail_frac"] = ff = max( - [max(ss["sig_vm"] / beam.material.allowable_stress) for ss in sss] + [ + max(ss["sig_vm"] / beam.material.allowable_stress) + for ss in sss + ] ) d_["fails"] = fail = ff > 1 / SF @@ -1722,7 +1743,9 @@ def run_failure_sections( # Remote Sync Util (locally run with section) -def run_combo_failure_analysis(inst, combo, run_full: bool = False, SF: float = 1.0): +def run_combo_failure_analysis( + inst, combo, run_full: bool = False, SF: float = 1.0 +): """runs a single load combo and adds 2d section failures""" inst.resetSystemLogs() # use parallel failure section @@ -1835,7 +1858,9 @@ def run_failure_sections( # Remote Analysis -def parallel_run_failure_analysis(struct, SF=1.0, run_full=False, purge=False, **kw): +def parallel_run_failure_analysis( + struct, SF=1.0, run_full=False, purge=False, **kw +): """ Failure Determination: Beam Stress Estiates are used to determine if a 2D FEA beam/combo analysis should be run. The maximum beam vonmises stress is compared the the beam material allowable stress / saftey factor. @@ -1946,7 +1971,9 @@ def remote_section( d_["stress_analysis"] = s d_["stress_results"] = sss d_["stress_vm_max"] = max([max(ss["sig_vm"]) for ss in sss]) - d_["fail_frac"] = ff = max([max(ss["sig_vm"] / allowable_stress) for ss in sss]) + d_["fail_frac"] = ff = max( + [max(ss["sig_vm"] / allowable_stress) for ss in sss] + ) d_["fails"] = fail = ff > 1 / SF return d_ @@ -2176,7 +2203,9 @@ def remote_failure_sections( allowable = r["allowable"] cur.append( - remote_section.remote(beam_ref, beamnm, forces, allowable, c, x, SF=SF) + remote_section.remote( + beam_ref, beamnm, forces, allowable, c, x, SF=SF + ) ) # run the damn thing @@ -2192,7 +2221,9 @@ def remote_failure_sections( secton_results[res["beam"]][(combo, x)] = res if fail: - log.warning(f"beam {beamnm} failed @ {x*100:3.0f}%| {c}") + log.warning( + f"beam {beamnm} failed @ {x*100:3.0f}%| {c}" + ) if fail_fast: return secton_results if not run_full: diff --git a/engforge/eng/structure_beams.py b/engforge/eng/structure_beams.py index c50e818..2bfc088 100644 --- a/engforge/eng/structure_beams.py +++ b/engforge/eng/structure_beams.py @@ -47,11 +47,15 @@ def rotation_matrix_from_vectors(vec1, vec2): if any(v): # if not all zeros then c = numpy.dot(a, b) s = numpy.linalg.norm(v) - kmat = numpy.array([[0, -v[2], v[1]], [v[2], 0, -v[0]], [-v[1], v[0], 0]]) + kmat = numpy.array( + [[0, -v[2], v[1]], [v[2], 0, -v[0]], [-v[1], v[0], 0]] + ) return numpy.eye(3) + kmat + kmat.dot(kmat) * ((1 - c) / (s**2)) else: - return numpy.eye(3) # cross of all zeros only occurs on identical directions + return numpy.eye( + 3 + ) # cross of all zeros only occurs on identical directions @forge @@ -150,7 +154,10 @@ def update_section(self, section): self.debug(f"checking input values") assert all( - [val is not None for val in (self.in_Iy, self.in_Ix, self.in_J, self.in_A)] + [ + val is not None + for val in (self.in_Iy, self.in_Ix, self.in_J, self.in_A) + ] ) # Assemble tensor @@ -302,7 +309,7 @@ def section_mass(self) -> float: def mass(self) -> float: return self.material.density * self.Vol - #@system_property(category="mfg,material,beams") + # @system_property(category="mfg,material,beams") @system_property def cost(self) -> float: return self.mass * self.material.cost_per_kg @@ -395,7 +402,9 @@ def estimate_stress(self, force_calc=True, **forces): else: return self._fallback_estimate_stress(**forces) - def _fallback_estimate_stress(self, n, vx, vy, mxx, myy, mzz, SM=5, **extra): + def _fallback_estimate_stress( + self, n, vx, vy, mxx, myy, mzz, SM=5, **extra + ): """sum the absolute value of each stress component. This isn't accurate but each value here should represent the worst sections, and take the 1-norm to max for each type of stress""" if self.section.x_bounds is None: @@ -604,7 +613,9 @@ def max_moment_y(self) -> float: return self.member.max_moment("My", self.structure.current_combo) # Load Application - def get_valid_force_choices(only_local=False, only_global=False, use_moment=True): + def get_valid_force_choices( + only_local=False, only_global=False, use_moment=True + ): if only_local or only_global: assert only_global != only_local, "choose local or global" @@ -646,7 +657,9 @@ def apply_pt_load(self, x_frac, case=None, **kwargs): self.member.name, Fkey, Fval, x, case=case ) - def apply_distributed_load(self, start_factor=1, end_factor=1, case=None, **kwargs): + def apply_distributed_load( + self, start_factor=1, end_factor=1, case=None, **kwargs + ): """add forces in global vector""" if case is None: case = self.structure.default_case @@ -695,7 +708,9 @@ def apply_local_distributed_load( case=case, ) - def apply_gravity_force_distribution(self, sv=1, ev=1, z_dir="FZ", z_mag=-1): + def apply_gravity_force_distribution( + self, sv=1, ev=1, z_dir="FZ", z_mag=-1 + ): # TODO: ensure that integral of sv, ev is 1, and all positive self.debug(f"applying gravity distribution to {self.name}") for case in self.structure.gravity_cases: diff --git a/engforge/eng/thermodynamics.py b/engforge/eng/thermodynamics.py index c472fb6..1150136 100644 --- a/engforge/eng/thermodynamics.py +++ b/engforge/eng/thermodynamics.py @@ -195,7 +195,10 @@ def Tout(self) -> float: return self.Tin * ( 1 - self.efficiency - * (1.0 - (1 / self.pressure_ratio) ** ((self.gamma - 1.0) / self.gamma)) + * ( + 1.0 + - (1 / self.pressure_ratio) ** ((self.gamma - 1.0) / self.gamma) + ) ) # return self.Tin * self.pressure_ratio**((self.gamma-1.0)/self.gamma)/ self.efficiency diff --git a/engforge/engforge_attributes.py b/engforge/engforge_attributes.py index e496180..2e9872d 100644 --- a/engforge/engforge_attributes.py +++ b/engforge/engforge_attributes.py @@ -71,7 +71,9 @@ def collect_inst_attributes(self, **kw): return out @classmethod - def _get_init_attrs_data(cls, subclass_of: type, exclude=False,attr_type=False): + def _get_init_attrs_data( + cls, subclass_of: type, exclude=False, attr_type=False + ): choose = issubclass if exclude: choose = lambda ty, type_set: not issubclass(ty, type_set) @@ -143,7 +145,9 @@ def check_ref_slot_type(cls, sys_key: str) -> list: sub_clss = cls._extract_type(slts[fst].type) out = [] for acpt in sub_clss: - if isinstance(acpt, type) and issubclass(acpt, Configuration): + if isinstance(acpt, type) and issubclass( + acpt, Configuration + ): vals = acpt.check_ref_slot_type(".".join(rem)) # print(f'recursive find {acpt}.{rem} = {vals}') if vals: @@ -167,9 +171,9 @@ def slot_refs(cls, recache=False): return o @classmethod - def slots_attributes(cls,attr_type=False) -> typing.Dict[str, "Attribute"]: + def slots_attributes(cls, attr_type=False) -> typing.Dict[str, "Attribute"]: """Lists all slots attributes for class""" - return cls._get_init_attrs_data(Slot,attr_type=attr_type) + return cls._get_init_attrs_data(Slot, attr_type=attr_type) @classmethod def signals_attributes(cls) -> typing.Dict[str, "Attribute"]: @@ -229,9 +233,9 @@ def numeric_fields(cls): @classmethod def table_fields(cls): - """the table attributes corresponding to """ - #TODO: add list/numpy fields with vector stats - keeps = (str, float, int) + """the table attributes corresponding to""" + # TODO: add list/numpy fields with vector stats + keeps = (str, float, int) typ = cls._get_init_attrs_data(keeps) return {k: v for k, v in typ.items()} @@ -250,7 +254,7 @@ def as_dict(self): o = {k: getattr(self, k, None) for k, v in inputs.items()} return o - #TODO: refactor this, allowing a nesting return option for sub components, by default True (later to be reverted to False, as a breaking change). this messes up hashing and we can just the other object hash + # TODO: refactor this, allowing a nesting return option for sub components, by default True (later to be reverted to False, as a breaking change). this messes up hashing and we can just the other object hash @property def input_as_dict(self): """returns values as they are in the class instance, but converts classes inputs to their input_as_dict""" @@ -267,8 +271,9 @@ def input_as_dict(self): def table_row_dict(self): """returns values as they would be put in a table row from this instance ignoring any sub components""" from engforge.configuration import Configuration + o = {k: getattr(self, k, None) for k in self.table_fields()} - return o + return o @property def numeric_as_dict(self): @@ -284,12 +289,14 @@ def numeric_as_dict(self): # Hashes # TODO: issue with logging sub-items - def hash(self,*args, **input_kw): + def hash(self, *args, **input_kw): """hash by parm or by input_kw, only input can be hashed by lookup as system properties can create a recursive loop and should be deterministic from input""" - d = {k:v for k,v in self.input_as_dict.items() if k in args} - d.update(input_kw) #override with input_kw - return d,deepdiff.DeepHash(d, ignore_encoding_errors=True,significant_digits = 6) - + d = {k: v for k, v in self.input_as_dict.items() if k in args} + d.update(input_kw) # override with input_kw + return d, deepdiff.DeepHash( + d, ignore_encoding_errors=True, significant_digits=6 + ) + def hash_with(self, **input_kw): """ Generates a hash for the object's dictionary representation, updated with additional keyword arguments. @@ -321,9 +328,9 @@ def input_hash(self): @property def table_hash(self): - d,hsh = self.hash(**self.table_row_dict) + d, hsh = self.hash(**self.table_row_dict) return hsh[d] - + @property def numeric_hash(self): d = self.numeric_as_dict diff --git a/engforge/env_var.py b/engforge/env_var.py index 77212df..d44026a 100644 --- a/engforge/env_var.py +++ b/engforge/env_var.py @@ -139,7 +139,9 @@ def secret(self): # Provide warning that the secret is being replaced if not self._upgrd_warn: self._upgrd_warn = True - self.info(f"upgrading: {self.var_name} from {id(self)}->{id(sec)}") + self.info( + f"upgrading: {self.var_name} from {id(self)}->{id(sec)}" + ) # Monkeypatch dictionary self.__dict__ = sec.__dict__ @@ -162,7 +164,9 @@ def secret(self): secval = self.default else: if self.fail_on_missing: - raise FileNotFoundError(f"Could Not Find Env Variable {self.var_name}") + raise FileNotFoundError( + f"Could Not Find Env Variable {self.var_name}" + ) else: if self.var_name not in warned: self.debug(f"Env Var: {self.var_name} Not Found!") @@ -208,7 +212,9 @@ def print_env_vars(cls): global HOSTNAME, SLACK_WEBHOOK -HOSTNAME = EnvVariable("FORGE_HOSTNAME", default=host, obscure=False, dontovrride=True) +HOSTNAME = EnvVariable( + "FORGE_HOSTNAME", default=host, obscure=False, dontovrride=True +) SLACK_WEBHOOK = EnvVariable( "FORGE_SLACK_LOG_WEBHOOK", default=None, obscure=False, dontovrride=True ) diff --git a/engforge/logging.py b/engforge/logging.py index ffbddc8..16e92cf 100644 --- a/engforge/logging.py +++ b/engforge/logging.py @@ -18,7 +18,9 @@ LOG_LEVEL = logging.INFO -def change_all_log_levels(new_log_level: int = 20, inst=None, check_function=None): +def change_all_log_levels( + new_log_level: int = 20, inst=None, check_function=None +): """Changes All Log Levels With pyee broadcast before reactor is running :param new_log_level: int - changes unit level log level (10-msg,20-debug,30-info,40-warning,50-error,60-crit) :param check_function: callable -> bool - (optional) if provided if check_function(unit) is true then the new_log_level is applied @@ -27,7 +29,9 @@ def change_all_log_levels(new_log_level: int = 20, inst=None, check_function=No new_log_level = int(new_log_level) # Float Case Is Handled assert ( - isinstance(new_log_level, int) and new_log_level >= 1 and new_log_level <= 100 + isinstance(new_log_level, int) + and new_log_level >= 1 + and new_log_level <= 100 ) global LOG_LEVEL @@ -55,13 +59,17 @@ class LoggingMixin(logging.Filter): slack_webhook_url = None # log_silo = False - change_all_log_lvl = lambda s, *a, **kw: change_all_log_levels(*a, inst=s,**kw) + change_all_log_lvl = lambda s, *a, **kw: change_all_log_levels( + *a, inst=s, **kw + ) @property def logger(self): global LOG_LEVEL if self._log is None: - inst_log_name = "engforgelog_" + self.identity + "_" + str(uuid.uuid4()) + inst_log_name = ( + "engforgelog_" + self.identity + "_" + str(uuid.uuid4()) + ) self._log = logging.getLogger(inst_log_name) self._log.setLevel(level=LOG_LEVEL) @@ -155,7 +163,9 @@ def info(self, *args): def warning(self, *args): """Writes to log as a warning""" self.logger.warning( - self.message_with_identiy("WARN: " + self.extract_message(args), "yellow") + self.message_with_identiy( + "WARN: " + self.extract_message(args), "yellow" + ) ) def error(self, error, msg=""): diff --git a/engforge/patterns.py b/engforge/patterns.py index f8b76ce..eb85d08 100644 --- a/engforge/patterns.py +++ b/engforge/patterns.py @@ -71,7 +71,9 @@ class SingletonMeta(type): def __call__(cls, *args, **kwargs): if cls not in cls._instances: - cls._instances[cls] = super(SingletonMeta, cls).__call__(*args, **kwargs) + cls._instances[cls] = super(SingletonMeta, cls).__call__( + *args, **kwargs + ) return cls._instances[cls] @classmethod diff --git a/engforge/problem_context.py b/engforge/problem_context.py index 2869a17..658d626 100644 --- a/engforge/problem_context.py +++ b/engforge/problem_context.py @@ -152,7 +152,9 @@ class ProbLog(LoggingMixin): save_modes = ["vars", "nums", "all", "prob"] transfer_kw = ["system", "_dxdt"] -root_possible = list(root_defined.keys()) + list("_" + k for k in root_defined.keys()) +root_possible = list(root_defined.keys()) + list( + "_" + k for k in root_defined.keys() +) # TODO: output options extend_dataframe=True,return_dataframe=True,condensed_dataframe=True,return_system=True,return_problem=True,return_df=True,return_data=True # TODO: connect save_data() output to _data table. @@ -209,7 +211,8 @@ class ProblemExec: - _problem_id: uuid for subproblems, or True for top level, None means uninitialized """ - full_update = True #TODO: cant justify setting this to false for performance gains. accuracy comes first. see event based update + + full_update = True # TODO: cant justify setting this to false for performance gains. accuracy comes first. see event based update # TODO: convert this to a system based cache where there is a unique problem for each system instance. On subprobem copy a system and add o dictionary. class_cache = None # ProblemExec is assigned below @@ -245,7 +248,9 @@ class ProblemExec: _converged: bool # Interior Context Options - enter_refresh: bool = True #TODO: allow this off (or lower impact) with event system + enter_refresh: bool = ( + True # TODO: allow this off (or lower impact) with event system + ) save_on_exit: bool = False save_mode: str = "all" level_name: str = None # target this context with the level name @@ -292,7 +297,9 @@ def __getattr__(self, name): # Default behaviour return self.__getattribute__(name) - def __init__(self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts): + def __init__( + self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts + ): """ Initializes the ProblemExec. @@ -337,7 +344,7 @@ def __init__(self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts): self.persist_contexts() # temp solver storage #TODO - #self.solver_hist = expiringdict.ExpiringDict(100, 60) + # self.solver_hist = expiringdict.ExpiringDict(100, 60) if self.log_level < 5: if hasattr(self.class_cache, "session"): @@ -428,8 +435,8 @@ def __init__(self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts): self.class_cache.session._prob_levels[self.level_name] = self # error if the system is different (it shouldn't be!) if self.system is not system: - #TODO: subproblems allow different systems, but the top level should be the same - #idea - use (system,pid) as key for problems_dict, (system,True) would be root problem. This breaks checking for `class_cache.session` though one could gather that from the root problem key` + # TODO: subproblems allow different systems, but the top level should be the same + # idea - use (system,pid) as key for problems_dict, (system,True) would be root problem. This breaks checking for `class_cache.session` though one could gather that from the root problem key` raise IllegalArgument( f"somethings wrong! change of comp! {self.system} -> {system}" ) @@ -476,7 +483,9 @@ def __init__(self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts): self.set_checkpoint() if log.log_level < 10: - self.info(f"new execution context for {system}| {opts} | {self._slv_kw}") + self.info( + f"new execution context for {system}| {opts} | {self._slv_kw}" + ) elif log.log_level <= 3: self.msg(f"new execution context for {system}| {self._slv_kw}") @@ -594,7 +603,7 @@ def get_sesh(self, sesh=None): # Update Methods def refresh_references(self, sesh=None): """refresh the system references""" - + if sesh is None: sesh = self.sesh @@ -621,36 +630,46 @@ def update_dynamics(self, sesh=None): self.info(f"update dynamics") self.system.setup_global_dynamics() - def full_refresh(self,sesh=None): + def full_refresh(self, sesh=None): """a more time consuming but throughout refresh of the system""" if self.log_level < 5: self.info(f"full refresh") check_dynamics = sesh.check_dynamics - sesh._num_refs = sesh.system.system_references(numeric_only=True,none_ok=True, only_inst=False,ignore_none_comp=False,recache=True) + sesh._num_refs = sesh.system.system_references( + numeric_only=True, + none_ok=True, + only_inst=False, + ignore_none_comp=False, + recache=True, + ) sesh._sys_refs = sesh.system.solver_vars( check_dynamics=check_dynamics, addable=sesh._num_refs, **sesh._slv_kw, ) - sesh.update_methods(sesh=sesh) - + sesh.update_methods(sesh=sesh) + def min_refresh(self, sesh=None): """what things need to be refreshed per execution, this is important whenever items are replaced""" - #TODO: replace this function with an event based responsiblity model. + # TODO: replace this function with an event based responsiblity model. sesh = sesh if sesh is not None else self.sesh if self.log_level < 5: self.info(f"min refresh") if sesh.full_update: - #TODO: dont require this + # TODO: dont require this sesh.full_refresh(sesh=sesh) - + # final ref's after update # after updates sesh._all_refs = sesh.system.system_references( - recache=True, check_config=False, ignore_none_comp=False,none_ok=True, only_inst=False + recache=True, + check_config=False, + ignore_none_comp=False, + none_ok=True, + only_inst=False, ) # sesh._attr_sys_key_map = sesh.attribute_sys_key_map @@ -658,10 +677,10 @@ def min_refresh(self, sesh=None): sesh.Xref = sesh.all_problem_vars sesh.Yref = sesh.sys_solver_objectives() - cons = {} #TODO: parse additional constraints + cons = {} # TODO: parse additional constraints sesh.constraints = sesh.sys_solver_constraints(cons) - def print_all_info(self,keys:str=None,comps:str=None): + def print_all_info(self, keys: str = None, comps: str = None): """ Print all the information of each component's dictionary. Parameters: @@ -670,28 +689,31 @@ def print_all_info(self,keys:str=None,comps:str=None): Returns: None (except stdout :) """ from pprint import pprint - keys = keys.split(',') - comps = (comps+',').split(',') #always top level - print(f'CONTEXT: {self}') - - mtch = lambda key,ptrns: any([fnmatch.fnmatch(key.lower(),ptn.lower()) for ptn in ptrns]) - #check your comps + keys = keys.split(",") + comps = (comps + ",").split(",") # always top level + print(f"CONTEXT: {self}") + + mtch = lambda key, ptrns: any( + [fnmatch.fnmatch(key.lower(), ptn.lower()) for ptn in ptrns] + ) + + # check your comps itrs = self.all_comps.copy() - itrs[''] = Ref(self.system,'',True,False) + itrs[""] = Ref(self.system, "", True, False) - #check your comps - for cn,comp in itrs.items(): - if comps is not None and not mtch(cn,comps): + # check your comps + for cn, comp in itrs.items(): + if comps is not None and not mtch(cn, comps): continue - + dct = comp.value().as_dict - if keys: #filter keys - dct = {k:v for k,v in dct.items() if mtch(k,keys)} + if keys: # filter keys + dct = {k: v for k, v in dct.items() if mtch(k, keys)} if dct: print(f'INFO: {cn if cn else ""}') pprint(dct) - print('-'*80) + print("-" * 80) @property def check_dynamics(self): @@ -746,7 +768,11 @@ def __enter__(self): self.class_cache.level_number = 0 if self.log_level < 10: - refs = {k: v for k, v in self.sesh._sys_refs.get("attrs", {}).items() if v} + refs = { + k: v + for k, v in self.sesh._sys_refs.get("attrs", {}).items() + if v + } self.debug( f"creating execution context for {self.system}| {self._slv_kw}| {refs}" ) @@ -819,7 +845,9 @@ def __exit__(self, exc_type, exc_value, traceback): if type(self.problems_dict) is not dict: self.problems_dict.pop(self._problem_id, None) del self.class_cache.session - raise KeyError(f"cant exit to level! {exc_value.level} not found!!") + raise KeyError( + f"cant exit to level! {exc_value.level} not found!!" + ) else: if self.log_level <= 18: @@ -853,7 +881,7 @@ def debug_levels(self): raise IllegalArgument(f"no session available") # Multi Context Exiting: - #TODO: rethink this + # TODO: rethink this def persist_contexts(self): """convert all contexts to a new storage format""" self.info(f"persisting contexts!") @@ -961,7 +989,10 @@ def save_data(self, index=None, force=False, **add_data): self.warning(f"no data saved, nothing changed") def clean_context(self): - if hasattr(self.class_cache, "session") and self.class_cache.session is self: + if ( + hasattr(self.class_cache, "session") + and self.class_cache.session is self + ): if self.log_level <= 8: self.debug(f"closing execution session") self.class_cache.level_number = 0 @@ -1005,7 +1036,9 @@ def integrate(self, endtime, dt=0.001, max_step_dt=0.01, X0=None, **kw): dt = max_step_dt if self.log_level < 15: - self.info(f"simulating {system},{sesh}| int:{intl_refs} | refs: {refs}") + self.info( + f"simulating {system},{sesh}| int:{intl_refs} | refs: {refs}" + ) if not intl_refs: raise Exception(f"no transient parameters found") @@ -1118,7 +1151,7 @@ def integral_rate(self, t, x, dt, Xss=None, Yobj=None, **kw): self.info( f'exiting solver {t} {ss_out["Xans"]} {ss_out["Xstart"]}' ) - pbx.set_ref_values(ss_out["Xans"],scope='intgrl') + pbx.set_ref_values(ss_out["Xans"], scope="intgrl") pbx.exit_to_level("ss_slvr", False) else: self.warning( @@ -1223,7 +1256,7 @@ def handle_solution(self, answer, Xref, Yref, output): # Output Results Xa = {p: answer.x[i] for i, p in enumerate(vars)} output["Xans"] = Xa - Ref.refset_input(Xref, Xa,scope='solvd') + Ref.refset_input(Xref, Xa, scope="solvd") Yout = {p: yit.value(yit.comp, self) for p, yit in Yref.items()} output["Yobj"] = Yout @@ -1231,7 +1264,9 @@ def handle_solution(self, answer, Xref, Yref, output): Ycon = {} if sesh.constraints["constraints"]: x_in = answer.x - for c, k in zip(sesh.constraints["constraints"], sesh.constraints["info"]): + for c, k in zip( + sesh.constraints["constraints"], sesh.constraints["info"] + ): cv = c["fun"](x_in, self, {}) Ycon[k] = cv output["Ycon"] = Ycon @@ -1344,7 +1379,9 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): ) slv_inst = sys_refs.get("type", {}).get("solver", {}) - trv_inst = {v.var: v for v in sys_refs.get("type", {}).get("time", {}).values()} + trv_inst = { + v.var: v for v in sys_refs.get("type", {}).get("time", {}).values() + } sys_refs = sys_refs.get("attrs", {}) if add_con is None: @@ -1419,12 +1456,22 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): combo_var = ctype["combo_var"] active = ctype.get("active", True) in_activate = ( - any([arg_var_compare(combo_var, v) for v in activated]) + any( + [ + arg_var_compare(combo_var, v) + for v in activated + ] + ) if activated else False ) in_deactivate = ( - any([arg_var_compare(combo_var, v) for v in deactivated]) + any( + [ + arg_var_compare(combo_var, v) + for v in deactivated + ] + ) if deactivated else False ) @@ -1435,12 +1482,16 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): # Check active or activated if not active and not activated: if log.log_level < 3: - self.msg(f"skip con: inactive {var} {slvr} {ctype}") + self.msg( + f"skip con: inactive {var} {slvr} {ctype}" + ) continue elif not active and not in_activate: if log.log_level < 3: - self.msg(f"skip con: inactive {var} {slvr} {ctype}") + self.msg( + f"skip con: inactive {var} {slvr} {ctype}" + ) continue elif active and in_deactivate: @@ -1458,7 +1509,9 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): continue if log.log_level < 10: - self.debug(f"adding var constraint {var,slvr,ctype,combos}") + self.debug( + f"adding var constraint {var,slvr,ctype,combos}" + ) # get the index of the variable x_inx = Xvars.index(slvr) @@ -1522,7 +1575,9 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): cval, **kw, ) - con_info.append(f"val_{ref.comp.classname}_{kind}_{slvr}") + con_info.append( + f"val_{ref.comp.classname}_{kind}_{slvr}" + ) con_list.append(ccst) else: @@ -1534,7 +1589,9 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): for slvr, ref in self.problem_ineq.items(): slv = slv_inst[slvr] slv_constraints = slv.constraints - parent = self.get_parent_key(slvr, look_back_num=2) # get the parent comp + parent = self.get_parent_key( + slvr, look_back_num=2 + ) # get the parent comp for ctype in slv_constraints: cval = ctype["value"] kind = ctype["type"] @@ -1556,7 +1613,9 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): ) for slvr, ref in self.problem_eq.items(): - parent = self.get_parent_key(slvr, look_back_num=2) # get the parent comp + parent = self.get_parent_key( + slvr, look_back_num=2 + ) # get the parent comp if slvr in slv_inst and slvr in all_refz.get("solver.eq", {}): slv = slv_inst[slvr] slv_constraints = slv.constraints @@ -1603,7 +1662,9 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): # General method to distribute input to internal components @classmethod - def parse_default(self, key, defaults, input_dict, rmv=False, empty_str=True): + def parse_default( + self, key, defaults, input_dict, rmv=False, empty_str=True + ): """splits strings or lists and returns a list of options for the key, if nothing found returns None if fail set to True raises an exception, otherwise returns the default value""" if key in input_dict: # kwargs will no longer have key! @@ -1696,7 +1757,9 @@ def output_state(self) -> dict: elif "prob" == sesh.save_mode: raise NotImplementedError(f"problem save mode not implemented") else: - raise KeyError(f"unknown save mode {sesh.save_mode}, not in {save_modes}") + raise KeyError( + f"unknown save mode {sesh.save_mode}, not in {save_modes}" + ) out = Ref.refset_get(refs, sys=sesh.system, prob=self) # Integration @@ -1712,15 +1775,17 @@ def get_ref_values(self, refs=None): refs = sesh.all_system_references return Ref.refset_get(refs, sys=self.system, prob=self) - def set_ref_values(self, values, refs=None,scope='sref'): + def set_ref_values(self, values, refs=None, scope="sref"): """returns the values of the refs""" # TODO: add checks for the refs if refs is None: sesh = self.sesh refs = sesh.all_comps_and_vars - return Ref.refset_input(refs, values,scope=scope) + return Ref.refset_input(refs, values, scope=scope) - def change_sys_var(self, key, value, refs=None, doset=True, attr_key_map=None): + def change_sys_var( + self, key, value, refs=None, doset=True, attr_key_map=None + ): """use this function to change the value of a system var and update the start state, multiple uses in the same context will not change the record preserving the start value :param key: a string corresponding to a ref, or an `attrs.Attribute` of one of the system or its component's. @@ -1763,7 +1828,9 @@ def revert_to_start(self): rs = list(self.record_state.values()) self.debug(f"reverting to start: {xs} -> {rs}") # TODO: STRICT MODE Fail for refset_input - Ref.refset_input(sesh.all_comps_and_vars, self.x_start, fail=False,scope='rvtst') + Ref.refset_input( + sesh.all_comps_and_vars, self.x_start, fail=False, scope="rvtst" + ) def activate_temp_state(self, new_state=None): # TODO: determine when components change, and update refs accordingly! @@ -1772,11 +1839,18 @@ def activate_temp_state(self, new_state=None): if new_state: if self.log_level < 3: self.debug(f"new-state: {self.temp_state}") - Ref.refset_input(sesh.all_comps_and_vars, new_state, fail=False,scope='ntemp') + Ref.refset_input( + sesh.all_comps_and_vars, new_state, fail=False, scope="ntemp" + ) elif self.temp_state: if self.log_level < 3: self.debug(f"act-state: {self.temp_state}") - Ref.refset_input(sesh.all_comps_and_vars, self.temp_state, fail=False,scope='atemp') + Ref.refset_input( + sesh.all_comps_and_vars, + self.temp_state, + fail=False, + scope="atemp", + ) elif self.log_level < 3: self.debug(f"no-state: {new_state}") @@ -2074,9 +2148,12 @@ def numeric_data(self): filter_non_numeric = lambda kv: ( False if isinstance(kv[1], (list, dict, tuple)) else True ) - f_numrow = lambda in_dict: dict(filter(filter_non_numeric, in_dict.items())) + f_numrow = lambda in_dict: dict( + filter(filter_non_numeric, in_dict.items()) + ) return [ - f_numrow(kv[-1]) for kv in sorted(sesh.data.items(), key=lambda kv: kv[0]) + f_numrow(kv[-1]) + for kv in sorted(sesh.data.items(), key=lambda kv: kv[0]) ] @property @@ -2225,4 +2302,6 @@ def level_name(self): @level_name.setter def level_name(self, value): - raise AttributeError(f"cannot set level_name of top level problem context") + raise AttributeError( + f"cannot set level_name of top level problem context" + ) diff --git a/engforge/properties.py b/engforge/properties.py index b516182..9d1a931 100644 --- a/engforge/properties.py +++ b/engforge/properties.py @@ -21,7 +21,8 @@ class PropertyLog(LoggingMixin): log = PropertyLog() -_basic_valild_types = (int, str, float) #check for return_type +_basic_valild_types = (int, str, float) # check for return_type + class engforge_prop: """ @@ -32,12 +33,13 @@ class engforge_prop: def our_custom_function(self) -> return_type: pass """ + valid_types = _basic_valild_types must_return = False def __init__(self, fget=None, fset=None, fdel=None, *args, **kwargs): """call with the function you want to wrap in a decorator""" - self.valild_types = (int, str, float) #check for return_type + self.valild_types = (int, str, float) # check for return_type self.fget = fget if fget: self.gname = fget.__name__ @@ -45,7 +47,9 @@ def __init__(self, fget=None, fset=None, fdel=None, *args, **kwargs): self.fset = fset self.fdel = fdel - def __call__(self, fget=None, fset=None, fdel=None, doc=None, *args, **kwargs): + def __call__( + self, fget=None, fset=None, fdel=None, doc=None, *args, **kwargs + ): """this will be called when input is provided before property is set""" if fget and self.fget is None: self.gname = fget.__name__ @@ -99,11 +103,13 @@ def deleter(self, fdel): class cache_prop(engforge_prop): - allow_set: bool = False # keep this flag false to maintain current persistent value + allow_set: bool = ( + False # keep this flag false to maintain current persistent value + ) def __init__(self, *args, **kwargs): self.allow_set = True - self.valild_types = (int, str, float) #check for return_type + self.valild_types = (int, str, float) # check for return_type super().__init__(*args, **kwargs) def __set__(self, instance, value): @@ -157,7 +163,7 @@ def function(...): < this uses __init__ to assign function @system_property(desc='really nice',label='funky function') def function(...): < this uses __call__ to assign function """ - self.valild_types = (int, str, float) #check for return_type + self.valild_types = (int, str, float) # check for return_type self.fget = fget if fget: self.get_func_return(fget) @@ -342,7 +348,9 @@ def __get__(self, instance: "TabulationMixin", objtype=None): if not hasattr(instance, self.private_var): from engforge.tabulation import TabulationMixin - if instance.__class__ is not None and not IS_BUILD: # its an instance + if ( + instance.__class__ is not None and not IS_BUILD + ): # its an instance assert issubclass( instance.__class__, TabulationMixin ), f"incorrect class: {instance.__class__.__name__}" diff --git a/engforge/solveable.py b/engforge/solveable.py index 87b187a..26ad657 100644 --- a/engforge/solveable.py +++ b/engforge/solveable.py @@ -37,7 +37,7 @@ def _update_func(comp, eval_kw): def updt(*args, **kw): eval_kw.update(kw) if log.log_level <= 12: - log.info(f"update| {comp.name} ")#=5) + log.info(f"update| {comp.name} ") # =5) return comp.update(comp.parent, *args, **eval_kw) if log.log_level <= 12: @@ -75,9 +75,9 @@ def updt(*args, **kw): def updt(*args, **kw): if log.log_level <= 12: - log.info(f"update costs {comp.name} ")#=5) + log.info(f"update costs {comp.name} ") # =5) comp.system_properties_classdef(True) - #comp.update(comp.parent, *args, **kw) #called as update without cm + # comp.update(comp.parent, *args, **kw) #called as update without cm return comp.update_dflt_costs() if log.log_level <= 12: @@ -145,18 +145,18 @@ def collect_update_refs(self, eval_kw=None, ignore=None): ignore = set() elif self in ignore: return - - key= 'top' + + key = "top" if self.__class__.update != SolveableMixin.update: ref = Ref(self, _update_func(self, eval_kw if eval_kw else {})) - updt_refs[key] = ref + updt_refs[key] = ref if isinstance(self, (CostModel, Economics)): ref = Ref(self, _cost_update(self)) - updt_refs[key + "._cost_model_"] = ref + updt_refs[key + "._cost_model_"] = ref for key, comp in self.internal_configurations(False).items(): - #for key,lvl,comp in self.go_through_configurations(check_config=False): + # for key,lvl,comp in self.go_through_configurations(check_config=False): if ignore is not None and comp in ignore: continue @@ -195,13 +195,13 @@ def collect_post_update_refs(self, eval_kw=None, ignore=None): ignore = set() elif self in ignore: return - + if self.__class__.post_update != SolveableMixin.post_update: ref = Ref(self, _post_update_func(self, eval_kw if eval_kw else {})) - updt_refs['top'] = ref + updt_refs["top"] = ref for key, comp in self.internal_configurations(False).items(): - #for key,lvl,comp in self.go_through_configurations(check_config=False): + # for key,lvl,comp in self.go_through_configurations(check_config=False): if ignore is not None and comp in ignore: continue @@ -332,7 +332,8 @@ def _gen(gen, compkey): yield compkey, itemkey iter_vals = { - cn: _gen(comp._item_gen(), cn) for cn, comp in components.items() + cn: _gen(comp._item_gen(), cn) + for cn, comp in components.items() } for out in itertools.product(*list(iter_vals.values())): @@ -350,7 +351,9 @@ def comp_references(self, ignore_none_comp=True, **kw): #FIXME: by instance recache on iterative component change or other signals """ out = {} - for key,lvl,comp in self.go_through_configurations(parent_level=1,**kw): + for key, lvl, comp in self.go_through_configurations( + parent_level=1, **kw + ): if ignore_none_comp and not isinstance(comp, SolveableMixin): self.warning(f"ignoring {key} {lvl}|{comp}") continue @@ -368,7 +371,7 @@ def _iterate_input_matrix( force_solve=False, return_results=False, method_kw: dict = None, - print_kw:dict=None, + print_kw: dict = None, **kwargs, ): """applies a permutation of input vars for vars. runs the system instance by applying input to the system and its slot-components, ensuring that the targeted attributes actualy exist. @@ -379,7 +382,7 @@ def _iterate_input_matrix( :param eval_kw: a dictionary of keyword arguments to pass to the eval function of each component by their name and a set of keyword args. Use this to set values in the component that are not inputs to the system. No iteration occurs upon these values, they are static and irrevertable :param sys_kw: a dictionary of keyword arguments to pass to the eval function of each system by their name and a set of keyword args. Use this to set values in the component that are not inputs to the system. No iteration occurs upon these values, they are static and irrevertable :param print_kw: a dictionary of keyword arguments to pass to the print_all_info function of the current context - + :param kwargs: inputs are run on a product basis asusming they correspond to actual scoped vars (system.var or system.slot.var) @@ -388,8 +391,12 @@ def _iterate_input_matrix( from engforge.system import System from engforge.problem_context import ProblemExec - self.debug(f"running [Solver].{method} {self.identity} with input {kwargs}") - assert hasattr(ProblemExec.class_cache, "session"), "must be active context!" + self.debug( + f"running [Solver].{method} {self.identity} with input {kwargs}" + ) + assert hasattr( + ProblemExec.class_cache, "session" + ), "must be active context!" # create iterable null for sequence if sequence is None or not sequence: sequence = [{}] @@ -435,7 +442,11 @@ def _iterate_input_matrix( # Run The Method with inputs provisioned out = method(icur, eval_kw, sys_kw, cb=cb, **method_kw) - if print_kw and hasattr(self, "last_context") and self.last_context: + if ( + print_kw + and hasattr(self, "last_context") + and self.last_context + ): self.last_context.print_all_info(**print_kw) if return_results: @@ -501,7 +512,9 @@ def parse_run_kwargs(self, **kwargs): comp_args = {k: v for k, v in kwargs.items() if "." in k} # check vars - inpossible = set.union(set(self.input_fields()), set(self.slots_attributes())) + inpossible = set.union( + set(self.input_fields()), set(self.slots_attributes()) + ) argdiff = set(var_args).difference(inpossible) assert not argdiff, f"bad input {argdiff}" @@ -511,7 +524,9 @@ def parse_run_kwargs(self, **kwargs): assert not compdiff, f"bad slot references {compdiff}" _input = {} - test = lambda v, add: isinstance(v, (int, float, str, *add)) or v is None + test = ( + lambda v, add: isinstance(v, (int, float, str, *add)) or v is None + ) # vars input for k, v in kwargs.items(): @@ -531,7 +546,9 @@ def parse_run_kwargs(self, **kwargs): assert test(v, addty), f"bad values {k}:{v}" v = [v] else: - assert all([test(vi, addty) for vi in v]), f"bad values: {k}:{v}" + assert all( + [test(vi, addty) for vi in v] + ), f"bad values: {k}:{v}" if k not in _input: _input[k] = v @@ -585,7 +602,9 @@ def locate_ref(self, key, fail=True, **kw): func = copy.copy(key) return Ref(comp, func, **kw) else: - assert "comp" not in kw, f"comp kwarg not allowed with string key {key}" + assert ( + "comp" not in kw + ), f"comp kwarg not allowed with string key {key}" if "." in key: args = key.split(".") @@ -608,7 +627,10 @@ def locate_ref(self, key, fail=True, **kw): # val= cls.system_properties_classdef()[key] return Ref(self, key, **kw) - elif key in self.internal_configurations() or key in self.slots_attributes(): + elif ( + key in self.internal_configurations() + or key in self.slots_attributes() + ): return Ref(self, key, **kw) # Fail on comand but otherwise return val @@ -629,7 +651,7 @@ def system_references(self, recache=False, numeric_only=False, **kw): ): return self._prv_system_references - #TODO: system references are really important and nesting them together complicates the refresh process. Each component should be able to refresh itself and its children on set_state, as well as alert parents to change. Ideally the `Ref` objects could stay the same and no `recache` would need to occur. This would be a huge performance boost and fix a lot of the issues with the current system. + # TODO: system references are really important and nesting them together complicates the refresh process. Each component should be able to refresh itself and its children on set_state, as well as alert parents to change. Ideally the `Ref` objects could stay the same and no `recache` would need to occur. This would be a huge performance boost and fix a lot of the issues with the current system. out = self.internal_references(recache, numeric_only=numeric_only) tatr = out["attributes"] @@ -656,7 +678,9 @@ def system_references(self, recache=False, numeric_only=False, **kw): comp_dict[key] = comp if parent in comp_dict: attr_name = key.split(".")[-1] - comp_set_ref[key] = Ref(comp_dict[parent], attr_name, False, True) + comp_set_ref[key] = Ref( + comp_dict[parent], attr_name, False, True + ) # Fill in for k, v in satr.items(): @@ -753,7 +777,9 @@ def collect_solver_refs( _var = getattr(conf, pre_var) if isinstance(_var, AttributeInstance): slv_type = _var - conf.msg(f"slv type: {conf.classname}.{pre_var} -> {_var}") + conf.msg( + f"slv type: {conf.classname}.{pre_var} -> {_var}" + ) val_type = ck_type[pre_var] @@ -767,7 +793,9 @@ def collect_solver_refs( cls_dict[atype][scope_name] = val_type if conf.log_level <= 5: - conf.msg(f"rec: {var_name} {k} {pre} {val} {slv_type}") + conf.msg( + f"rec: {var_name} {k} {pre} {val} {slv_type}" + ) # Check to skip this item # keep references even if null @@ -788,7 +816,9 @@ def collect_solver_refs( pre, scope_name, val_type, check_kw ): if conf.log_level <= 5: - conf.msg(f"chk skip {scope_name} {k} {pre} {val}") + conf.msg( + f"chk skip {scope_name} {k} {pre} {val}" + ) if pre not in skipped: skipped[pre] = [] @@ -835,7 +865,11 @@ def collect_solver_refs( # eval each ref for inclusion for var, ref in refs.items(): key_segs = var.split(".") - key = "" if len(key_segs) == 1 else ".".join(key_segs[:-1]) + key = ( + "" + if len(key_segs) == 1 + else ".".join(key_segs[:-1]) + ) scoped_name = f"{var}" conf = dyn_comp.get(key) @@ -934,7 +968,9 @@ def get_system_input_refs( if p in SKIP_REF and not all: continue if all: - refs[(f"{ckey}." if ckey else "") + p] = Ref(comp, p, False, True) + refs[(f"{ckey}." if ckey else "") + p] = Ref( + comp, p, False, True + ) continue elif atr.type: ty = atr.type diff --git a/engforge/solver.py b/engforge/solver.py index 33057fa..96db6d8 100644 --- a/engforge/solver.py +++ b/engforge/solver.py @@ -112,7 +112,9 @@ def solver_vars(self, check_dynamics=True, addable=None, **kwargs): for av in addvar: # list/dict-keys matches = set(fnmatch.filter(avars, av)) # Lookup Constraints if input is a dictionary - if isinstance(addvar, dict) and isinstance(addvar[av], dict): + if isinstance(addvar, dict) and isinstance( + addvar[av], dict + ): const = base_const.copy() const.update(addvar[av]) elif isinstance(addvar, dict): @@ -161,7 +163,9 @@ def run(self, **kwargs): with ProblemExec(self, kwargs, level_name="run") as pbx: # problem context removes slv/args from kwargs - return self._iterate_input_matrix(self.eval, return_results=True, **kwargs) + return self._iterate_input_matrix( + self.eval, return_results=True, **kwargs + ) def run_internal_systems(self, sys_kw=None): """runs internal systems with potentially scoped kwargs""" @@ -186,7 +190,9 @@ def run_internal_systems(self, sys_kw=None): comp.eval(**sys_kw_comp) # Single Point Flow - def eval(self, Xo=None, eval_kw: dict = None, sys_kw: dict = None, cb=None, **kw): + def eval( + self, Xo=None, eval_kw: dict = None, sys_kw: dict = None, cb=None, **kw + ): """Evaluates the system with pre/post execute methodology :param kw: kwargs come from `sys_kw` input in run ect. :param cb: an optional callback taking the system as an argument of the form (self,eval_kw,sys_kw,**kw) @@ -284,7 +290,7 @@ def solver(self, enter_refresh=True, save_on_exit=True, **kw): # depending on the solver success, failure or no solution, we can exit the solver if has_ans and out["ans"] and out["ans"].success: # this is where you want to be! <<< - pbx.set_ref_values(out["Xans"],scope='solvr') + pbx.set_ref_values(out["Xans"], scope="solvr") pbx.exit_to_level("sys_slvr", False) elif has_ans and out["ans"] is None: diff --git a/engforge/solver_utils.py b/engforge/solver_utils.py index 9f4ede8..776421c 100644 --- a/engforge/solver_utils.py +++ b/engforge/solver_utils.py @@ -156,7 +156,9 @@ def ref_to_val_constraint( return ref # Make Objective - return create_constraint(system, comp, Xrefs, contype, ref, ctx, *args, **kwargs) + return create_constraint( + system, comp, Xrefs, contype, ref, ctx, *args, **kwargs + ) def create_constraint( @@ -173,7 +175,9 @@ def create_constraint( ), f"bad constraint type: {contype}" if comp.log_level < 5: - comp.debug(f"create constraint {contype} {ref} {args} {kwargs}| {con_args}") + comp.debug( + f"create constraint {contype} {ref} {args} {kwargs}| {con_args}" + ) # its a function _fun = lambda *args, **kw: ref.value(*args, **kw) @@ -316,7 +320,9 @@ def ext_str_list(extra_kw, key, default=None): ] -def combo_filter(attr_name, var_name, solver_inst, extra_kw, combos=None) -> bool: +def combo_filter( + attr_name, var_name, solver_inst, extra_kw, combos=None +) -> bool: # TODO: allow solver_inst to be None for dyn-classes # proceed to filter active items if vars / combos inputs is '*' select all, otherwise discard if not active # corresondes to problem_context.slv_dflt_options @@ -481,7 +487,9 @@ def filt_active(var, inst, extra_kw=None, dflt=False): from engforge.attr_signals import SignalInstance # not considered - if not isinstance(inst, (SolverInstance, IntegratorInstance, SignalInstance)): + if not isinstance( + inst, (SolverInstance, IntegratorInstance, SignalInstance) + ): return True activate = ext_str_list(extra_kw, "activate", []) diff --git a/engforge/system.py b/engforge/system.py index 67b9d1f..53d24e0 100644 --- a/engforge/system.py +++ b/engforge/system.py @@ -127,7 +127,9 @@ def _anything_changed(self): """looks at internal components as well as flag for anything chagned.""" if self._anything_changed_: return True - elif any([c.anything_changed for k, c in self.comp_references().items()]): + elif any( + [c.anything_changed for k, c in self.comp_references().items()] + ): return True return False diff --git a/engforge/system_reference.py b/engforge/system_reference.py index 70079cd..26035fd 100644 --- a/engforge/system_reference.py +++ b/engforge/system_reference.py @@ -15,6 +15,7 @@ import copy from difflib import get_close_matches + class RefLog(LoggingMixin): pass @@ -22,7 +23,7 @@ class RefLog(LoggingMixin): log = RefLog() -def refset_input(refs, delta_dict, chk=True, fail=True, warn=True,scope='ref'): +def refset_input(refs, delta_dict, chk=True, fail=True, warn=True, scope="ref"): """change a set of refs with a dictionary of values. If chk is True k will be checked for membership in refs""" keys = set(refs.keys()) for k, v in delta_dict.items(): @@ -31,15 +32,17 @@ def refset_input(refs, delta_dict, chk=True, fail=True, warn=True,scope='ref'): continue memb = k in refs - #if a match or not checking go ahead. + # if a match or not checking go ahead. if not chk or memb: refs[k].set_value(v) - #TODO: handle setting non-ref values such as dictionaries - + # TODO: handle setting non-ref values such as dictionaries + elif fail and chk and not memb: close = get_close_matches(k, keys) - raise KeyError(f"{scope}| key {k} not in refs. did you mean {close}?") + raise KeyError( + f"{scope}| key {k} not in refs. did you mean {close}?" + ) elif warn and chk and not memb: close = get_close_matches(k, keys) log.warning(f"{scope}| key {k} not in refs. did you mean {close}") @@ -47,7 +50,7 @@ def refset_input(refs, delta_dict, chk=True, fail=True, warn=True,scope='ref'): def refset_get(refs, *args, **kw): out = {} - scope = kw.get('scope','ref') + scope = kw.get("scope", "ref") for k in refs: try: # print(k,refs[k]) @@ -203,7 +206,7 @@ def set(self, comp, key, use_call=True, allow_set=True, eval_f=None): self.hxd = str(hex(id(self)))[-6:] - #TODO: update with change (pyee?) + # TODO: update with change (pyee?) self.setup_calls() def setup_calls(self): @@ -223,9 +226,9 @@ def setup_calls(self): self._value_eval = lambda *a, **kw: self.key(*a, **kw) else: # do not cross reference vars! - #TODO: allo for comp/key changes with events - if self.key == '': #return the component - p = lambda *a, **kw: self.comp + # TODO: allo for comp/key changes with events + if self.key == "": # return the component + p = lambda *a, **kw: self.comp elif self.use_dict: p = lambda *a, **kw: self.comp.get(self.key) elif self.key in self.comp.__dict__: diff --git a/engforge/tabulation.py b/engforge/tabulation.py index b22fe97..ddec3ca 100644 --- a/engforge/tabulation.py +++ b/engforge/tabulation.py @@ -95,19 +95,25 @@ def dataframe(self): :rtype: pandas.DataFrame """ """""" - if hasattr(self, "last_context") and hasattr(self.last_context, "dataframe"): + if hasattr(self, "last_context") and hasattr( + self.last_context, "dataframe" + ): return self.last_context.dataframe - if hasattr(self,'_patch_dataframe') and self._patch_dataframe is not None: + if ( + hasattr(self, "_patch_dataframe") + and self._patch_dataframe is not None + ): return self._patch_dataframe return pandas.DataFrame([]) @dataframe.setter - def dataframe(self,input_dataframe): - if hasattr(self, "last_context") and hasattr(self.last_context, "dataframe"): - raise Exception(f'may not set dataframe on run component') + def dataframe(self, input_dataframe): + if hasattr(self, "last_context") and hasattr( + self.last_context, "dataframe" + ): + raise Exception(f"may not set dataframe on run component") self._patch_dataframe = input_dataframe - @property def plotable_variables(self): """Checks columns for ones that only contain numeric types or haven't been explicitly skipped""" @@ -199,7 +205,9 @@ def system_properties_types(self) -> list: @instance_cached def system_properties_keys(self) -> list: """Returns the table property keys""" - tabulated_properties = [k for k, obj in self.system_properties_def.items()] + tabulated_properties = [ + k for k, obj in self.system_properties_def.items() + ] return tabulated_properties @instance_cached @@ -213,7 +221,9 @@ def system_properties_description(self) -> list: @classmethod def cls_all_property_labels(cls): - return [obj.label for k, obj in cls.system_properties_classdef().items()] + return [ + obj.label for k, obj in cls.system_properties_classdef().items() + ] @classmethod def cls_all_property_keys(cls): @@ -233,7 +243,7 @@ def system_properties_classdef(cls, recache=False): """Combine other parent-classes table properties into this one, in the case of subclassed system_properties""" from engforge.tabulation import TabulationMixin - cls_key =f"_{cls.__name__}_system_properties" + cls_key = f"_{cls.__name__}_system_properties" # Use a cache for deep recursion if not recache and hasattr(cls, cls_key): res = getattr(cls, cls_key) @@ -269,16 +279,23 @@ def system_properties_classdef(cls, recache=False): if prop and isinstance(prop, system_property): __system_properties[k] = prop if log.log_level <= 3: - log.msg(f"adding system property {mrv.__name__}.{k}") + log.msg( + f"adding system property {mrv.__name__}.{k}" + ) - setattr(cls,cls_key, __system_properties) + setattr(cls, cls_key, __system_properties) return __system_properties @classmethod def pre_compile(cls): cls._anything_changed = True # set default on class - if any([v.stochastic for k, v in cls.system_properties_classdef(True).items()]): + if any( + [ + v.stochastic + for k, v in cls.system_properties_classdef(True).items() + ] + ): log.info(f"setting always save on {cls.__name__}") cls._always_save_data = True diff --git a/engforge/test/_pre_test_structures.py b/engforge/test/_pre_test_structures.py index 4851881..bc3dff0 100644 --- a/engforge/test/_pre_test_structures.py +++ b/engforge/test/_pre_test_structures.py @@ -40,7 +40,9 @@ def test_beam(self): self.subtest_assert_near(self.bm.section_mass, 41.9) self.subtest_assert_near(self.bm.max_von_mises(), 27.4e6) - self.subtest_assert_near(float(self.bm.data_dict["min_deflection_y"]), -0.0076) + self.subtest_assert_near( + float(self.bm.data_dict["min_deflection_y"]), -0.0076 + ) # self.subtest_assert_near(float(self.bm.data_dict["max_shear_y"]), 3000) self.subtest_assert_near(float(self.bm.data_dict["max_shear_y"]), 3000) @@ -62,7 +64,9 @@ def test_beam(self): def subtest_assert_near(self, value, truth, pct=0.025, **kw): with self.subTest(**kw): - self.assertAlmostEqual(value, truth, delta=max(abs(truth * pct), abs(pct))) + self.assertAlmostEqual( + value, truth, delta=max(abs(truth * pct), abs(pct)) + ) class test_truss(unittest.TestCase): @@ -86,7 +90,9 @@ def setUp(self): for n1 in self.st.nodes.values(): for n2 in self.st.nodes.values(): L = numpy.sqrt( - (n1.X - n2.X) ** 2.0 + (n1.Y - n2.Y) ** 2.0 + (n1.Z - n2.Z) ** 2.0 + (n1.X - n2.X) ** 2.0 + + (n1.Y - n2.Y) ** 2.0 + + (n1.Z - n2.Z) ** 2.0 ) if ( @@ -197,7 +203,9 @@ def subtest_member(self, nodea, nodeb, result_key, truth, pct=0.025): dopasst = abs(value - truth) <= abs(truth) * pct if not dopasst: - print(f"fails {key} {result_key}| {value:3.5f} == {truth:3.5f}?") + print( + f"fails {key} {result_key}| {value:3.5f} == {truth:3.5f}?" + ) self.subtest_assert_near(value, truth, pct=pct) def subtest_assert_near(self, value, truth, pct=0.025): diff --git a/engforge/test/report_testing.py b/engforge/test/report_testing.py index aac0572..8a10737 100644 --- a/engforge/test/report_testing.py +++ b/engforge/test/report_testing.py @@ -52,7 +52,9 @@ class OtherComponent(Component): if suprise: ones = attr.ib(default=1111) - wacky = attr.ib(default="one hundred and 111", validator=STR_VALIDATOR()) + wacky = attr.ib( + default="one hundred and 111", validator=STR_VALIDATOR() + ) more = attr.ib(default="other stuff", validator=STR_VALIDATOR()) always_save_data = True @@ -137,7 +139,9 @@ def suprise(self): tc = analysis.internal_component # 1) Ensure exists Reflect The Database - db = DBConnection("reports", host="localhost", user="postgres", passd="dumbpass") + db = DBConnection( + "reports", host="localhost", user="postgres", passd="dumbpass" + ) db.ensure_database_exists() # db.engine.echo = True diff --git a/engforge/test/test_comp_iter.py b/engforge/test/test_comp_iter.py index 737bff8..7d7d628 100644 --- a/engforge/test/test_comp_iter.py +++ b/engforge/test/test_comp_iter.py @@ -110,7 +110,9 @@ def test_keys(self): self.assertTrue(mtch, msg=f"missing keys: {should_keys - sys_key}") # save the data to table - self.system.run(revert_last=False, revert_every=False, save_on_exit=True) + self.system.run( + revert_last=False, revert_every=False, save_on_exit=True + ) df = self.system.last_context.dataframe self.assertTrue(len(df) == 1, msg=f"len: {len(df)}|\n{str(df)}") diff --git a/engforge/test/test_costs.py b/engforge/test/test_costs.py index 2399c51..4a593a0 100644 --- a/engforge/test/test_costs.py +++ b/engforge/test/test_costs.py @@ -96,7 +96,8 @@ def test_econ_array(self): df = er.dataframe tc = ( - df["econ.summary.total_cost"] == np.array([161.0, 220.0, 305.0, 390.0]) + df["econ.summary.total_cost"] + == np.array([161.0, 220.0, 305.0, 390.0]) ).all() self.assertTrue(tc) diff --git a/engforge/test/test_dynamics_spaces.py b/engforge/test/test_dynamics_spaces.py index 3e9b6b2..212abe8 100644 --- a/engforge/test/test_dynamics_spaces.py +++ b/engforge/test/test_dynamics_spaces.py @@ -28,7 +28,9 @@ def test_steady_state(self): # TODO: check dxdt=0 combo results (dynamics/rates==states) - ans = ds.run(dxdt=0, combos="time", revert_last=False, revert_every=False) + ans = ds.run( + dxdt=0, combos="time", revert_last=False, revert_every=False + ) output = ans["output"][0] self.assertTrue(output["success"]) diff --git a/engforge/test/test_four_bar.py b/engforge/test/test_four_bar.py index 3bdefef..44d25b2 100644 --- a/engforge/test/test_four_bar.py +++ b/engforge/test/test_four_bar.py @@ -56,7 +56,9 @@ def r3_x_zero(self) -> float: def r3_x(self) -> float: """length defined by gamma/theta""" return ( - self.x2 + self.ra * numpy.cos(self.theta) - self.r1 * numpy.cos(self.gamma) + self.x2 + + self.ra * numpy.cos(self.theta) + - self.r1 * numpy.cos(self.gamma) ) @system_prop @@ -68,7 +70,9 @@ def y_zero(self) -> float: def y(self) -> float: """length defined by gamma/theta""" return ( - self.x2 + self.ra * numpy.sin(self.theta) - self.r1 * numpy.sin(self.gamma) + self.x2 + + self.ra * numpy.sin(self.theta) + - self.r1 * numpy.sin(self.gamma) ) @system_prop diff --git a/engforge/test/test_performance.py b/engforge/test/test_performance.py index bf90ef1..480dec8 100644 --- a/engforge/test/test_performance.py +++ b/engforge/test/test_performance.py @@ -77,7 +77,9 @@ def test_transient(self): parser.add_argument( "-a", "--all", action="store_true", dest="all", help="run all tests" ) - parser.add_argument("-b", "--base", action="store_true", help="run all tests") + parser.add_argument( + "-b", "--base", action="store_true", help="run all tests" + ) args = parser.parse_args() diff --git a/engforge/test/test_pipes.py b/engforge/test/test_pipes.py index eae7305..06d8e70 100644 --- a/engforge/test/test_pipes.py +++ b/engforge/test/test_pipes.py @@ -22,16 +22,32 @@ def test_pipe_analysis(self): n4za = PipeNode(x=10, y=10, z=10) n2zoa = FlowInput(x=10, y=5, z=15, flow_in=0.6) - pipe1 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n1, node_e=n2) - pipe2 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n1, node_e=n3) + pipe1 = Pipe( + v=random.random(), D=0.1 * random.random(), node_s=n1, node_e=n2 + ) + pipe2 = Pipe( + v=random.random(), D=0.1 * random.random(), node_s=n1, node_e=n3 + ) pipe3 = Pipe(v=random.random(), D=0.1, node_s=n2, node_e=n4) - pipe4 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n3, node_e=n4) - pipe5 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n4, node_e=n5) + pipe4 = Pipe( + v=random.random(), D=0.1 * random.random(), node_s=n3, node_e=n4 + ) + pipe5 = Pipe( + v=random.random(), D=0.1 * random.random(), node_s=n4, node_e=n5 + ) - pipe5 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n2z, node_e=n2) - pipe6 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n3, node_e=n3z) - pipe7 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n4, node_e=n4z) - pipe8 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n2, node_e=n4z) + pipe5 = Pipe( + v=random.random(), D=0.1 * random.random(), node_s=n2z, node_e=n2 + ) + pipe6 = Pipe( + v=random.random(), D=0.1 * random.random(), node_s=n3, node_e=n3z + ) + pipe7 = Pipe( + v=random.random(), D=0.1 * random.random(), node_s=n4, node_e=n4z + ) + pipe8 = Pipe( + v=random.random(), D=0.1 * random.random(), node_s=n2, node_e=n4z + ) pipe9 = Pipe( v=random.random(), D=0.1 * random.random(), node_s=n4zo, node_e=n4z ) diff --git a/engforge/test/test_problem.py b/engforge/test/test_problem.py index 5c57223..0109ccd 100644 --- a/engforge/test/test_problem.py +++ b/engforge/test/test_problem.py @@ -133,7 +133,9 @@ def test_slide_crank_design(self): def test_slide_crank_design_one_match(self): sm = SliderCrank(Tg=0) - pbx = ProblemExec(sm, {"combos": "design", "dxdt": None, "both_match": False}) + pbx = ProblemExec( + sm, {"combos": "design", "dxdt": None, "both_match": False} + ) atx = pbx.ref_attrs self.assertEqual( chk(atx, "solver.var"), @@ -144,7 +146,9 @@ def test_slide_crank_design_one_match(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv", "size")), ) - self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) + self.assertEqual( + chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) + ) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) @@ -156,7 +160,9 @@ def test_slide_crank_design_one_match(self): def test_slide_crank_design_slv_design(self): sm = SliderCrank(Tg=0) - pbx = ProblemExec(sm, {"combos": "design", "slv_vars": "*slv", "dxdt": None}) + pbx = ProblemExec( + sm, {"combos": "design", "slv_vars": "*slv", "dxdt": None} + ) atx = pbx.ref_attrs self.assertEqual(chk(atx, "solver.var"), set()) self.assertEqual(chk(atx, "solver.obj"), set(())) @@ -192,7 +198,9 @@ def test_slide_crank_design_slv(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv")), ) - self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) + self.assertEqual( + chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) + ) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) @@ -224,7 +232,9 @@ def test_slide_crank_design_slv_one_match(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv")), ) - self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) + self.assertEqual( + chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) + ) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) diff --git a/engforge/test/test_solver.py b/engforge/test/test_solver.py index 92dfd16..286837d 100644 --- a/engforge/test/test_solver.py +++ b/engforge/test/test_solver.py @@ -44,7 +44,9 @@ def test_comps_solver_obj(self): for act in [True, False]: for cmb in [inv, inv.split(",")]: actv = acts if act else aacts - extra = dict(combos=cmb, slv_vars="*", activate=actv, only_active=True) + extra = dict( + combos=cmb, slv_vars="*", activate=actv, only_active=True + ) info = self.sc.solver_vars(**extra)["attrs"] ans = {"x", "y", "z", "comp.x", "comp.y", "comp.z"} self.assertEqual(set(info["solver.var"]), ans) @@ -173,14 +175,18 @@ def setUp(self) -> None: self.sc = CubeSystem(comp=CubeComp()) def test_exec_results(self): - extra = dict(combos=indep_l, slv_vars=indep, activate=[], only_active=True) + extra = dict( + combos=indep_l, slv_vars=indep, activate=[], only_active=True + ) with ProblemExec(self.sc, extra) as pb: o = self.sc.execute(**extra) self.assertDictEqual(o["Xstart"], o["Xans"]) def test_run_results(self): """test that inputs stay the same when no objecives present""" - extra = dict(combos=indep, slv_vars=indep_l, activate=[], only_active=True) + extra = dict( + combos=indep, slv_vars=indep_l, activate=[], only_active=True + ) scx, scy, scz = self.sc.x, self.sc.y, self.sc.z sccx, sccy, sccz = self.sc.comp.x, self.sc.comp.y, self.sc.comp.z diff --git a/engforge/test/test_tabulation.py b/engforge/test/test_tabulation.py index 613f61d..30bab65 100644 --- a/engforge/test/test_tabulation.py +++ b/engforge/test/test_tabulation.py @@ -57,7 +57,9 @@ def setUp(self): # rfil = os.path.join(self.test_dir, fil) def test_property_labels(self): - ans = set(["four", "test_two", "test_one", "three", "converged", "run_id"]) + ans = set( + ["four", "test_two", "test_one", "three", "converged", "run_id"] + ) self.assertEqual(set(self.test_config.system_properties_labels), ans) def test_property_types(self): @@ -121,7 +123,9 @@ def test_assemble_data_on_input(self): # Change Something cur_val = self.test_config.attrs_prop new_val = 6 + cur_val - self.test_config.info(f"setting attrs prop on in {cur_val } => {new_val}") + self.test_config.info( + f"setting attrs prop on in {cur_val } => {new_val}" + ) self.test_config.attrs_prop = new_val px.save_data() @@ -135,7 +139,9 @@ def test_dataframe(self, iter=5): with ProblemExec(self.test_config, {}) as px: cur_val = self.test_config.attrs_prop attr_in[i] = val = cur_val + i**2.0 - self.test_config.info(f"setting attrs prop df {cur_val } => {val}") + self.test_config.info( + f"setting attrs prop df {cur_val } => {val}" + ) self.test_config.attrs_prop = val px.save_data() px.exit_with_state() @@ -225,7 +231,9 @@ def test_input(self, num=10): with ProblemExec(self.test_config, {}) as px: for i in range(num): with ProblemExec(self.test_config, {}) as px: - self.test_config.attrs_prop = i + self.test_config.attrs_prop + self.test_config.attrs_prop = ( + i + self.test_config.attrs_prop + ) px.save_data() postdict = self.test_config.data_dict self.assertDictEqual(cur_dict, postdict) diff --git a/engforge/typing.py b/engforge/typing.py index e634f62..9109207 100644 --- a/engforge/typing.py +++ b/engforge/typing.py @@ -53,8 +53,10 @@ def Options(*choices, **kwargs): :param kwargs: keyword args passed to attrs field""" assert choices, f"must have some choices!" assert "type" not in kwargs, "options type set is str" - valids = set((str,int,float,bool,type(None))) - assert set([type(c) for c in choices]).issubset(valids), f"choices must be in {valids}" + valids = set((str, int, float, bool, type(None))) + assert set([type(c) for c in choices]).issubset( + valids + ), f"choices must be in {valids}" assert "on_setattr" not in kwargs validators = [attrs.validators.in_(choices)] diff --git a/examples/air_filter.py b/examples/air_filter.py index 1c98fc6..023c14c 100644 --- a/examples/air_filter.py +++ b/examples/air_filter.py @@ -48,7 +48,9 @@ class Airfilter(System): pr_eq = Solver.constraint_equality("sum_dP", 0, combos="flow") - flow_curve = Plot.define("throttle", "w", kind="lineplot", title="Flow Curve") + flow_curve = Plot.define( + "throttle", "w", kind="lineplot", title="Flow Curve" + ) @system_property def dP_parasitic(self) -> float: diff --git a/examples/spring_mass.py b/examples/spring_mass.py index 42a1660..f2ab064 100644 --- a/examples/spring_mass.py +++ b/examples/spring_mass.py @@ -68,5 +68,7 @@ def accl(self) -> float: if __name__ == "__main__": # Run The System, Compare damping `u`=0 & 0.1 sm = SpringMass(x=0.0) - trdf = sm.simulate(dt=0.01, endtime=10, u=[0.0, 0.1], combos="*", slv_vars="*") + trdf = sm.simulate( + dt=0.01, endtime=10, u=[0.0, 0.1], combos="*", slv_vars="*" + ) trdf.groupby("run_id").plot("time", "x") diff --git a/test/test_problem.py b/test/test_problem.py index 44ac653..3b5c7f4 100644 --- a/test/test_problem.py +++ b/test/test_problem.py @@ -133,7 +133,9 @@ def test_slide_crank_design(self): def test_slide_crank_design_one_match(self): sm = SliderCrank(Tg=0) - pbx = ProblemExec(sm, {"combos": "design", "dxdt": None, "both_match": False}) + pbx = ProblemExec( + sm, {"combos": "design", "dxdt": None, "both_match": False} + ) atx = pbx.ref_attrs self.assertEqual( chk(atx, "solver.var"), @@ -144,7 +146,9 @@ def test_slide_crank_design_one_match(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv", "size")), ) - self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) + self.assertEqual( + chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) + ) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) @@ -156,7 +160,9 @@ def test_slide_crank_design_one_match(self): def test_slide_crank_design_slv_design(self): sm = SliderCrank(Tg=0) - pbx = ProblemExec(sm, {"combos": "design", "slv_vars": "*slv", "dxdt": None}) + pbx = ProblemExec( + sm, {"combos": "design", "slv_vars": "*slv", "dxdt": None} + ) atx = pbx.ref_attrs self.assertEqual(chk(atx, "solver.var"), set()) self.assertEqual(chk(atx, "solver.obj"), set(())) @@ -192,7 +198,9 @@ def test_slide_crank_design_slv(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv")), ) - self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) + self.assertEqual( + chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) + ) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) @@ -224,7 +232,9 @@ def test_slide_crank_design_slv_one_match(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv")), ) - self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) + self.assertEqual( + chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) + ) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) From be00cc0df9706914b37c0f1b58439d188ecc338b Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Mon, 28 Oct 2024 18:33:29 -0400 Subject: [PATCH 15/15] black formatting default --- .pre-commit-config.yaml | 11 ++ docs/conf.py | 8 +- engforge/_testing_components.py | 64 ++---- engforge/analysis.py | 4 +- engforge/attr_plotting.py | 36 +--- engforge/attr_signals.py | 4 +- engforge/attr_slots.py | 4 +- engforge/attr_solver.py | 8 +- engforge/attributes.py | 12 +- engforge/common.py | 4 +- engforge/component_collections.py | 6 +- engforge/components.py | 5 +- engforge/configuration.py | 34 +--- engforge/datastores/data.py | 21 +- engforge/datastores/gdocs.py | 267 +++++++------------------- engforge/datastores/reporting.py | 95 +++------ engforge/datastores/secrets.py | 35 +--- engforge/dynamics.py | 38 +--- engforge/eng/costs.py | 112 +++-------- engforge/eng/fluid_material.py | 4 +- engforge/eng/geometry.py | 43 ++--- engforge/eng/prediction.py | 21 +- engforge/eng/structure.py | 61 ++---- engforge/eng/structure_beams.py | 29 +-- engforge/eng/thermodynamics.py | 5 +- engforge/engforge_attributes.py | 8 +- engforge/env_var.py | 12 +- engforge/logging.py | 20 +- engforge/patterns.py | 4 +- engforge/problem_context.py | 100 +++------- engforge/properties.py | 12 +- engforge/solveable.py | 62 ++---- engforge/solver.py | 12 +- engforge/solver_utils.py | 16 +- engforge/system.py | 4 +- engforge/system_reference.py | 4 +- engforge/tabulation.py | 32 +-- engforge/test/_pre_test_structures.py | 16 +- engforge/test/report_testing.py | 8 +- engforge/test/test_comp_iter.py | 4 +- engforge/test/test_costs.py | 3 +- engforge/test/test_dynamics_spaces.py | 4 +- engforge/test/test_four_bar.py | 8 +- engforge/test/test_performance.py | 4 +- engforge/test/test_pipes.py | 32 +-- engforge/test/test_problem.py | 20 +- engforge/test/test_solver.py | 12 +- engforge/test/test_tabulation.py | 16 +- examples/air_filter.py | 4 +- examples/spring_mass.py | 4 +- test/test_problem.py | 20 +- 51 files changed, 351 insertions(+), 1021 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..cbc3411 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 199c0f5..fbc5966 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,11 +47,11 @@ False # Remove 'view source code' from top of page (for html, not python) ) autodoc_inherit_docstrings = True # If no docstring, inherit from base class -set_type_checking_flag = ( - True # Enable 'expensive' imports for sphinx_autodoc_typehints -) +set_type_checking_flag = True # Enable 'expensive' imports for sphinx_autodoc_typehints # nbsphinx_allow_errors = True # Continue through Jupyter errors -autodoc_typehints = "description" # Sphinx-native method. Not as good as sphinx_autodoc_typehints +autodoc_typehints = ( + "description" # Sphinx-native method. Not as good as sphinx_autodoc_typehints +) add_module_names = False # Remove namespaces from class/method signatures html_theme = "sphinx_rtd_theme" diff --git a/engforge/_testing_components.py b/engforge/_testing_components.py index 4a9d469..cf43fa8 100644 --- a/engforge/_testing_components.py +++ b/engforge/_testing_components.py @@ -95,23 +95,15 @@ class SpaceMixin(SolveableInterface): lenP = Solver.con_ineq("edge_margin", combos="ineq_length", active=False) # Objectives - size_goal = Solver.objective( - "goal", combos="prop_goal", kind="min", active=False - ) + size_goal = Solver.objective("goal", combos="prop_goal", kind="min", active=False) - size = Solver.objective( - "volume", combos="obj_size", kind="max", active=False - ) + size = Solver.objective("volume", combos="obj_size", kind="max", active=False) sizeF = Solver.objective( wrap_f(Fun_size), combos="obj_size", kind="max", active=False ) - eff = Solver.objective( - "cost_to_volume", combos="obj_eff", kind="min", active=False - ) - effF = Solver.objective( - wrap_f(Fun_eff), combos="obj_eff", kind="min", active=False - ) + eff = Solver.objective("cost_to_volume", combos="obj_eff", kind="min", active=False) + effF = Solver.objective(wrap_f(Fun_eff), combos="obj_eff", kind="min", active=False) @system_property def combine_length(self) -> float: @@ -184,13 +176,9 @@ class CubeSystem(System, SpaceMixin): goal_vol_frac: float = 0.5 - sys_budget = Solver.con_ineq( - "total_budget", "system_cost", combos="total_budget" - ) + sys_budget = Solver.con_ineq("total_budget", "system_cost", combos="total_budget") - sys_length = Solver.con_ineq( - "total_length", "system_length", combos="total_length" - ) + sys_length = Solver.con_ineq("total_length", "system_length", combos="total_length") volfrac = Solver.con_eq( "goal_vol_frac", "vol_frac", combos="vol_frac_eq", active=False @@ -331,8 +319,7 @@ class DynamicSystem(System): def spring_accel(self) -> float: # print(self.comp.v,self.comp.x,self.comp.a) return ( - -self.comp.v * self.comp.b - - (self.comp.x - self.comp.x0) * self.comp.K + -self.comp.v * self.comp.b - (self.comp.x - self.comp.x0) * self.comp.K ) / self.comp.M @system_property @@ -346,9 +333,7 @@ def delta_a(self) -> float: def create_state_matrix(self, *args, **kwargs) -> np.ndarray: """creates the state matrix for the system""" - return np.array( - [[0, 1.0], [-self.K / self.Mass, -1 * self.Damp / self.Mass]] - ) + return np.array([[0, 1.0], [-self.K / self.Mass, -1 * self.Damp / self.Mass]]) def create_state_constants(self, *args, **kwargs) -> np.ndarray: """creates the input matrix for the system, called B""" @@ -356,9 +341,7 @@ def create_state_constants(self, *args, **kwargs) -> np.ndarray: def update_state(self, *args, **kwargs) -> np.ndarray: """creates the state matrix for the system""" - return np.array( - [[0, 1.0], [-self.K / self.Mass, -1 * self.Damp / self.Mass]] - ) + return np.array([[0, 1.0], [-self.K / self.Mass, -1 * self.Damp / self.Mass]]) def update_state_constants(self, *args, **kwargs) -> np.ndarray: """creates the input matrix for the system, called B""" @@ -466,9 +449,7 @@ class Airfilter(System): pr_eq = Solver.constraint_equality("sum_dP", 0, combos="flow") - flow_curve = Plot.define( - "throttle", "w", kind="lineplot", title="Flow Curve" - ) + flow_curve = Plot.define("throttle", "w", kind="lineplot", title="Flow Curve") @system_property def dP_parasitic(self) -> float: @@ -549,9 +530,7 @@ class SliderCrank(System, CostModel): rg_slv.add_var_constraint(0.0125, "min") rg_slv = Solver.declare_var("Lo", combos="design") - rg_slv.add_var_constraint( - lambda s, p: s.Rc * s.Lo_factor, "min", combos="design" - ) + rg_slv.add_var_constraint(lambda s, p: s.Rc * s.Lo_factor, "min", combos="design") offy_slv = Solver.declare_var("y_offset", combos="design") offx_slv = Solver.declare_var("x_offset", combos="design") @@ -641,8 +620,7 @@ def gamma(self) -> float: def motion_curve(self) -> np.ndarray: x = ( self.Rc * np.cos(theta) - + (self.Lo**2 - (self.Rc * np.sin(theta) - self.y_offset) ** 2) - ** 0.5 + + (self.Lo**2 - (self.Rc * np.sin(theta) - self.y_offset) ** 2) ** 0.5 ) return x @@ -701,12 +679,7 @@ def ds_goal(self) -> float: @system_property def mass_main_gear(self) -> float: - return ( - np.pi - * self.gear_material.density - * self.Ro**2 - * self.gear_thickness - ) + return np.pi * self.gear_material.density * self.Ro**2 * self.gear_thickness @system_property def Imain_gear(self) -> float: @@ -714,12 +687,7 @@ def Imain_gear(self) -> float: @system_property def mass_pwr_gear(self) -> float: - return ( - np.pi - * self.gear_material.density - * self.Rg**2 - * self.gear_thickness - ) + return np.pi * self.gear_material.density * self.Rg**2 * self.gear_thickness @system_property def Ipwr_gear(self) -> float: @@ -809,9 +777,7 @@ def cost_maintenance(self): def cost_tax(self): return 1 - @cost_property( - mode=quarterly, category="opex,tax", label="quarterly wage tax" - ) + @cost_property(mode=quarterly, category="opex,tax", label="quarterly wage tax") def cost_wage_tax(self): return 5 * 3 diff --git a/engforge/analysis.py b/engforge/analysis.py index bc7b707..0014b5f 100644 --- a/engforge/analysis.py +++ b/engforge/analysis.py @@ -64,9 +64,7 @@ def uploaded(self): def run(self, *args, **kwargs): """Analysis.run() passes inputs to the assigned system and saves data via the system.run(cb=callback), once complete `Analysis.post_process()` is run also being passed input arguments, then plots & reports are made""" - self.info( - f"running analysis {self.identity} with input {args} {kwargs}" - ) + self.info(f"running analysis {self.identity} with input {args} {kwargs}") cb = lambda *args, **kw: self.system.last_context.save_data(force=True) out = self.system.run(*args, **kwargs, cb=cb) self.post_process(*args, **kwargs) diff --git a/engforge/attr_plotting.py b/engforge/attr_plotting.py index 1206083..8762cdd 100644 --- a/engforge/attr_plotting.py +++ b/engforge/attr_plotting.py @@ -257,9 +257,7 @@ def __init__(self, system: "System", plot_cls: "Plot"): self.system = system _sys_refs = self.system.system_references() - sys_refs = { - k: v for atr, grp in _sys_refs.items() for k, v in grp.items() - } + sys_refs = {k: v for atr, grp in _sys_refs.items() for k, v in grp.items()} diff = set() varss = set() @@ -278,15 +276,11 @@ def __init__(self, system: "System", plot_cls: "Plot"): if self.system.log_level < 10: log.debug(f"system references: {sys_refs}") if diff: - log.warning( - f"has diff {diff}| found: {varss}| possible: {sys_refs}" - ) + log.warning(f"has diff {diff}| found: {varss}| possible: {sys_refs}") if diff: # raise KeyError(f"has system diff: {diff} found: {vars}| from: {sys_ref}") - log.warning( - f"has system diff: {diff} found: {vars}| from: {sys_refs}" - ) + log.warning(f"has system diff: {diff} found: {vars}| from: {sys_refs}") self.refs = {k: sys_refs[k] for k in varss} @@ -323,9 +317,7 @@ def __call__(self, **override_kw): log.debug(f"overriding vars {override_kw}") args.update(**override_kw) - log.info( - f"plotting {self.system.identity}| {self.identity} with {args}" - ) + log.info(f"plotting {self.system.identity}| {self.identity} with {args}") fig = ax = f(data=self.system.dataframe, **args, **extra) return self.process_fig(fig, title) @@ -410,14 +402,10 @@ def validate_plot_args(cls, system: "System"): diff.add(vers) if log.log_level <= 10: - log.debug( - f"{cls.__name__} has vars: {attr_keys} and bad input: {diff}" - ) + log.debug(f"{cls.__name__} has vars: {attr_keys} and bad input: {diff}") if diff: - log.warning( - f"bad plot vars: {diff} do not exist in system: {valid}" - ) + log.warning(f"bad plot vars: {diff} do not exist in system: {valid}") # TODO: fix time being defined on components # raise KeyError( # f"bad plot vars: {diff} do not exist in system: {valid}" @@ -480,9 +468,7 @@ def __call__(self, **override_kw): log.debug(f"overriding vars {override_kw}") args.update(**override_kw) - log.info( - f"plotting {self.system.identity}| {self.identity} with {args}" - ) + log.info(f"plotting {self.system.identity}| {self.identity} with {args}") # PLOTTING # Make the axes and plot @@ -586,9 +572,7 @@ class Trace(PlotBase): always = ("x", "y", "y2") @classmethod - def define( - cls, x="time", y: trace_type = None, y2=None, kind="line", **kwargs - ): + def define(cls, x="time", y: trace_type = None, y2=None, kind="line", **kwargs): """Defines a plot that will be matplotlib, with validation happening as much as possible in the define method #Plot Choice @@ -764,9 +748,7 @@ def define( """ # Validate Plot - assert ( - _type in PLOT_KINDS - ), f"type {_type} must be in {PLOT_KINDS.keys()}" + assert _type in PLOT_KINDS, f"type {_type} must be in {PLOT_KINDS.keys()}" kinds = PLOT_KINDS[_type] assert kind in kinds, f"plot kind {kind} not in {kinds}" diff --git a/engforge/attr_signals.py b/engforge/attr_signals.py index 44cf6ea..62820a9 100644 --- a/engforge/attr_signals.py +++ b/engforge/attr_signals.py @@ -37,9 +37,7 @@ def apply(self): """sets `target` from `source`""" val = self.source.value() if self.system.log_level < 10: - self.system.msg( - f"Signal| applying {self.source}|{val} to {self.target}" - ) + self.system.msg(f"Signal| applying {self.source}|{val} to {self.target}") self.target.set_value(val) @property diff --git a/engforge/attr_slots.py b/engforge/attr_slots.py index 8278fa5..05f10b5 100644 --- a/engforge/attr_slots.py +++ b/engforge/attr_slots.py @@ -190,9 +190,7 @@ def make_factory(cls, **kwargs): log.debug(f"slot factory: {accepted},{cls.dflt_kw},{cls.default_ok}") if cls.dflt_kw: - return attrs.Factory( - cls.make_accepted(accepted, **cls.dflt_kw), False - ) + return attrs.Factory(cls.make_accepted(accepted, **cls.dflt_kw), False) elif cls.default_ok: return attrs.Factory(accepted, False) else: diff --git a/engforge/attr_solver.py b/engforge/attr_solver.py index f24f47f..4da79f7 100644 --- a/engforge/attr_solver.py +++ b/engforge/attr_solver.py @@ -212,9 +212,7 @@ def configure_for_system(cls, name, config_class, cb=None, **kwargs): :returns: [optional] a dictionary of options to be used in the make_attribute method """ pre_name = cls.name # random attr name - super(Solver, cls).configure_for_system( - name, config_class, cb, **kwargs - ) + super(Solver, cls).configure_for_system(name, config_class, cb, **kwargs) # change name of constraint var if if cls.slvtype == "var": @@ -401,9 +399,7 @@ def add_var_constraint(cls, value, kind="min", **kwargs): var = cls.var assert var is not None, "must provide var on non-var solvers" - assert ( - cls.slvtype == "var" - ), "only Solver.declare_var can have constraints" + assert cls.slvtype == "var", "only Solver.declare_var can have constraints" assert kind in ("min", "max") combo_dflt = "default,lim" diff --git a/engforge/attributes.py b/engforge/attributes.py index 30ea302..1853384 100644 --- a/engforge/attributes.py +++ b/engforge/attributes.py @@ -25,9 +25,7 @@ class AttributeInstance: # TODO: universal slots method # __slots__ = ["system", "class_attr"] - def __init__( - self, class_attr: "CLASS_ATTR", system: "System", **kwargs - ) -> None: + def __init__(self, class_attr: "CLASS_ATTR", system: "System", **kwargs) -> None: self.class_attr = class_attr self.system = system self.compile(**kwargs) @@ -237,9 +235,7 @@ def collect_cls(cls, system) -> dict: """collects all the attributes for a system""" if not isinstance(system, type): system = system.__class__ - return { - k: at.type for k, at in system._get_init_attrs_data(cls).items() - } + return {k: at.type for k, at in system._get_init_attrs_data(cls).items()} @classmethod def collect_attr_inst(cls, system, handle_inst=True) -> dict: @@ -248,9 +244,7 @@ def collect_attr_inst(cls, system, handle_inst=True) -> dict: out = {} for k, v in cattr.items(): inst = getattr(system, k) - if ( - inst is None and getattr(cls.instance_class, "none_ok", False) - ) or ( + if (inst is None and getattr(cls.instance_class, "none_ok", False)) or ( cls.instance_class is not None and not isinstance(inst, cls.instance_class) ): diff --git a/engforge/common.py b/engforge/common.py index 9213087..3168d17 100644 --- a/engforge/common.py +++ b/engforge/common.py @@ -58,9 +58,7 @@ def get_size(obj, seen=None): size += sum([get_size(k, seen) for k in obj.keys()]) elif hasattr(obj, "__dict__"): size += get_size(obj.__dict__, seen) - elif hasattr(obj, "__iter__") and not isinstance( - obj, (str, bytes, bytearray) - ): + elif hasattr(obj, "__iter__") and not isinstance(obj, (str, bytes, bytearray)): size += sum([get_size(i, seen) for i in obj]) return size diff --git a/engforge/component_collections.py b/engforge/component_collections.py index 61d902e..ff83bfa 100644 --- a/engforge/component_collections.py +++ b/engforge/component_collections.py @@ -28,11 +28,7 @@ def check_comp_type(instance, attr, value): """ensures the input component type is a Component""" from engforge.eng.costs import CostModel - if ( - not instance.wide - and isinstance(value, type) - and issubclass(value, CostModel) - ): + if not instance.wide and isinstance(value, type) and issubclass(value, CostModel): raise TypeError(f"Cost Mixin Not Supported As Iter Type! {value}") if isinstance(value, type) and issubclass(value, Component): diff --git a/engforge/components.py b/engforge/components.py index 2016867..e1908cd 100644 --- a/engforge/components.py +++ b/engforge/components.py @@ -30,10 +30,7 @@ def last_context(self): """get the last context run, or the parent's""" if hasattr(self, "_last_context"): # cleanup parent context - if ( - hasattr(self, "_parent_context") - and not self.parent.last_context - ): + if hasattr(self, "_parent_context") and not self.parent.last_context: del self._last_context del self._parent_context else: diff --git a/engforge/configuration.py b/engforge/configuration.py index f3be4a4..add80b4 100644 --- a/engforge/configuration.py +++ b/engforge/configuration.py @@ -56,9 +56,7 @@ def name_generator(instance): """a name generator for the instance""" base = str(instance.__class__.__name__).lower() + "-" if instance.__class__._use_random_name: - out = base + randomname.get_name( - adj=NAME_ADJ.secret, noun=NAME_NOUN.secret - ) + out = base + randomname.get_name(adj=NAME_ADJ.secret, noun=NAME_NOUN.secret) else: out = base log.debug(f"generated name: {out}") @@ -166,9 +164,7 @@ def property_changed(instance, variable, value): if not session and instance._anything_changed: # Bypass Check since we've already flagged for an update if log.log_level <= 2: - log.debug( - f"already property changed {instance}{variable.name} {value}" - ) + log.debug(f"already property changed {instance}{variable.name} {value}") return value # elif session: @@ -179,9 +175,7 @@ def property_changed(instance, variable, value): # session.change_sys_var(variable,value,doset=False) attrs = attr.fields(instance.__class__) # check identity of variable cur = getattr(instance, variable.name) - is_different = ( - value != cur if isinstance(value, (int, float, str)) else True - ) + is_different = value != cur if isinstance(value, (int, float, str)) else True is_var = variable in attrs chgnw = instance._anything_changed @@ -247,9 +241,7 @@ def signals_slots_handler( # Fields for t in fields: if t.name in PROTECTED_NAMES: - raise Exception( - f"cannot use {t.name} as a field name, its protected" - ) + raise Exception(f"cannot use {t.name} as a field name, its protected") if t.type is None: log.warning(f"{cls.__name__}.{t.name} has no type") @@ -419,9 +411,7 @@ def internal_configurations( if not use_dict: # slots obj = {k: obj.get(k, None) for k in slots} - return { - k: v for k, v in obj.items() if chk(k, v) and not k.startswith("_") - } + return {k: v for k, v in obj.items() if chk(k, v) and not k.startswith("_")} def copy_config_at_state( self, level=None, levels_deep: int = -1, changed: dict = None, **kw @@ -461,9 +451,7 @@ def copy_config_at_state( if config in changed: ccomp = changed[config] else: - ccomp = config.copy_config_at_state( - level + 1, levels_deep, changed - ) + ccomp = config.copy_config_at_state(level + 1, levels_deep, changed) kwcomps[key] = ccomp # Finally make the new system with changed internals @@ -575,9 +563,7 @@ def __attrs_post_init__(self): # subclass instance instance init causes conflicts in structures self.__on_init__() - runs = set( - (self.__class__.__on_init__,) - ) # keep track of unique functions + runs = set((self.__class__.__on_init__,)) # keep track of unique functions if self._subclass_init: try: for comp in self.__class__.mro(): @@ -638,11 +624,7 @@ def filename(self): .title() ) filename = "".join( - [ - c - for c in fil - if c.isalpha() or c.isdigit() or c == "_" or c == "-" - ] + [c for c in fil if c.isalpha() or c.isdigit() or c == "_" or c == "-"] ).rstrip() return filename diff --git a/engforge/datastores/data.py b/engforge/datastores/data.py index bcae433..a7f798e 100644 --- a/engforge/datastores/data.py +++ b/engforge/datastores/data.py @@ -289,9 +289,7 @@ def get(self, key=None, on_missing=None, retry=True, ttl=None): ttl -= 1 if ttl > 0: time.sleep(self.sleep_time * (self.retries - ttl)) - return self.get( - key=key, on_missing=on_missing, retry=True, ttl=ttl - ) + return self.get(key=key, on_missing=on_missing, retry=True, ttl=ttl) else: self.error(e, "Issue Getting Item From Cache") @@ -299,10 +297,7 @@ def expire(self): """wrapper for diskcache expire method that only permits expiration on a certain interval :return: bool, True if expired called""" now = time.time() - if ( - self.last_expire is None - or now - self.last_expire > self.expire_threshold - ): + if self.last_expire is None or now - self.last_expire > self.expire_threshold: self.cache.expire() self.last_expire = now return True @@ -341,9 +336,7 @@ class DBConnection(LoggingMixin, metaclass=InputSingletonMeta): # TODO: Make Threadsafe W/ ThreadPoolExecutor! # we love postgres! - _connection_template = ( - "postgresql://{user}:{passd}@{host}:{port}/{database}" - ) + _connection_template = "postgresql://{user}:{passd}@{host}:{port}/{database}" pool_size = 20 max_overflow = 0 @@ -366,9 +359,7 @@ class DBConnection(LoggingMixin, metaclass=InputSingletonMeta): connect_args = {"connect_timeout": 5} - def __init__( - self, database_name=None, host=None, user=None, passd=None, **kwargs - ): + def __init__(self, database_name=None, host=None, user=None, passd=None, **kwargs): """On the Singleton DBconnection.instance(): __init__(*args,**kwargs) will get called, technically you could do it this way but won't be thread safe, or a single instance :param database_name: the name for the database inside the db server @@ -443,9 +434,7 @@ def configure(self): # self.scopefunc = functools.partial(context.get, "uuid") - self.session_factory = sessionmaker( - bind=self.engine, expire_on_commit=True - ) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=True) self.Session = scoped_session(self.session_factory) @contextmanager diff --git a/engforge/datastores/gdocs.py b/engforge/datastores/gdocs.py index f22147b..fd3242c 100644 --- a/engforge/datastores/gdocs.py +++ b/engforge/datastores/gdocs.py @@ -177,10 +177,8 @@ def is_folder(self): if self.is_drivefile: if any( [ - self.item["mimeType"] - == "application/vnd.google-apps.folder", - self.item["mimeType"] - == "application/vnd.google-apps.shortcut", + self.item["mimeType"] == "application/vnd.google-apps.folder", + self.item["mimeType"] == "application/vnd.google-apps.shortcut", ] ): return True @@ -191,10 +189,8 @@ def is_file(self): if self.is_drivefile: if not any( [ - self.item["mimeType"] - == "application/vnd.google-apps.folder", - self.item["mimeType"] - == "application/vnd.google-apps.shortcut", + self.item["mimeType"] == "application/vnd.google-apps.folder", + self.item["mimeType"] == "application/vnd.google-apps.shortcut", ] ): return True @@ -216,9 +212,7 @@ def is_protected(self): return True # Protected Folder Name if self.title in self.drive.protected_filenames and self.is_folder: - self.debug( - f'folder has a protected filename {self.item["title"]}' - ) + self.debug(f'folder has a protected filename {self.item["title"]}') return True # File Id is protected elif self.id in self.drive.protected_ids: @@ -291,9 +285,7 @@ def absolute_paths(self): npaths = self.best_nodeid_paths if npaths: return [ - os.path.join( - *[self.drive.item_nodes[itdd].title for itdd in path_list] - ) + os.path.join(*[self.drive.item_nodes[itdd].title for itdd in path_list]) for path_list in npaths ] @@ -302,9 +294,7 @@ def best_nodeid_path(self): try: with self.filesystem as fs: return list( - nx.shortest_path( - fs, source=self.drive.sync_root_id, target=self.id - ) + nx.shortest_path(fs, source=self.drive.sync_root_id, target=self.id) ) except Exception as e: self.error(e) @@ -324,9 +314,7 @@ def absolute_path(self): parents = self.parents if parents: - self._absolute_path = os.path.join( - parents[0].absolute_path, self.title - ) + self._absolute_path = os.path.join(parents[0].absolute_path, self.title) # self.debug(f'caching file abspath for node {self._absolute_path}') else: self._absolute_path = self.title @@ -358,9 +346,7 @@ def ensure_http(self): def delete(self): self.ensure_http() if not self.is_protected and not self.drive.dry_run: - with self.drive.rate_limit_manager( - self.delete, 2, gfileMeta=self.item - ): + with self.drive.rate_limit_manager(self.delete, 2, gfileMeta=self.item): self.info(f"deleting: {self.title}") self.item.Trash() self.drive.sleep() @@ -484,8 +470,7 @@ def absolute_path(self): timestamp_divider = 24 * 3600 # a day in seconds days_since_2020 = ( - lambda item: (item.createdDate.timestamp() - default_timestamp) - / timestamp_divider + lambda item: (item.createdDate.timestamp() - default_timestamp) / timestamp_divider ) content_size = lambda item: len(item.contents) @@ -760,19 +745,15 @@ def authoirze_google_integrations(self, retry=True, ttl=3): self.sleep() self.gauth.auth_method = "service" - self.gauth.credentials = ( - ServiceAccountCredentials.from_json_keyfile_name( - self.creds_file, scope - ) + self.gauth.credentials = ServiceAccountCredentials.from_json_keyfile_name( + self.creds_file, scope ) self.gauth.Authorize() self.sleep() # Do Sheets Authentication - self.gsheets = pygsheets.authorize( - service_account_file=self.creds_file - ) + self.gsheets = pygsheets.authorize(service_account_file=self.creds_file) self.sleep() self.gdrive = GoogleDrive(self.gauth) @@ -811,13 +792,11 @@ def reset(self): def sync_path(self, path): """Sync path likes absolute paths to google drive relative""" - assert os.path.commonpath( - [self.filepath_root, path] - ) == os.path.commonpath([self.filepath_root]) - - self.debug( - f"finding relative path from {path} and {self.filepath_root}" + assert os.path.commonpath([self.filepath_root, path]) == os.path.commonpath( + [self.filepath_root] ) + + self.debug(f"finding relative path from {path} and {self.filepath_root}") rel_root = os.path.relpath(path, self.filepath_root) # self.debug(f'getting gdrive path {self.full_sync_root} and {rel_root}') @@ -898,9 +877,7 @@ class token: if protect: self.protected_ids.add(parent_id) - if ( - pool is None - ): # and self.use_threadpool #Always use threads for this! + if pool is None: # and self.use_threadpool #Always use threads for this! if top: self.debug("GET DRIVE INFO USING THREADS!!!") pool = ThreadPoolExecutor(max_workers=self.num_threads) @@ -954,9 +931,7 @@ class token: ) # TODO: handle duplicates - if ( - item.is_folder - ): # if it had contents it was already cached!! + if item.is_folder: # if it had contents it was already cached!! # Either do not stop or ensure that the item path is in the target if ( stop_when_found is None @@ -1030,9 +1005,7 @@ def rate_limit_manager(self, retry_function, multiplier, *args, **kwargs): except ConnectionError as err: self.info("connection reset, retrying auth") - self.hit_rate_limit( - 30.0 + 30.0 * random.random(), multiplier=multiplier - ) + self.hit_rate_limit(30.0 + 30.0 * random.random(), multiplier=multiplier) self.authoirze_google_integrations() self.sleep(10.0 + 10.0 * random.random()) return retry_function(*args, **kwargs) @@ -1131,9 +1104,7 @@ def initalize_google_drive_root(self): if "client" in sfol.lower(): for clientname in engforge_clients(): if clientname.lower() != "archive": - self.get_or_create_folder( - clientname, parent_id=fol["id"] - ) + self.get_or_create_folder(clientname, parent_id=fol["id"]) # Sync Methods def generate_sync_filepath_pairs(self, skip_existing=True): @@ -1149,9 +1120,7 @@ def generate_sync_filepath_pairs(self, skip_existing=True): parent_id = self.target_folder_id self.cache_directory(parent_id) - for i, (dirpath, dirnames, dirfiles) in enumerate( - os.walk(self.filepath_root) - ): + for i, (dirpath, dirnames, dirfiles) in enumerate(os.walk(self.filepath_root)): # we will modify dirnames in place to eliminate options for walking self.debug("looping through directory {}".format(dirpath)) # Handle File Sync Ignores @@ -1175,8 +1144,7 @@ def generate_sync_filepath_pairs(self, skip_existing=True): if any( [ - os.path.commonpath([dirpath, spath]) - == os.path.commonpath([spath]) + os.path.commonpath([dirpath, spath]) == os.path.commonpath([spath]) for spath in skipped_paths ] ): @@ -1254,9 +1222,7 @@ def sync(self, force=False): self.thread_time_multiplier = 1.0 # ensure its normal # #Conventional way make folders if they don't exist submitted_set = set() - with ThreadPoolExecutor( - max_workers=self.num_threads - ) as pool: + with ThreadPoolExecutor(max_workers=self.num_threads) as pool: for lpath, gpath in self.generate_sync_filepath_pairs( skip_existing=not force ): @@ -1282,9 +1248,7 @@ def sync(self, force=False): ) else: if gpath in self.item_paths: - self.debug( - "found existing {}".format(gpath) - ) + self.debug("found existing {}".format(gpath)) self.thread_time_multiplier = 1.0 @@ -1302,9 +1266,7 @@ def sync_path_pair_single(self, filepath, syncpath): if syncpath in self.item_paths: self.debug(f"updating {filepath} -> {syncpath}") fid = self.get_gpath_id(syncpath) - self.upload_or_update_file( - par_id, file_path=filepath, file_id=fid - ) + self.upload_or_update_file(par_id, file_path=filepath, file_id=fid) self.cache_directory(par_id) else: self.debug(f"uploading {filepath} -> {syncpath}") @@ -1319,9 +1281,7 @@ def sync_path_pair_thread(self, filepath, syncpath, parent_id=None): if syncpath in self.item_paths: self.debug(f"updating {filepath} -> {syncpath}") fid = self.get_gpath_id(syncpath) - self.upload_or_update_file( - parent_id, file_path=filepath, file_id=fid - ) + self.upload_or_update_file(parent_id, file_path=filepath, file_id=fid) self.cache_directory(parent_id) else: self.debug(f"uploading {filepath} -> {syncpath}") @@ -1350,13 +1310,13 @@ def create_file(self, input_args, file_path=None, content=None): with self.rate_limit_manager( self.create_file, 2, input_args, file_path, content ): - if not self.dry_run and any( - (file_path is not None, content is not None) - ): + if not self.dry_run and any((file_path is not None, content is not None)): action = "creating" if "id" not in input_args else "updating" if content is not None: - gfile_path = file_path # Direct conversion, we have to input correctly + gfile_path = ( + file_path # Direct conversion, we have to input correctly + ) if isinstance(content, (str, unicode)): self.debug( f"{action} file with content string {input_args} -> {gfile_path}" @@ -1374,15 +1334,11 @@ def create_file(self, input_args, file_path=None, content=None): file_name = os.path.basename(file_path) gfile_path = self.sync_path(file_path) - self.debug( - f"{action} file with args {input_args} -> {gfile_path}" - ) + self.debug(f"{action} file with args {input_args} -> {gfile_path}") file.SetContentFile(file_path) self.sleep() - file.Upload( - param={"supportsTeamDrives": True} - ) # Upload the file. + file.Upload(param={"supportsTeamDrives": True}) # Upload the file. self.sleep() file.FetchMetadata(fields="permissions,labels,mimeType") self.sleep() @@ -1420,9 +1376,7 @@ def upload_or_update_file( if file_id is not None: # Override takes precident self.debug( - "updating id: {} {}->{}".format( - file_id, parent_id, gfile_path - ) + "updating id: {} {}->{}".format(file_id, parent_id, gfile_path) ) fil = self.create_file( @@ -1458,9 +1412,7 @@ def upload_or_update_file( if file_id is not None: self.debug( - "updating file w/ content id:{} par:{}".format( - file_id, parent_id - ) + "updating file w/ content id:{} par:{}".format(file_id, parent_id) ) fil = self.create_file( {"id": file_id, "parents": [{"id": parent_id}]}, @@ -1473,9 +1425,7 @@ def upload_or_update_file( file_name = os.path.basename(file_path) self.debug( - "creating file w/ content {}->{}".format( - parent_id, file_path - ) + "creating file w/ content {}->{}".format(parent_id, file_path) ) fil = self.create_file( {"title": file_name, "parents": [{"id": parent_id}]}, @@ -1503,9 +1453,7 @@ def create_folder(self, input_args, upload=True): self.create_folder, 2, input_args, upload, gfileMeta=file ): if upload and not self.dry_run: - file.Upload( - param={"supportsTeamDrives": True} - ) # Upload the file. + file.Upload(param={"supportsTeamDrives": True}) # Upload the file. self.sleep() self.debug(f"uploaded {input_args}") file.FetchMetadata(fields="permissions,labels,mimeType") @@ -1561,9 +1509,7 @@ def get_or_create_folder(self, folder_name, parent_id=None, **kwargs): protect_name = not folder_name in self.protected_filenames - self.debug( - "found folder in parent {}: {}".format(parent_id, parent_folders) - ) + self.debug("found folder in parent {}: {}".format(parent_id, parent_folders)) if folder_name not in parent_folders: # Create It self.debug(f"creating {parent_id}->{folder_name}") @@ -1577,9 +1523,7 @@ def get_or_create_folder(self, folder_name, parent_id=None, **kwargs): return self.cache_item(fol) else: # Grab It - self.debug( - "found folder {} in parent {}".format(folder_name, parent_id) - ) + self.debug("found folder {} in parent {}".format(folder_name, parent_id)) return parent_items[folder_name] # Duplicate Handiling @@ -1625,9 +1569,7 @@ def group_duplicate_pairs(self, duplicate_paths_identified): duplicates_fil_sets = {} # path: [it1,it2...] if duplicate_paths_identified: - self.info( - f"duplicate paths identified {len(duplicate_paths_identified)}" - ) + self.info(f"duplicate paths identified {len(duplicate_paths_identified)}") files, folders = {}, {} for mtch_id, mtch in duplicate_paths_identified.items(): @@ -1654,9 +1596,7 @@ def group_duplicate_pairs(self, duplicate_paths_identified): for mtch_id, mtch in files.items() if not any( [ - self.path_contains( - fol.absolute_path, mtch.absolute_path - ) + self.path_contains(fol.absolute_path, mtch.absolute_path) for fid, fol in folders.items() ] ) @@ -1693,16 +1633,12 @@ def group_duplicate_pairs(self, duplicate_paths_identified): else: oked_paths.add(fil.absolute_path) - self.info( - f"duplicate files paths to fix: {len(duplicate_file_paths)}" - ) + self.info(f"duplicate files paths to fix: {len(duplicate_file_paths)}") check_fol_fil_intersection = { path: mtch for path, mtch in duplicates_fil_sets.items() - if any( - [self.path_contains(folpath, path) for folpath in dup_paths] - ) + if any([self.path_contains(folpath, path) for folpath in dup_paths]) } assert len(check_fol_fil_intersection) == 0 @@ -1723,9 +1659,7 @@ def remove_duplicates(self): # Proceed with work self.process_duplicates(duplicates_fols, duplicates_fil_sets) - def process_duplicates( - self, duplicate_folder_groups, duplicate_file_groups - ): + def process_duplicates(self, duplicate_folder_groups, duplicate_file_groups): """evals a set of duplicates and adjusts the fileystem as nessicary, the files and folders input should have no interaction""" self.delete_duplicate_items(duplicate_file_groups) @@ -1791,9 +1725,7 @@ def score_folders_suggest_action(items): ] # Sort by highest score - order_items = sorted( - group, key=lambda itd: itd["score"], reverse=True - ) + order_items = sorted(group, key=lambda itd: itd["score"], reverse=True) keep = order_items[0] # keep oldest folder remove = order_items[1:] out = {"keep": keep, "remove": remove} @@ -1875,10 +1807,7 @@ def ensure_g_path_get_id(self, gpath): else: self.debug(f"path doesnt exist, create it {current_pos}") if parent_id is not None and not any( - [ - fol.startswith(".") - for fol in current_pos.split(os.sep) - ] + [fol.startswith(".") for fol in current_pos.split(os.sep)] ): fol = self.get_or_create_folder(sub, parent_id) if fol is not None: @@ -1890,9 +1819,7 @@ def ensure_g_path_get_id(self, gpath): if parent_id is not None: return parent_id - elif not any( - [fol.startswith(".") for fol in current_pos.split(os.sep)] - ): + elif not any([fol.startswith(".") for fol in current_pos.split(os.sep)]): self.warning(f"Failed To get Gpath {gpath} Trying again ") return self.ensure_g_path_get_id(gpath) @@ -1906,14 +1833,10 @@ def get_gpath_id(self, gpath): matches = list(fids[paths == gpath]) if len(matches) > 1: - self.warning( - f"gpathid found matches {matches} for {gpath}, using first" - ) + self.warning(f"gpathid found matches {matches} for {gpath}, using first") # TODO: Handle this case! nodes = [self.item_nodes[mtch] for mtch in matches] - snodes = sorted( - nodes, key=lambda it: days_since_2020(it), reverse=True - ) + snodes = sorted(nodes, key=lambda it: days_since_2020(it), reverse=True) if snodes[0].is_folder: return snodes[0].id # This returns the oldest folder @@ -1987,11 +1910,7 @@ def file_nodes(self): @property def file_cache(self): with self.filesystem as fs: - return { - node.id: node.absolute_path - for node in fs.nodes() - if node.is_file - } + return {node.id: node.absolute_path for node in fs.nodes() if node.is_file} @property def file_paths(self): @@ -2007,9 +1926,7 @@ def folder_nodes(self): def folder_cache(self): with self.filesystem as fs: return { - node.id: node.absolute_path - for node in fs.nodes() - if node.is_folder + node.id: node.absolute_path for node in fs.nodes() if node.is_folder } @property @@ -2042,10 +1959,7 @@ def removeNode(self, node): self.protected_ids.remove(node.id) def cache_item(self, item_meta): - if ( - "teamDriveId" in item_meta - and item_meta["teamDriveId"] == self.sync_root_id - ): + if "teamDriveId" in item_meta and item_meta["teamDriveId"] == self.sync_root_id: if item_meta["id"] not in self.item_nodes: node = FileNode(self, item_meta) self.addFileNode(node) @@ -2092,9 +2006,7 @@ def search_items(self, q="", parent_id=None, **kwargs): existing_contents = {} success = False - with self.rate_limit_manager( - self.search_items, 2, q, parent_id, **kwargs - ): + with self.rate_limit_manager(self.search_items, 2, q, parent_id, **kwargs): self.sleep() for output in self.gdrive.ListFile(input_args): for file in output: @@ -2143,9 +2055,7 @@ def all_in_folder(self, folder_id=None): self.debug(f"searching {folder_id} for anything") output = list( - self.search_items( - f"'{folder_id}' in parents and trashed=false", folder_id - ) + self.search_items(f"'{folder_id}' in parents and trashed=false", folder_id) ) for file in output: yield file @@ -2168,9 +2078,7 @@ def hit_rate_limit(self, sleep_time=5, multiplier=2): self.max_sleep_time, ) else: - self._sleep_time = min( - self._sleep_time * multiplier, self.max_sleep_time - ) + self._sleep_time = min(self._sleep_time * multiplier, self.max_sleep_time) dynamicsleep = sleep_time * 0.5 + random.random() * sleep_time self.warning( @@ -2257,10 +2165,7 @@ def context(self, filepath_root, sync_root): old_syncroot = self.sync_root try: - if ( - self.filepath_root == filepath_root - and self.sync_root == sync_root - ): + if self.filepath_root == filepath_root and self.sync_root == sync_root: self.debug(f"Alread in context {filepath_root} -> {sync_root}") yield self # no work to do @@ -2272,9 +2177,7 @@ def context(self, filepath_root, sync_root): yield self except Exception as e: - self.error( - e, f"Issue In Drive Context {filepath_root} -> {sync_root}" - ) + self.error(e, f"Issue In Drive Context {filepath_root} -> {sync_root}") finally: self.filepath_root = old_filepath @@ -2301,9 +2204,7 @@ def filepath_root(self, filepath_root): self.filepath_inferred = False self.debug(f"using verified filepath {self._filepath_root}") else: - raise SourceFolderNotFound( - f"could not find filepath {filepath_root}" - ) + raise SourceFolderNotFound(f"could not find filepath {filepath_root}") elif not self.explict_input_only and self.guess_sync_path: # Infer It if in_client_dir(): @@ -2314,17 +2215,14 @@ def filepath_root(self, filepath_root): self.filepath_inferred = True self.debug(f"using infered client path: {self._filepath_root}") elif ( - self._shared_drive == self.default_shared_drive - and in_dropbox_dir() + self._shared_drive == self.default_shared_drive and in_dropbox_dir() ): # OTTERSYNC if in_wsl(): self._filepath_root = client_path(skip_wsl=False) else: self._filepath_root = client_path(skip_wsl=False) self.filepath_inferred = True - self.debug( - f"using infered engforge path: {self._filepath_root}" - ) + self.debug(f"using infered engforge path: {self._filepath_root}") if do_reinitalize: self.reset_target_id() @@ -2351,9 +2249,7 @@ def shared_drive(self, shared_drive): # 1) Look for credentials in service account creds if "shared_drive" in creds_info: self._shared_drive = creds_info["shared_drive"] - self.debug( - f"using service account shared drive: {self._shared_drive}" - ) + self.debug(f"using service account shared drive: {self._shared_drive}") # 2) Look for enviornmental variable elif "CLIENT_GDRIVE_PATH" in os.environ and os.environ[ @@ -2376,9 +2272,7 @@ def shared_drive(self, shared_drive): drive_canidate = drive_canidate.replace("shared:", "") if drive_canidate in self.shared_drives: self._shared_drive = f"shared:{drive_canidate}" - self.debug( - f"using env-var shared drive: {self._shared_drive}" - ) + self.debug(f"using env-var shared drive: {self._shared_drive}") else: raise SharedDriveNotFound( f"env var shared drive not found {gpath}->{drive_canidate}" @@ -2444,22 +2338,15 @@ def sync_root(self, sync_root): creds_info = self.credentials_info # 1) Look for credentials in service account creds - if ( - "shared_sync_path" in creds_info - and creds_info["shared_sync_path"] - ): + if "shared_sync_path" in creds_info and creds_info["shared_sync_path"]: self._sync_root = creds_info["shared_sync_path"] - self.debug( - f"using service account sync path: {self._sync_root}" - ) + self.debug(f"using service account sync path: {self._sync_root}") # 2) Look for enviornmental variable elif ( "CLIENT_GDRIVE_PATH" in os.environ and os.environ["CLIENT_GDRIVE_PATH"] - and os.environ["CLIENT_GDRIVE_PATH"].startswith( - self.shared_drive - ) + and os.environ["CLIENT_GDRIVE_PATH"].startswith(self.shared_drive) and os.environ["CLIENT_GDRIVE_PATH"] != self.shared_drive ): gpath = os.environ["CLIENT_GDRIVE_PATH"] @@ -2476,9 +2363,7 @@ def sync_root(self, sync_root): else: drive_canidate = "" # root else: - drive_canidate = ( - gpath # one could only assume this is correct - ) + drive_canidate = gpath # one could only assume this is correct self._sync_root = drive_canidate self.debug(f"using env-var sync path: {self._sync_root}") @@ -2495,12 +2380,8 @@ def sync_root(self, sync_root): ): # in a client directory, so map to ClientFolders or Relative Dir if self.shared_drive == self.default_shared_drive: # Map to ClientFolders/Client/matching/path... - relpath = os.path.relpath( - guessfilepath, engforge_projects() - ) - self._sync_root = os.path.join( - self.default_sync_path, relpath - ) + relpath = os.path.relpath(guessfilepath, engforge_projects()) + self._sync_root = os.path.join(self.default_sync_path, relpath) else: # Map to /matching/path... since client drive assumed root self._sync_root = os.path.relpath( @@ -2510,9 +2391,7 @@ def sync_root(self, sync_root): f"client infered sync path from given filepath : {self._sync_root}" ) - elif in_dropbox_dir( - guessfilepath - ): # we're not in a client folder + elif in_dropbox_dir(guessfilepath): # we're not in a client folder if self.shared_drive == self.default_shared_drive: # Map to Dropbox/whatever/path self._sync_root = os.path.relpath( @@ -2652,9 +2531,7 @@ def main_cli(): parser = argparse.ArgumentParser("Otter Drive Sync From Folder") - parser.add_argument( - "--shared-drive", "-D", default=None, help="shared drive name" - ) + parser.add_argument("--shared-drive", "-D", default=None, help="shared drive name") parser.add_argument( "--syncpath", "-S", @@ -2680,9 +2557,7 @@ def main_cli(): action="store_true", help="dry run, dont do any work! (WIP)", ) - parser.add_argument( - "--sync", action="store_true", help="sync the directories!" - ) + parser.add_argument("--sync", action="store_true", help="sync the directories!") parser.add_argument( "--remove-duplicates", action="store_true", help="remove any duplicates" ) diff --git a/engforge/datastores/reporting.py b/engforge/datastores/reporting.py index 1aa4108..23da8bb 100644 --- a/engforge/datastores/reporting.py +++ b/engforge/datastores/reporting.py @@ -213,9 +213,7 @@ def __init__(self, result_id_in, **kwargs): elif isinstance(val, (float, int)): # dynamic mapping if not numpy.isnan(val): - atobj = self.attr_class( - result_id_in, key=key, value=val - ) + atobj = self.attr_class(result_id_in, key=key, value=val) self.attr_store[key] = atobj # self[key] = atobj #dont use orm for upload! @@ -246,10 +244,7 @@ def all_fields(cls): if cls._mapped_component is None: return [] all_table_fields = set( - [ - attr.lower() - for attr in cls._mapped_component.cls_all_property_labels() - ] + [attr.lower() for attr in cls._mapped_component.cls_all_property_labels()] ) all_attr_fields = set( [ @@ -342,9 +337,7 @@ def __on_init__(self): self.db.ensure_database_exists(create_meta=False) self.base.metadata.bind = self.db.engine - self.base.metadata.reflect( - self.db.engine, autoload=True, keep_existing=True - ) + self.base.metadata.reflect(self.db.engine, autoload=True, keep_existing=True) self._component_cls_table_mapping = {} self.initalize() @@ -397,9 +390,7 @@ def filter_by_validators(self, attr_obj): [gtype in attr_obj.validator.type for gtype in (str, float, int)] ): return True - if any( - [gtype is attr_obj.validator.type for gtype in (str, float, int)] - ): + if any([gtype is attr_obj.validator.type for gtype in (str, float, int)]): return True return False @@ -408,10 +399,7 @@ def validator_to_column(self, attr_obj): [gtype in attr_obj.validator.type for gtype in (float, int)] ): return Column(Numeric, default=attr_obj.default, nullable=True) - if ( - type(attr_obj.validator.type) is tuple - and str in attr_obj.validator.type - ): + if type(attr_obj.validator.type) is tuple and str in attr_obj.validator.type: return Column( String(DEFAULT_STRING_LENGTH), default=attr_obj.default, @@ -443,15 +431,9 @@ def dict_attr( "__abstract__": False, } - if ( - results_table is not None and not is_analysis - ): # analysis won't be mapped - default_attr["__tablename__"] = ( - f"{results_table}_{table_abrv}{name}" - ) - root_obj, _ = self.mapped_tables[ - results_table - ] # analysis won't be mapped + if results_table is not None and not is_analysis: # analysis won't be mapped + default_attr["__tablename__"] = f"{results_table}_{table_abrv}{name}" + root_obj, _ = self.mapped_tables[results_table] # analysis won't be mapped backref_name = f"db_comp_attr_{name}" default_attr["result_id"] = Column( Integer, ForeignKey(f"{results_table}.id"), primary_key=True @@ -492,9 +474,7 @@ def default_attr(self, comp_cls, results_table=None): tbl_name = f"{results_table}_{table_abrv}{name}" backref_name = f"db_comp_{name}" root_obj, _ = self.mapped_tables[results_table] - default_attr["__tablename__"] = ( - f"{results_table}_{table_abrv}{name}" - ) + default_attr["__tablename__"] = f"{results_table}_{table_abrv}{name}" default_attr["result_id"] = Column( Integer, ForeignKey(f"{results_table}.id"), primary_key=True ) @@ -503,9 +483,7 @@ def default_attr(self, comp_cls, results_table=None): ) else: # its an analysis - default_attr["created"] = Column( - DateTime, server_default=func.now() - ) + default_attr["created"] = Column(DateTime, server_default=func.now()) default_attr["active"] = Column(Boolean(), server_default="t") default_attr["run_id"] = Column( String(36) @@ -517,9 +495,7 @@ def default_attr(self, comp_cls, results_table=None): # These use keys of system_property or attr and lower() so no spaces or capitals def all_possible_component_fields(self, cls): - all_table_fields = set( - [attr.lower() for attr in cls.cls_all_property_keys()] - ) + all_table_fields = set([attr.lower() for attr in cls.cls_all_property_keys()]) all_attr_fields = set( [attr.lower() for attr in cls.cls_all_attrs_fields().keys()] ) @@ -543,14 +519,10 @@ def component_attr(self, comp_cls, results_table=None): } ) - component_attr.update( - self.default_attr(comp_cls, results_table=results_table) - ) + component_attr.update(self.default_attr(comp_cls, results_table=results_table)) return component_attr - def db_component_dict( - self, comp_cls, results_table=None, is_analysis=False - ): + def db_component_dict(self, comp_cls, results_table=None, is_analysis=False): """returns the nessicary table types to make for reflection of input component class this method represents the dynamic creation of SQLA typing and attributes via __dict__ @@ -656,9 +628,7 @@ def check_or_create_db_type(self, tablename, type_tuple): return cls_db def map_component(self, dbcomponent, component): - self.debug( - f"mapping {dbcomponent},{component} -> {dbcomponent.__tablename__}" - ) + self.debug(f"mapping {dbcomponent},{component} -> {dbcomponent.__tablename__}") # add to internal mapping self._component_cls_table_mapping[dbcomponent.__tablename__] = ( dbcomponent, @@ -675,22 +645,16 @@ def map_component(self, dbcomponent, component): if isinstance(component, Analysis): component = component.__class__ if dbcomponent.__tablename__ not in atables: - self.info( - f"adding analysis record { dbcomponent.__tablename__}" - ) + self.info(f"adding analysis record { dbcomponent.__tablename__}") with self.db.session_scope() as sesh: rec = AnalysisRegistry(dbcomponent, component) sesh.add(rec) - elif isinstance(component, Component) or issubclass( - component, Component - ): + elif isinstance(component, Component) or issubclass(component, Component): if isinstance(component, Component): component = component.__class__ if dbcomponent.__tablename__ not in ctables: - self.info( - f"adding comonent record { dbcomponent.__tablename__}" - ) + self.info(f"adding comonent record { dbcomponent.__tablename__}") with self.db.session_scope() as sesh: rec = ComponentRegistry(dbcomponent, component) sesh.add(rec) @@ -769,9 +733,7 @@ def get_analysis_class(self, analysis): elif issubclass(analysis, Analysis): analysis_cls = analysis else: - self.warning( - "ensure-analysis: analysis not mapped, using direct input" - ) + self.warning("ensure-analysis: analysis not mapped, using direct input") analysis_cls = analysis return analysis_cls @@ -820,9 +782,7 @@ def gen(table): if lvl > 0: for comp in comps: cmp = comp["conf"] - data_gens[self.mapped_classes[cmp.__class__]] = gen( - cmp.TABLE - ) + data_gens[self.mapped_classes[cmp.__class__]] = gen(cmp.TABLE) # with self.db.scoped_session() as sesh: any_succeeded = True @@ -872,9 +832,7 @@ def gen(table): others.append(cmp_result) else: - any_succeeded = ( - True # here's ur fricken evidence ur honor - ) + any_succeeded = True # here's ur fricken evidence ur honor # the scoped session rolls back anything created this time :) if not any_succeeded: @@ -884,14 +842,14 @@ def gen(table): add_list = ( main_attrs + others - + flatten( - [list(item.attr_store.values()) for item in others] - ) + + flatten([list(item.attr_store.values()) for item in others]) ) sesh.add_all(add_list) - inx += 1 # index += 1 is done at end of analysis so we should model that + inx += ( + 1 # index += 1 is done at end of analysis so we should model that + ) except AvoidDuplicateAbortUpload: pass # this is fine @@ -943,10 +901,7 @@ def db_columns(cls): @classmethod def all_fields(cls): all_table_fields = set( - [ - attr.lower() - for attr in cls._mapped_component.cls_all_property_labels() - ] + [attr.lower() for attr in cls._mapped_component.cls_all_property_labels()] ) all_attr_fields = set( [ diff --git a/engforge/datastores/secrets.py b/engforge/datastores/secrets.py index 2a11c23..73d52fb 100644 --- a/engforge/datastores/secrets.py +++ b/engforge/datastores/secrets.py @@ -61,9 +61,7 @@ class Secrets(Configuration, metaclass=InputSingletonMeta): credential_key = attr.ib() # What to refer to this credential as credential_location = attr.ib() # Where to find the credential file - _aws_kms: pysecret.AWSSecret = ( - None # a storage for the aws key managment object - ) + _aws_kms: pysecret.AWSSecret = None # a storage for the aws key managment object public_up_level = True # makes the public cred record one level up (creds are stored protected). Yes its hacky but we have deadlines @@ -319,9 +317,7 @@ def local_credential_files(self): paths = [ os.path.join(root, acc) - for acc in itertools.accumulate( - rel_paths, func=os.path.join, initial=root - ) + for acc in itertools.accumulate(rel_paths, func=os.path.join, initial=root) ] creds_paths = {} @@ -389,9 +385,7 @@ def local_credential_packages(self): def decredential_file(filepath): for cred in self.creds_folders: if filepath.endswith(cred): - return str( - pathlib.Path(filepath).parent - ) # Can't be dtwo types + return str(pathlib.Path(filepath).parent) # Can't be dtwo types raw_dict = { decredential_file(key): creds @@ -403,16 +397,10 @@ def decredential_file(filepath): # assert all(list(map( ))) paths_contains_creds = lambda path: [ - cred - for key, creds in raw_dict.items() - for cred in creds - if key in path + cred for key, creds in raw_dict.items() for cred in creds if key in path ] - output = { - key: paths_contains_creds(key) - for key, creds in raw_dict.items() - } + output = {key: paths_contains_creds(key) for key, creds in raw_dict.items()} stage_packages[stage] = output return stage_packages @@ -501,10 +489,7 @@ def identity(self): else: self._ident = f"secrets-{self.stage_name}" - if ( - self.stored_client_name - and not self.stored_client_name in self._ident - ): + if self.stored_client_name and not self.stored_client_name in self._ident: self._log = None # lazy cycle log name return self._ident @@ -519,9 +504,7 @@ def sync(self): if "CLIENT_GDRIVE_SYNC" in os.environ: self.info("got CLIENT_GDRIVE_SYNC") - CLIENT_GDRIVE_SYNC = self.bool_from( - os.environ["CLIENT_GDRIVE_SYNC"] - ) + CLIENT_GDRIVE_SYNC = self.bool_from(os.environ["CLIENT_GDRIVE_SYNC"]) if "CLIENT_GMAIL" in os.environ: self.info("got CLIENT_GMAIL") @@ -556,9 +539,7 @@ def sync(self): and "SLACK_WEBHOOK_NOTIFICATION" in os.environ ): self.info("getting slack webhook") - self.SLACK_WEBHOOK_NOTIFICATION = os.environ[ - "SLACK_WEBHOOK_NOTIFICATION" - ] + self.SLACK_WEBHOOK_NOTIFICATION = os.environ["SLACK_WEBHOOK_NOTIFICATION"] # TODO: Add CLI Method diff --git a/engforge/dynamics.py b/engforge/dynamics.py index c7f73b3..2b94abd 100644 --- a/engforge/dynamics.py +++ b/engforge/dynamics.py @@ -50,9 +50,7 @@ class INDEX_MAP: oppo = {str: int, int: str} def __init__(self, datas: list): - self.data = [ - data if not data.startswith(".") else data[1:] for data in datas - ] + self.data = [data if not data.startswith(".") else data[1:] for data in datas] self.index = {} def get(self, key): @@ -74,25 +72,17 @@ def __call__(self, key): @staticmethod def indify(arr, *args): - return [ - arr[arg] if isinstance(arg, int) else arr.index(arg) for arg in args - ] + return [arr[arg] if isinstance(arg, int) else arr.index(arg) for arg in args] def remap_indexes_to(self, new_index, *args, invert=False, old_data=None): if old_data is None: old_data = self.data opt1 = {arg: self.indify(old_data, arg)[0] for arg in args} opt2 = { - arg: ( - self.indify(old_data, val)[0] - if (not isinstance(val, str)) - else val - ) + arg: (self.indify(old_data, val)[0] if (not isinstance(val, str)) else val) for arg, val in opt1.items() } - oop1 = { - arg: self.indify(new_index, val)[0] for arg, val in opt2.items() - } + oop1 = {arg: self.indify(new_index, val)[0] for arg, val in opt2.items()} oop2 = { arg: ( self.indify(new_index, val)[0] @@ -188,15 +178,11 @@ def dynamic_output_size(self): @property def dynamic_state(self) -> np.array: - return np.array( - [getattr(self, var, np.nan) for var in self.dynamic_state_vars] - ) + return np.array([getattr(self, var, np.nan) for var in self.dynamic_state_vars]) @property def dynamic_input(self) -> np.array: - return np.array( - [getattr(self, var, np.nan) for var in self.dynamic_input_vars] - ) + return np.array([getattr(self, var, np.nan) for var in self.dynamic_input_vars]) @property def dynamic_output(self) -> np.array: @@ -210,15 +196,11 @@ def create_state_matrix(self, **kwargs) -> np.ndarray: def create_input_matrix(self, **kwargs) -> np.ndarray: """creates the input matrix for the system, called B""" - return np.zeros( - (self.dynamic_state_size, max(self.dynamic_input_size, 1)) - ) + return np.zeros((self.dynamic_state_size, max(self.dynamic_input_size, 1))) def create_output_matrix(self, **kwargs) -> np.ndarray: """creates the input matrix for the system, called C""" - return np.zeros( - (max(self.dynamic_output_size, 1), self.dynamic_state_size) - ) + return np.zeros((max(self.dynamic_output_size, 1), self.dynamic_state_size)) def create_feedthrough_matrix(self, **kwargs) -> np.ndarray: """creates the input matrix for the system, called D""" @@ -531,9 +513,7 @@ def ref_dXdt(self, name: str): accss.__name__ = f"ref_dXdt_{name}" return Ref(self, accss) - def determine_nearest_stationary_state( - self, t=0, X=None, U=None - ) -> np.ndarray: + def determine_nearest_stationary_state(self, t=0, X=None, U=None) -> np.ndarray: """determine the nearest stationary state""" if X is None: diff --git a/engforge/eng/costs.py b/engforge/eng/costs.py index 442ec10..c913d1c 100644 --- a/engforge/eng/costs.py +++ b/engforge/eng/costs.py @@ -70,9 +70,7 @@ class CostLog(LoggingMixin): "maintenance": lambda inst, term, econ: True if term >= 1 else False, "always": lambda inst, term, econ: True, "end": lambda inst, term, econ: ( - True - if hasattr(econ, "term_length") and term == econ.term_length - 1 - else False + True if hasattr(econ, "term_length") and term == econ.term_length - 1 else False ), } @@ -227,9 +225,7 @@ def update_dflt_costs(self): for k, v in self._slot_costs.items(): # Check if the cost model will be accessed no_comp = k not in current_comps - is_cost = not no_comp and isinstance( - current_comps[k], CostModel - ) + is_cost = not no_comp and isinstance(current_comps[k], CostModel) dflt_is_cost_comp = all( [isinstance(v, CostModel), isinstance(v, Component)] ) @@ -258,9 +254,7 @@ def set_default_costs(self): @classmethod def subcls_compile(cls): - assert not issubclass( - cls, ComponentIter - ), "component iter not supported" + assert not issubclass(cls, ComponentIter), "component iter not supported" log.debug(f"compiling costs {cls}") cls.reset_cls_costs() @@ -279,9 +273,7 @@ def default_cost( assert not isinstance( cost, type ), f"insantiate classes before adding as a cost!" - assert ( - slot_name in cls.slots_attributes() - ), f"slot {slot_name} doesnt exist" + assert slot_name in cls.slots_attributes(), f"slot {slot_name} doesnt exist" assert isinstance(cost, (float, int, dict)) or isinstance( cost, CostModel ), "only numeric types or CostModel instances supported" @@ -291,9 +283,7 @@ def default_cost( if warn_on_non_costmodel and not any( [issubclass(at, CostModel) for at in atypes] ): - log.warning( - f"assigning cost to non CostModel based slot {slot_name}" - ) + log.warning(f"assigning cost to non CostModel based slot {slot_name}") cls._slot_costs[slot_name] = cost @@ -309,9 +299,7 @@ def custom_cost( assert not isinstance( cost, type ), f"insantiate classes before adding as a cost!" - assert ( - slot_name in self.slots_attributes() - ), f"slot {slot_name} doesnt exist" + assert slot_name in self.slots_attributes(), f"slot {slot_name} doesnt exist" assert isinstance(cost, (float, int, dict)) or isinstance( cost, CostModel ), "only numeric types or CostModel instances supported" @@ -321,9 +309,7 @@ def custom_cost( if warn_on_non_costmodel and not any( [issubclass(at, CostModel) for at in atypes] ): - self.warning( - f"assigning cost to non CostModel based slot {slot_name}" - ) + self.warning(f"assigning cost to non CostModel based slot {slot_name}") # convert from classinfo if self._slot_costs is self.__class__._slot_costs: @@ -370,18 +356,14 @@ def future_costs(self) -> float: initial_costs = self.costs_at_term(0, False) return numpy.nansum(list(initial_costs.values())) - def sum_costs( - self, saved: set = None, categories: tuple = None, term=0, econ=None - ): + def sum_costs(self, saved: set = None, categories: tuple = None, term=0, econ=None): """sums costs of cost_property's in this item that are present at term=0, and by category if define as input""" if saved is None: saved = set((self,)) # item cost included! elif self not in saved: saved.add(self) itemcst = list( - self.dict_itemized_costs( - saved, categories, term, econ=econ - ).values() + self.dict_itemized_costs(saved, categories, term, econ=econ).values() ) csts = [self.sub_costs(saved, categories, term), numpy.nansum(itemcst)] return numpy.nansum(csts) @@ -407,9 +389,7 @@ def dict_itemized_costs( } return costs - def sub_costs( - self, saved: set = None, categories: tuple = None, term=0, econ=None - ): + def sub_costs(self, saved: set = None, categories: tuple = None, term=0, econ=None): """gets items from CostModel's defined in a Slot attribute or in a slot default, tolerrant to nan's in cost definitions""" if saved is None: saved = set() @@ -701,9 +681,7 @@ def term_fgen(self, comp, prop): ) def sum_term_fgen(self, ref_group): - term_funs = [ - self.term_fgen(ref.comp, self.get_prop(ref)) for ref in ref_group - ] + term_funs = [self.term_fgen(ref.comp, self.get_prop(ref)) for ref in ref_group] return lambda term: numpy.nansum([t(term) for t in term_funs]) # Gather & Set References (the magic!) @@ -800,9 +778,7 @@ def abriv(val): ckey = f"{base.replace('lifecycle.','')}.cost.{cst}" # print(ckey,str(self._comp_costs.keys())) comp_costs[base][cst] = val - comp_nums[base][cst] = get_num_from_cost_prop( - self._comp_costs[ckey] - ) + comp_nums[base][cst] = get_num_from_cost_prop(self._comp_costs[ckey]) elif col.startswith("summary."): summary[col.replace("summary.", "")] = val else: @@ -833,16 +809,12 @@ def abriv(val): itcst = "{val:>24}".format(val="TOTAL----->") self.info("=" * 80) self.info( - fmt.format( - key="COMBINED", fmt=itcst, total=abriv(total_cost), pct=100 - ) + fmt.format(key="COMBINED", fmt=itcst, total=abriv(total_cost), pct=100) ) self.info("-" * 80) # itemization sgroups = lambda kv: sum(list(kv[-1].values())) - for base, items in sorted( - comp_costs.items(), key=sgroups, reverse=True - ): + for base, items in sorted(comp_costs.items(), key=sgroups, reverse=True): # skip if all zeros (allow for net negative costs) if (subtot := sum([abs(v) for v in items.values()])) > 0: # self.info(f' {base:<35}| total ---> {abriv(subtot)} | {subtot*100/total_cost:3.0f}%') @@ -869,14 +841,10 @@ def abriv(val): pct = val * 100 / total_cost num = comp_nums[base][key] itcst = ( - f"{abriv(val/num):^18} x {num:3.0f}" - if num != 0 - else "0" + f"{abriv(val/num):^18} x {num:3.0f}" if num != 0 else "0" ) self.info( - fmt.format( - key="-" + key, fmt=itcst, total=tot, pct=pct - ) + fmt.format(key="-" + key, fmt=itcst, total=tot, pct=pct) ) self.info("-" * 80) # section break self.info("#" * 80) @@ -939,9 +907,7 @@ def lifecycle_dataframe(self) -> pandas.DataFrame: ) row["levelized_cost"] = tc * (1 + self.discount_rate) ** (-1 * t) row["output"] = output = self.calculate_production(self.parent, t) - row["levelized_output"] = output * (1 + self.discount_rate) ** ( - -1 * t - ) + row["levelized_output"] = output * (1 + self.discount_rate) ** (-1 * t) return pandas.DataFrame(out) @@ -978,9 +944,7 @@ def _gather_cost_references(self, parent: "System"): self._comp_categories = collections.defaultdict(list) self._comp_costs = dict() - for key, level, conf in parent.go_through_configurations( - check_config=False - ): + for key, level, conf in parent.go_through_configurations(check_config=False): # skip self if conf is self: continue @@ -1021,9 +985,7 @@ def _gather_cost_references(self, parent: "System"): compcanidate = child._slot_costs[comp_key] if isinstance(compcanidate, CostModel): self.debug(f"dflt child costmodel {kbase}.{comp_key}") - self._extract_cost_references( - compcanidate, bse + "cost." - ) + self._extract_cost_references(compcanidate, bse + "cost.") else: _key = bse + "cost.item_cost" self.debug(f"dflt child cost for {kbase}.{comp_key}") @@ -1037,9 +999,7 @@ def _gather_cost_references(self, parent: "System"): cc = "unit" self._comp_costs[_key] = ref self._cost_categories["category." + cc].append(ref) - self._comp_categories[bse + "category." + cc].append( - ref - ) + self._comp_categories[bse + "category." + cc].append(ref) # 2. try looking at the parent elif ( @@ -1133,34 +1093,24 @@ def _extract_cost_references(self, conf: "CostModel", bse: str): self.debug(f"skipping key {_key}") # add base class slot values when comp was none (recursively) - for compnm, comp in conf.internal_configurations( - False, none_ok=True - ).items(): + for compnm, comp in conf.internal_configurations(False, none_ok=True).items(): if comp is None: if self.log_level < 5: self.msg( f"{conf} looking up base class costs for {compnm}", lvl=5, ) - comp_cls = conf.slots_attributes(attr_type=True)[ - compnm - ].accepted + comp_cls = conf.slots_attributes(attr_type=True)[compnm].accepted for cc in comp_cls: if issubclass(cc, CostModel): if cc._slot_costs: if self.log_level < 5: - self.msg( - f"{conf} looking up base slot cost for {cc}" - ) + self.msg(f"{conf} looking up base slot cost for {cc}") for k, v in cc._slot_costs.items(): - _key = ( - bse + compnm + "." + k + ".cost.item_cost" - ) + _key = bse + compnm + "." + k + ".cost.item_cost" if _key in CST: if self.log_level < 10: - self.debug( - f"{conf} skipping dflt key {_key}" - ) + self.debug(f"{conf} skipping dflt key {_key}") # break #skip if already added continue @@ -1183,9 +1133,7 @@ def _extract_cost_references(self, conf: "CostModel", bse: str): cc = "unit" self._comp_costs[_key] = ref - self._cost_categories[ - "category." + cc - ].append(ref) + self._cost_categories["category." + cc].append(ref) self._comp_categories[ bse + "category." + cc ].append(ref) @@ -1396,9 +1344,7 @@ def cost_categories_from_df(self, df): categories.add(val) return categories - def plot_cost_categories( - self, df, group, cmap="tab20c", make_title=None, ax=None - ): + def plot_cost_categories(self, df, group, cmap="tab20c", make_title=None, ax=None): categories = self.cost_categories_from_df(df) from matplotlib import cm @@ -1445,9 +1391,7 @@ def make_title(row): spec_costs = {k: v for k, v in cat_costs.items() if k in group} pos_costs = {k: v for k, v in spec_costs.items() if v >= 0} - neg_costs = { - k: v for k, v in spec_costs.items() if k not in pos_costs - } + neg_costs = {k: v for k, v in spec_costs.items() if k not in pos_costs} neg_amt = sum(list(neg_costs.values())) pos_amt = sum(list(pos_costs.values())) diff --git a/engforge/eng/fluid_material.py b/engforge/eng/fluid_material.py index 6bfb954..2adbd8f 100644 --- a/engforge/eng/fluid_material.py +++ b/engforge/eng/fluid_material.py @@ -254,9 +254,7 @@ def material(self) -> str: @classmethod def setup(cls): try: - CoolProp.apply_simple_mixing_rule( - cls.material, cls.material2, "linear" - ) + CoolProp.apply_simple_mixing_rule(cls.material, cls.material2, "linear") except Exception as e: pass # self.error(e,'issue setting mixing rule, but continuting.') diff --git a/engforge/eng/geometry.py b/engforge/eng/geometry.py index 45e268b..f375ede 100644 --- a/engforge/eng/geometry.py +++ b/engforge/eng/geometry.py @@ -371,9 +371,7 @@ class ShapelySection(Profile2D): # Mesh sizing coarse: bool = attrs.field(default=False) mesh_extent_decimation = attrs.field(default=100) - min_mesh_angle: float = attrs.field( - default=20 - ) # below 20.7 garunteed to work + min_mesh_angle: float = attrs.field(default=20) # below 20.7 garunteed to work min_mesh_size: float = attrs.field(default=1e-5) # multiply by min goal_elements: float = attrs.field(default=1000) # multiply by min _mesh_size: float = attrs.field(default=attrs.Factory(get_mesh_size, True)) @@ -499,9 +497,7 @@ def from_cache(cls, hash_id): def prediction_weights(self, df, window, initial_weight=10): weights = numpy.ones(min(len(df), window)) weights[0] = initial_weight**2 # zero value is important! - weights[: getattr(self, "N_base", 100)] = ( - initial_weight # then base values - ) + weights[: getattr(self, "N_base", 100)] = initial_weight # then base values if hasattr(self, "N_pareto"): weights[: getattr(self, "N_pareto")] = ( initial_weight**0.5 @@ -509,9 +505,7 @@ def prediction_weights(self, df, window, initial_weight=10): # Dont emphasise fit above max margin dm = (df.fail_frac - self.max_margin).to_numpy() penalize_inx = dm > 0 - weights[penalize_inx] = np.maximum( - 1.0 / ((1.0 + dm[penalize_inx])), 0.1 - ) + weights[penalize_inx] = np.maximum(1.0 / ((1.0 + dm[penalize_inx])), 0.1) return weights def _subsample_data(self, X, y, window, weights): @@ -619,9 +613,7 @@ def mesh_section(self): self._A = self._sec.get_area() if self.material: - self._Ixx, self._Iyy, self._Ixy = self._sec.get_eic( - e_ref=self.material - ) + self._Ixx, self._Iyy, self._Ixy = self._sec.get_eic(e_ref=self.material) self._J = self._sec.get_ej() else: self._Ixx, self._Iyy, self._Ixy = self._sec.get_ic() @@ -670,9 +662,7 @@ def plot_mesh(self): def plot_mesh(self): return self._sec.plot_centroids() - def calculate_stress( - self, n=0, vx=0, vy=0, mxx=0, myy=0, mzz=0, **kw - ) -> float: + def calculate_stress(self, n=0, vx=0, vy=0, mxx=0, myy=0, mzz=0, **kw) -> float: return calculate_stress( self, n=n, vx=vx, vy=vy, mxx=mxx, myy=myy, mzz=mzz, **kw ) @@ -704,9 +694,7 @@ def estimate_stress( do_calc = not self.prediction or not self._fitted or under_size if do_calc or force_calc: if self._do_print and self.prediction: - print( - f"calc till {len(self.prediction_records)} <= {min_est_records}" - ) + print(f"calc till {len(self.prediction_records)} <= {min_est_records}") stress = calculate_stress( self, n=n, vx=vx, vy=vy, mxx=mxx, myy=myy, mzz=mzz, value=value ) @@ -731,9 +719,7 @@ def estimate_stress( # calculate stress if close to failure within saftey margin err = 1 - val mrg = self.fail_frac_criteria(calc_margin=calc_margin) - do_calc = abs(err) <= mrg or all( - [calc_every, (Nrec % calc_every) == 0] - ) + do_calc = abs(err) <= mrg or all([calc_every, (Nrec % calc_every) == 0]) oob = val <= self.max_margin and self.check_out_of_domain(data, 0.1) if self._do_print: self.info( @@ -878,9 +864,7 @@ def solve_fail(self, fail_parm, base_kw, guess=None, tol=1e-4, mult=1): return 1e6 # Determine Outer Bound Of Failures - def determine_failure_front( - self, pareto_inx=[0.5, 0.1], pareto_front=False - ): + def determine_failure_front(self, pareto_inx=[0.5, 0.1], pareto_front=False): self.info( f"determining failure front for cross section, with pareto inx: {pareto_inx}" ) @@ -949,8 +933,7 @@ def basis_expand( q = max(wt, 1) if normalize else 1 inxs = [self._prediction_parms.index(p) for p in parms] base_kw = { - p: w * self._basis[i] / q - for p, w, i in zip(parms, weight, inxs) + p: w * self._basis[i] / q for p, w, i in zip(parms, weight, inxs) } # print(base_kw) @@ -983,9 +966,7 @@ def train_until_valid(self, print_interval=50, max_iter=1000, est=False): self.calculate_stress(**inp) if i % print_interval == 0: - self.info( - f"training... current error: {self._training_history[-1]}" - ) + self.info(f"training... current error: {self._training_history[-1]}") i += 1 if i >= max_iter: self.info(f"training... max iterations reached") @@ -1035,9 +1016,7 @@ def find_radii(targetth): poss2 = int((poss + 1) % Imax) x_ = np.array([r[poss], r[poss2]]) y_ = np.array([inx[poss], inx[poss2]]) - itarget = (0 - x_[0]) * (y_[1] - y_[0]) / (x_[1] - x_[0]) + y_[ - 0 - ] + itarget = (0 - x_[0]) * (y_[1] - y_[0]) / (x_[1] - x_[0]) + y_[0] r_ = np.interp(itarget, inx2, R) out.add(round(r_, precision)) # print(itarget,r_) diff --git a/engforge/eng/prediction.py b/engforge/eng/prediction.py index c848df5..27b6846 100644 --- a/engforge/eng/prediction.py +++ b/engforge/eng/prediction.py @@ -85,9 +85,7 @@ def add_prediction_record( and std != 0 and not near_zero ): - g = 3 * min( - (abs(avg) - std) / abs(std), 1 - ) # negative when std > avg + g = 3 * min((abs(avg) - std) / abs(std), 1) # negative when std > avg a = min(abs(dev) / std / J, 1) # prob accept should be based on difference from average prob_deny = np.e**g @@ -95,9 +93,7 @@ def add_prediction_record( std_choice = std avg_choice = avg dev_choice = dev - choice = random.choices( - [True, False], [prob_accept, prob_deny] - )[0] + choice = random.choices([True, False], [prob_accept, prob_deny])[0] cur_stat["avg"] = avg + dev / N cur_stat["var"] = var + (dev**2 - var) / N @@ -153,9 +149,7 @@ def check_out_of_domain(self, record, extra_margin=1, target_items=1000): std = var**0.5 std_err = std / len(self.prediction_records) # check out of bounds - g = 3 * min( - (abs(avg) - std) / abs(std), 1 - ) # negative when std > avg + g = 3 * min((abs(avg) - std) / abs(std), 1) # negative when std > avg a = min(abs(dev) / std / J, 1) near_zero = abs(rec_val) / max(abs(avg), 1) <= 1e-3 prob_deny = np.e**g @@ -308,9 +302,7 @@ def observe_and_predict(self, row): model = mod_dict["mod"] if parm not in row: continue - X = pandas.DataFrame( - [{parm: row[parm] for parm in self._prediction_parms}] - ) + X = pandas.DataFrame([{parm: row[parm] for parm in self._prediction_parms}]) y = row[parm] x = abs(model.predict(X)) if y == 0: @@ -364,10 +356,7 @@ def check_and_retrain(self, records, min_rec=None): model = mod_dict["mod"] N = mod_dict["N"] # - if ( - Nrec > N * self._re_train_frac - or (Nrec - N) > self._re_train_maxiter - ): + if Nrec > N * self._re_train_frac or (Nrec - N) > self._re_train_maxiter: self.train_compare(df) return diff --git a/engforge/eng/structure.py b/engforge/eng/structure.py index 3ad40ce..62dc6d2 100644 --- a/engforge/eng/structure.py +++ b/engforge/eng/structure.py @@ -61,9 +61,7 @@ class StructureLog(LoggingMixin): k: v for k, v in filter( lambda kv: ( - issubclass(kv[1], geometry.Geometry) - if type(kv[1]) is type - else False + issubclass(kv[1], geometry.Geometry) if type(kv[1]) is type else False ), sections.__dict__.items(), ) @@ -327,9 +325,7 @@ def create_structure(self): pass # Execution - def execute( - self, combos: list = None, save=True, record=True, *args, **kwargs - ): + def execute(self, combos: list = None, save=True, record=True, *args, **kwargs): """wrapper allowing saving of data by load combo""" # the string input case, with csv support @@ -352,9 +348,7 @@ def execute( # run the analysis # self.index += 1 - combo_possible = self.struct_pre_execute( - combo - ) # can change combo + combo_possible = self.struct_pre_execute(combo) # can change combo if combo_possible: self.current_combo = combo_possible else: @@ -426,9 +420,7 @@ def struct_root_failure( res = target - ff if sols is not None: - sols.append( - {"x": x, "ff": ff, "obj": res, "kw": bkw, "mf": mf, "bf": bf} - ) + sols.append({"x": x, "ff": ff, "obj": res, "kw": bkw, "mf": mf, "bf": bf}) self.info(f"ran: {bkw} -> {ff:5.4f} | {res:5.4f}") return res @@ -754,9 +746,7 @@ def add_member(self, name, node1, node2, section, material=None, **kwargs): elif hasattr(section, "material"): material = section.material else: - raise ValueError( - "material not defined as input or from default sources!" - ) + raise ValueError("material not defined as input or from default sources!") uid = material.unique_id if uid not in self._materials: @@ -766,9 +756,7 @@ def add_member(self, name, node1, node2, section, material=None, **kwargs): ) self._materials[uid] = material - beam_attrs = { - k: v for k, v in kwargs.items() if k in Beam.input_attrs() - } + beam_attrs = {k: v for k, v in kwargs.items() if k in Beam.input_attrs()} kwargs = {k: v for k, v in kwargs.items() if k not in beam_attrs} B = beam = Beam( @@ -792,9 +780,7 @@ def add_member(self, name, node1, node2, section, material=None, **kwargs): ) if self.add_gravity_force: - beam.apply_gravity_force( - z_dir=self.gravity_dir, z_mag=self.gravity_scalar - ) + beam.apply_gravity_force(z_dir=self.gravity_dir, z_mag=self.gravity_scalar) return beam @@ -832,9 +818,7 @@ def add_member_with( self.beams[name] = beam if self.add_gravity_force: - beam.apply_gravity_force( - z_dir=self.gravity_dir, z_mag=self.gravity_scalar - ) + beam.apply_gravity_force(z_dir=self.gravity_dir, z_mag=self.gravity_scalar) self.frame.add_member( name, @@ -1447,9 +1431,7 @@ def current_failures( summary["mesh_stable"] = mesh_success # update failures - mesh_failures_count = len( - [v for k, v in mesh_failures.items() if v > 0.99] - ) + mesh_failures_count = len([v for k, v in mesh_failures.items() if v > 0.99]) max_fail_frac = max( [0] @@ -1712,10 +1694,7 @@ def run_failure_sections( d_["stress_results"] = sss = s.get_stress() d_["stress_vm_max"] = max([max(ss["sig_vm"]) for ss in sss]) d_["fail_frac"] = ff = max( - [ - max(ss["sig_vm"] / beam.material.allowable_stress) - for ss in sss - ] + [max(ss["sig_vm"] / beam.material.allowable_stress) for ss in sss] ) d_["fails"] = fail = ff > 1 / SF @@ -1743,9 +1722,7 @@ def run_failure_sections( # Remote Sync Util (locally run with section) -def run_combo_failure_analysis( - inst, combo, run_full: bool = False, SF: float = 1.0 -): +def run_combo_failure_analysis(inst, combo, run_full: bool = False, SF: float = 1.0): """runs a single load combo and adds 2d section failures""" inst.resetSystemLogs() # use parallel failure section @@ -1858,9 +1835,7 @@ def run_failure_sections( # Remote Analysis -def parallel_run_failure_analysis( - struct, SF=1.0, run_full=False, purge=False, **kw -): +def parallel_run_failure_analysis(struct, SF=1.0, run_full=False, purge=False, **kw): """ Failure Determination: Beam Stress Estiates are used to determine if a 2D FEA beam/combo analysis should be run. The maximum beam vonmises stress is compared the the beam material allowable stress / saftey factor. @@ -1971,9 +1946,7 @@ def remote_section( d_["stress_analysis"] = s d_["stress_results"] = sss d_["stress_vm_max"] = max([max(ss["sig_vm"]) for ss in sss]) - d_["fail_frac"] = ff = max( - [max(ss["sig_vm"] / allowable_stress) for ss in sss] - ) + d_["fail_frac"] = ff = max([max(ss["sig_vm"] / allowable_stress) for ss in sss]) d_["fails"] = fail = ff > 1 / SF return d_ @@ -2203,9 +2176,7 @@ def remote_failure_sections( allowable = r["allowable"] cur.append( - remote_section.remote( - beam_ref, beamnm, forces, allowable, c, x, SF=SF - ) + remote_section.remote(beam_ref, beamnm, forces, allowable, c, x, SF=SF) ) # run the damn thing @@ -2221,9 +2192,7 @@ def remote_failure_sections( secton_results[res["beam"]][(combo, x)] = res if fail: - log.warning( - f"beam {beamnm} failed @ {x*100:3.0f}%| {c}" - ) + log.warning(f"beam {beamnm} failed @ {x*100:3.0f}%| {c}") if fail_fast: return secton_results if not run_full: diff --git a/engforge/eng/structure_beams.py b/engforge/eng/structure_beams.py index 2bfc088..a6a7056 100644 --- a/engforge/eng/structure_beams.py +++ b/engforge/eng/structure_beams.py @@ -47,15 +47,11 @@ def rotation_matrix_from_vectors(vec1, vec2): if any(v): # if not all zeros then c = numpy.dot(a, b) s = numpy.linalg.norm(v) - kmat = numpy.array( - [[0, -v[2], v[1]], [v[2], 0, -v[0]], [-v[1], v[0], 0]] - ) + kmat = numpy.array([[0, -v[2], v[1]], [v[2], 0, -v[0]], [-v[1], v[0], 0]]) return numpy.eye(3) + kmat + kmat.dot(kmat) * ((1 - c) / (s**2)) else: - return numpy.eye( - 3 - ) # cross of all zeros only occurs on identical directions + return numpy.eye(3) # cross of all zeros only occurs on identical directions @forge @@ -154,10 +150,7 @@ def update_section(self, section): self.debug(f"checking input values") assert all( - [ - val is not None - for val in (self.in_Iy, self.in_Ix, self.in_J, self.in_A) - ] + [val is not None for val in (self.in_Iy, self.in_Ix, self.in_J, self.in_A)] ) # Assemble tensor @@ -402,9 +395,7 @@ def estimate_stress(self, force_calc=True, **forces): else: return self._fallback_estimate_stress(**forces) - def _fallback_estimate_stress( - self, n, vx, vy, mxx, myy, mzz, SM=5, **extra - ): + def _fallback_estimate_stress(self, n, vx, vy, mxx, myy, mzz, SM=5, **extra): """sum the absolute value of each stress component. This isn't accurate but each value here should represent the worst sections, and take the 1-norm to max for each type of stress""" if self.section.x_bounds is None: @@ -613,9 +604,7 @@ def max_moment_y(self) -> float: return self.member.max_moment("My", self.structure.current_combo) # Load Application - def get_valid_force_choices( - only_local=False, only_global=False, use_moment=True - ): + def get_valid_force_choices(only_local=False, only_global=False, use_moment=True): if only_local or only_global: assert only_global != only_local, "choose local or global" @@ -657,9 +646,7 @@ def apply_pt_load(self, x_frac, case=None, **kwargs): self.member.name, Fkey, Fval, x, case=case ) - def apply_distributed_load( - self, start_factor=1, end_factor=1, case=None, **kwargs - ): + def apply_distributed_load(self, start_factor=1, end_factor=1, case=None, **kwargs): """add forces in global vector""" if case is None: case = self.structure.default_case @@ -708,9 +695,7 @@ def apply_local_distributed_load( case=case, ) - def apply_gravity_force_distribution( - self, sv=1, ev=1, z_dir="FZ", z_mag=-1 - ): + def apply_gravity_force_distribution(self, sv=1, ev=1, z_dir="FZ", z_mag=-1): # TODO: ensure that integral of sv, ev is 1, and all positive self.debug(f"applying gravity distribution to {self.name}") for case in self.structure.gravity_cases: diff --git a/engforge/eng/thermodynamics.py b/engforge/eng/thermodynamics.py index 1150136..c472fb6 100644 --- a/engforge/eng/thermodynamics.py +++ b/engforge/eng/thermodynamics.py @@ -195,10 +195,7 @@ def Tout(self) -> float: return self.Tin * ( 1 - self.efficiency - * ( - 1.0 - - (1 / self.pressure_ratio) ** ((self.gamma - 1.0) / self.gamma) - ) + * (1.0 - (1 / self.pressure_ratio) ** ((self.gamma - 1.0) / self.gamma)) ) # return self.Tin * self.pressure_ratio**((self.gamma-1.0)/self.gamma)/ self.efficiency diff --git a/engforge/engforge_attributes.py b/engforge/engforge_attributes.py index 2e9872d..9672cde 100644 --- a/engforge/engforge_attributes.py +++ b/engforge/engforge_attributes.py @@ -71,9 +71,7 @@ def collect_inst_attributes(self, **kw): return out @classmethod - def _get_init_attrs_data( - cls, subclass_of: type, exclude=False, attr_type=False - ): + def _get_init_attrs_data(cls, subclass_of: type, exclude=False, attr_type=False): choose = issubclass if exclude: choose = lambda ty, type_set: not issubclass(ty, type_set) @@ -145,9 +143,7 @@ def check_ref_slot_type(cls, sys_key: str) -> list: sub_clss = cls._extract_type(slts[fst].type) out = [] for acpt in sub_clss: - if isinstance(acpt, type) and issubclass( - acpt, Configuration - ): + if isinstance(acpt, type) and issubclass(acpt, Configuration): vals = acpt.check_ref_slot_type(".".join(rem)) # print(f'recursive find {acpt}.{rem} = {vals}') if vals: diff --git a/engforge/env_var.py b/engforge/env_var.py index d44026a..77212df 100644 --- a/engforge/env_var.py +++ b/engforge/env_var.py @@ -139,9 +139,7 @@ def secret(self): # Provide warning that the secret is being replaced if not self._upgrd_warn: self._upgrd_warn = True - self.info( - f"upgrading: {self.var_name} from {id(self)}->{id(sec)}" - ) + self.info(f"upgrading: {self.var_name} from {id(self)}->{id(sec)}") # Monkeypatch dictionary self.__dict__ = sec.__dict__ @@ -164,9 +162,7 @@ def secret(self): secval = self.default else: if self.fail_on_missing: - raise FileNotFoundError( - f"Could Not Find Env Variable {self.var_name}" - ) + raise FileNotFoundError(f"Could Not Find Env Variable {self.var_name}") else: if self.var_name not in warned: self.debug(f"Env Var: {self.var_name} Not Found!") @@ -212,9 +208,7 @@ def print_env_vars(cls): global HOSTNAME, SLACK_WEBHOOK -HOSTNAME = EnvVariable( - "FORGE_HOSTNAME", default=host, obscure=False, dontovrride=True -) +HOSTNAME = EnvVariable("FORGE_HOSTNAME", default=host, obscure=False, dontovrride=True) SLACK_WEBHOOK = EnvVariable( "FORGE_SLACK_LOG_WEBHOOK", default=None, obscure=False, dontovrride=True ) diff --git a/engforge/logging.py b/engforge/logging.py index 16e92cf..2437e0c 100644 --- a/engforge/logging.py +++ b/engforge/logging.py @@ -18,9 +18,7 @@ LOG_LEVEL = logging.INFO -def change_all_log_levels( - new_log_level: int = 20, inst=None, check_function=None -): +def change_all_log_levels(new_log_level: int = 20, inst=None, check_function=None): """Changes All Log Levels With pyee broadcast before reactor is running :param new_log_level: int - changes unit level log level (10-msg,20-debug,30-info,40-warning,50-error,60-crit) :param check_function: callable -> bool - (optional) if provided if check_function(unit) is true then the new_log_level is applied @@ -29,9 +27,7 @@ def change_all_log_levels( new_log_level = int(new_log_level) # Float Case Is Handled assert ( - isinstance(new_log_level, int) - and new_log_level >= 1 - and new_log_level <= 100 + isinstance(new_log_level, int) and new_log_level >= 1 and new_log_level <= 100 ) global LOG_LEVEL @@ -59,17 +55,13 @@ class LoggingMixin(logging.Filter): slack_webhook_url = None # log_silo = False - change_all_log_lvl = lambda s, *a, **kw: change_all_log_levels( - *a, inst=s, **kw - ) + change_all_log_lvl = lambda s, *a, **kw: change_all_log_levels(*a, inst=s, **kw) @property def logger(self): global LOG_LEVEL if self._log is None: - inst_log_name = ( - "engforgelog_" + self.identity + "_" + str(uuid.uuid4()) - ) + inst_log_name = "engforgelog_" + self.identity + "_" + str(uuid.uuid4()) self._log = logging.getLogger(inst_log_name) self._log.setLevel(level=LOG_LEVEL) @@ -163,9 +155,7 @@ def info(self, *args): def warning(self, *args): """Writes to log as a warning""" self.logger.warning( - self.message_with_identiy( - "WARN: " + self.extract_message(args), "yellow" - ) + self.message_with_identiy("WARN: " + self.extract_message(args), "yellow") ) def error(self, error, msg=""): diff --git a/engforge/patterns.py b/engforge/patterns.py index eb85d08..f8b76ce 100644 --- a/engforge/patterns.py +++ b/engforge/patterns.py @@ -71,9 +71,7 @@ class SingletonMeta(type): def __call__(cls, *args, **kwargs): if cls not in cls._instances: - cls._instances[cls] = super(SingletonMeta, cls).__call__( - *args, **kwargs - ) + cls._instances[cls] = super(SingletonMeta, cls).__call__(*args, **kwargs) return cls._instances[cls] @classmethod diff --git a/engforge/problem_context.py b/engforge/problem_context.py index 658d626..a26de9d 100644 --- a/engforge/problem_context.py +++ b/engforge/problem_context.py @@ -152,9 +152,7 @@ class ProbLog(LoggingMixin): save_modes = ["vars", "nums", "all", "prob"] transfer_kw = ["system", "_dxdt"] -root_possible = list(root_defined.keys()) + list( - "_" + k for k in root_defined.keys() -) +root_possible = list(root_defined.keys()) + list("_" + k for k in root_defined.keys()) # TODO: output options extend_dataframe=True,return_dataframe=True,condensed_dataframe=True,return_system=True,return_problem=True,return_df=True,return_data=True # TODO: connect save_data() output to _data table. @@ -297,9 +295,7 @@ def __getattr__(self, name): # Default behaviour return self.__getattribute__(name) - def __init__( - self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts - ): + def __init__(self, system, kw_dict=None, Xnew=None, ctx_fail_new=False, **opts): """ Initializes the ProblemExec. @@ -483,9 +479,7 @@ def __init__( self.set_checkpoint() if log.log_level < 10: - self.info( - f"new execution context for {system}| {opts} | {self._slv_kw}" - ) + self.info(f"new execution context for {system}| {opts} | {self._slv_kw}") elif log.log_level <= 3: self.msg(f"new execution context for {system}| {self._slv_kw}") @@ -768,11 +762,7 @@ def __enter__(self): self.class_cache.level_number = 0 if self.log_level < 10: - refs = { - k: v - for k, v in self.sesh._sys_refs.get("attrs", {}).items() - if v - } + refs = {k: v for k, v in self.sesh._sys_refs.get("attrs", {}).items() if v} self.debug( f"creating execution context for {self.system}| {self._slv_kw}| {refs}" ) @@ -845,9 +835,7 @@ def __exit__(self, exc_type, exc_value, traceback): if type(self.problems_dict) is not dict: self.problems_dict.pop(self._problem_id, None) del self.class_cache.session - raise KeyError( - f"cant exit to level! {exc_value.level} not found!!" - ) + raise KeyError(f"cant exit to level! {exc_value.level} not found!!") else: if self.log_level <= 18: @@ -989,10 +977,7 @@ def save_data(self, index=None, force=False, **add_data): self.warning(f"no data saved, nothing changed") def clean_context(self): - if ( - hasattr(self.class_cache, "session") - and self.class_cache.session is self - ): + if hasattr(self.class_cache, "session") and self.class_cache.session is self: if self.log_level <= 8: self.debug(f"closing execution session") self.class_cache.level_number = 0 @@ -1036,9 +1021,7 @@ def integrate(self, endtime, dt=0.001, max_step_dt=0.01, X0=None, **kw): dt = max_step_dt if self.log_level < 15: - self.info( - f"simulating {system},{sesh}| int:{intl_refs} | refs: {refs}" - ) + self.info(f"simulating {system},{sesh}| int:{intl_refs} | refs: {refs}") if not intl_refs: raise Exception(f"no transient parameters found") @@ -1264,9 +1247,7 @@ def handle_solution(self, answer, Xref, Yref, output): Ycon = {} if sesh.constraints["constraints"]: x_in = answer.x - for c, k in zip( - sesh.constraints["constraints"], sesh.constraints["info"] - ): + for c, k in zip(sesh.constraints["constraints"], sesh.constraints["info"]): cv = c["fun"](x_in, self, {}) Ycon[k] = cv output["Ycon"] = Ycon @@ -1379,9 +1360,7 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): ) slv_inst = sys_refs.get("type", {}).get("solver", {}) - trv_inst = { - v.var: v for v in sys_refs.get("type", {}).get("time", {}).values() - } + trv_inst = {v.var: v for v in sys_refs.get("type", {}).get("time", {}).values()} sys_refs = sys_refs.get("attrs", {}) if add_con is None: @@ -1456,22 +1435,12 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): combo_var = ctype["combo_var"] active = ctype.get("active", True) in_activate = ( - any( - [ - arg_var_compare(combo_var, v) - for v in activated - ] - ) + any([arg_var_compare(combo_var, v) for v in activated]) if activated else False ) in_deactivate = ( - any( - [ - arg_var_compare(combo_var, v) - for v in deactivated - ] - ) + any([arg_var_compare(combo_var, v) for v in deactivated]) if deactivated else False ) @@ -1482,16 +1451,12 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): # Check active or activated if not active and not activated: if log.log_level < 3: - self.msg( - f"skip con: inactive {var} {slvr} {ctype}" - ) + self.msg(f"skip con: inactive {var} {slvr} {ctype}") continue elif not active and not in_activate: if log.log_level < 3: - self.msg( - f"skip con: inactive {var} {slvr} {ctype}" - ) + self.msg(f"skip con: inactive {var} {slvr} {ctype}") continue elif active and in_deactivate: @@ -1509,9 +1474,7 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): continue if log.log_level < 10: - self.debug( - f"adding var constraint {var,slvr,ctype,combos}" - ) + self.debug(f"adding var constraint {var,slvr,ctype,combos}") # get the index of the variable x_inx = Xvars.index(slvr) @@ -1575,9 +1538,7 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): cval, **kw, ) - con_info.append( - f"val_{ref.comp.classname}_{kind}_{slvr}" - ) + con_info.append(f"val_{ref.comp.classname}_{kind}_{slvr}") con_list.append(ccst) else: @@ -1589,9 +1550,7 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): for slvr, ref in self.problem_ineq.items(): slv = slv_inst[slvr] slv_constraints = slv.constraints - parent = self.get_parent_key( - slvr, look_back_num=2 - ) # get the parent comp + parent = self.get_parent_key(slvr, look_back_num=2) # get the parent comp for ctype in slv_constraints: cval = ctype["value"] kind = ctype["type"] @@ -1613,9 +1572,7 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): ) for slvr, ref in self.problem_eq.items(): - parent = self.get_parent_key( - slvr, look_back_num=2 - ) # get the parent comp + parent = self.get_parent_key(slvr, look_back_num=2) # get the parent comp if slvr in slv_inst and slvr in all_refz.get("solver.eq", {}): slv = slv_inst[slvr] slv_constraints = slv.constraints @@ -1662,9 +1619,7 @@ def sys_solver_constraints(self, add_con=None, combo_filter=True, **kw): # General method to distribute input to internal components @classmethod - def parse_default( - self, key, defaults, input_dict, rmv=False, empty_str=True - ): + def parse_default(self, key, defaults, input_dict, rmv=False, empty_str=True): """splits strings or lists and returns a list of options for the key, if nothing found returns None if fail set to True raises an exception, otherwise returns the default value""" if key in input_dict: # kwargs will no longer have key! @@ -1757,9 +1712,7 @@ def output_state(self) -> dict: elif "prob" == sesh.save_mode: raise NotImplementedError(f"problem save mode not implemented") else: - raise KeyError( - f"unknown save mode {sesh.save_mode}, not in {save_modes}" - ) + raise KeyError(f"unknown save mode {sesh.save_mode}, not in {save_modes}") out = Ref.refset_get(refs, sys=sesh.system, prob=self) # Integration @@ -1783,9 +1736,7 @@ def set_ref_values(self, values, refs=None, scope="sref"): refs = sesh.all_comps_and_vars return Ref.refset_input(refs, values, scope=scope) - def change_sys_var( - self, key, value, refs=None, doset=True, attr_key_map=None - ): + def change_sys_var(self, key, value, refs=None, doset=True, attr_key_map=None): """use this function to change the value of a system var and update the start state, multiple uses in the same context will not change the record preserving the start value :param key: a string corresponding to a ref, or an `attrs.Attribute` of one of the system or its component's. @@ -2148,12 +2099,9 @@ def numeric_data(self): filter_non_numeric = lambda kv: ( False if isinstance(kv[1], (list, dict, tuple)) else True ) - f_numrow = lambda in_dict: dict( - filter(filter_non_numeric, in_dict.items()) - ) + f_numrow = lambda in_dict: dict(filter(filter_non_numeric, in_dict.items())) return [ - f_numrow(kv[-1]) - for kv in sorted(sesh.data.items(), key=lambda kv: kv[0]) + f_numrow(kv[-1]) for kv in sorted(sesh.data.items(), key=lambda kv: kv[0]) ] @property @@ -2302,6 +2250,4 @@ def level_name(self): @level_name.setter def level_name(self, value): - raise AttributeError( - f"cannot set level_name of top level problem context" - ) + raise AttributeError(f"cannot set level_name of top level problem context") diff --git a/engforge/properties.py b/engforge/properties.py index 9d1a931..29b4282 100644 --- a/engforge/properties.py +++ b/engforge/properties.py @@ -47,9 +47,7 @@ def __init__(self, fget=None, fset=None, fdel=None, *args, **kwargs): self.fset = fset self.fdel = fdel - def __call__( - self, fget=None, fset=None, fdel=None, doc=None, *args, **kwargs - ): + def __call__(self, fget=None, fset=None, fdel=None, doc=None, *args, **kwargs): """this will be called when input is provided before property is set""" if fget and self.fget is None: self.gname = fget.__name__ @@ -103,9 +101,7 @@ def deleter(self, fdel): class cache_prop(engforge_prop): - allow_set: bool = ( - False # keep this flag false to maintain current persistent value - ) + allow_set: bool = False # keep this flag false to maintain current persistent value def __init__(self, *args, **kwargs): self.allow_set = True @@ -348,9 +344,7 @@ def __get__(self, instance: "TabulationMixin", objtype=None): if not hasattr(instance, self.private_var): from engforge.tabulation import TabulationMixin - if ( - instance.__class__ is not None and not IS_BUILD - ): # its an instance + if instance.__class__ is not None and not IS_BUILD: # its an instance assert issubclass( instance.__class__, TabulationMixin ), f"incorrect class: {instance.__class__.__name__}" diff --git a/engforge/solveable.py b/engforge/solveable.py index 26ad657..486cd13 100644 --- a/engforge/solveable.py +++ b/engforge/solveable.py @@ -332,8 +332,7 @@ def _gen(gen, compkey): yield compkey, itemkey iter_vals = { - cn: _gen(comp._item_gen(), cn) - for cn, comp in components.items() + cn: _gen(comp._item_gen(), cn) for cn, comp in components.items() } for out in itertools.product(*list(iter_vals.values())): @@ -351,9 +350,7 @@ def comp_references(self, ignore_none_comp=True, **kw): #FIXME: by instance recache on iterative component change or other signals """ out = {} - for key, lvl, comp in self.go_through_configurations( - parent_level=1, **kw - ): + for key, lvl, comp in self.go_through_configurations(parent_level=1, **kw): if ignore_none_comp and not isinstance(comp, SolveableMixin): self.warning(f"ignoring {key} {lvl}|{comp}") continue @@ -391,12 +388,8 @@ def _iterate_input_matrix( from engforge.system import System from engforge.problem_context import ProblemExec - self.debug( - f"running [Solver].{method} {self.identity} with input {kwargs}" - ) - assert hasattr( - ProblemExec.class_cache, "session" - ), "must be active context!" + self.debug(f"running [Solver].{method} {self.identity} with input {kwargs}") + assert hasattr(ProblemExec.class_cache, "session"), "must be active context!" # create iterable null for sequence if sequence is None or not sequence: sequence = [{}] @@ -512,9 +505,7 @@ def parse_run_kwargs(self, **kwargs): comp_args = {k: v for k, v in kwargs.items() if "." in k} # check vars - inpossible = set.union( - set(self.input_fields()), set(self.slots_attributes()) - ) + inpossible = set.union(set(self.input_fields()), set(self.slots_attributes())) argdiff = set(var_args).difference(inpossible) assert not argdiff, f"bad input {argdiff}" @@ -524,9 +515,7 @@ def parse_run_kwargs(self, **kwargs): assert not compdiff, f"bad slot references {compdiff}" _input = {} - test = ( - lambda v, add: isinstance(v, (int, float, str, *add)) or v is None - ) + test = lambda v, add: isinstance(v, (int, float, str, *add)) or v is None # vars input for k, v in kwargs.items(): @@ -546,9 +535,7 @@ def parse_run_kwargs(self, **kwargs): assert test(v, addty), f"bad values {k}:{v}" v = [v] else: - assert all( - [test(vi, addty) for vi in v] - ), f"bad values: {k}:{v}" + assert all([test(vi, addty) for vi in v]), f"bad values: {k}:{v}" if k not in _input: _input[k] = v @@ -602,9 +589,7 @@ def locate_ref(self, key, fail=True, **kw): func = copy.copy(key) return Ref(comp, func, **kw) else: - assert ( - "comp" not in kw - ), f"comp kwarg not allowed with string key {key}" + assert "comp" not in kw, f"comp kwarg not allowed with string key {key}" if "." in key: args = key.split(".") @@ -627,10 +612,7 @@ def locate_ref(self, key, fail=True, **kw): # val= cls.system_properties_classdef()[key] return Ref(self, key, **kw) - elif ( - key in self.internal_configurations() - or key in self.slots_attributes() - ): + elif key in self.internal_configurations() or key in self.slots_attributes(): return Ref(self, key, **kw) # Fail on comand but otherwise return val @@ -678,9 +660,7 @@ def system_references(self, recache=False, numeric_only=False, **kw): comp_dict[key] = comp if parent in comp_dict: attr_name = key.split(".")[-1] - comp_set_ref[key] = Ref( - comp_dict[parent], attr_name, False, True - ) + comp_set_ref[key] = Ref(comp_dict[parent], attr_name, False, True) # Fill in for k, v in satr.items(): @@ -777,9 +757,7 @@ def collect_solver_refs( _var = getattr(conf, pre_var) if isinstance(_var, AttributeInstance): slv_type = _var - conf.msg( - f"slv type: {conf.classname}.{pre_var} -> {_var}" - ) + conf.msg(f"slv type: {conf.classname}.{pre_var} -> {_var}") val_type = ck_type[pre_var] @@ -793,9 +771,7 @@ def collect_solver_refs( cls_dict[atype][scope_name] = val_type if conf.log_level <= 5: - conf.msg( - f"rec: {var_name} {k} {pre} {val} {slv_type}" - ) + conf.msg(f"rec: {var_name} {k} {pre} {val} {slv_type}") # Check to skip this item # keep references even if null @@ -816,9 +792,7 @@ def collect_solver_refs( pre, scope_name, val_type, check_kw ): if conf.log_level <= 5: - conf.msg( - f"chk skip {scope_name} {k} {pre} {val}" - ) + conf.msg(f"chk skip {scope_name} {k} {pre} {val}") if pre not in skipped: skipped[pre] = [] @@ -865,11 +839,7 @@ def collect_solver_refs( # eval each ref for inclusion for var, ref in refs.items(): key_segs = var.split(".") - key = ( - "" - if len(key_segs) == 1 - else ".".join(key_segs[:-1]) - ) + key = "" if len(key_segs) == 1 else ".".join(key_segs[:-1]) scoped_name = f"{var}" conf = dyn_comp.get(key) @@ -968,9 +938,7 @@ def get_system_input_refs( if p in SKIP_REF and not all: continue if all: - refs[(f"{ckey}." if ckey else "") + p] = Ref( - comp, p, False, True - ) + refs[(f"{ckey}." if ckey else "") + p] = Ref(comp, p, False, True) continue elif atr.type: ty = atr.type diff --git a/engforge/solver.py b/engforge/solver.py index 96db6d8..bb64337 100644 --- a/engforge/solver.py +++ b/engforge/solver.py @@ -112,9 +112,7 @@ def solver_vars(self, check_dynamics=True, addable=None, **kwargs): for av in addvar: # list/dict-keys matches = set(fnmatch.filter(avars, av)) # Lookup Constraints if input is a dictionary - if isinstance(addvar, dict) and isinstance( - addvar[av], dict - ): + if isinstance(addvar, dict) and isinstance(addvar[av], dict): const = base_const.copy() const.update(addvar[av]) elif isinstance(addvar, dict): @@ -163,9 +161,7 @@ def run(self, **kwargs): with ProblemExec(self, kwargs, level_name="run") as pbx: # problem context removes slv/args from kwargs - return self._iterate_input_matrix( - self.eval, return_results=True, **kwargs - ) + return self._iterate_input_matrix(self.eval, return_results=True, **kwargs) def run_internal_systems(self, sys_kw=None): """runs internal systems with potentially scoped kwargs""" @@ -190,9 +186,7 @@ def run_internal_systems(self, sys_kw=None): comp.eval(**sys_kw_comp) # Single Point Flow - def eval( - self, Xo=None, eval_kw: dict = None, sys_kw: dict = None, cb=None, **kw - ): + def eval(self, Xo=None, eval_kw: dict = None, sys_kw: dict = None, cb=None, **kw): """Evaluates the system with pre/post execute methodology :param kw: kwargs come from `sys_kw` input in run ect. :param cb: an optional callback taking the system as an argument of the form (self,eval_kw,sys_kw,**kw) diff --git a/engforge/solver_utils.py b/engforge/solver_utils.py index 776421c..9f4ede8 100644 --- a/engforge/solver_utils.py +++ b/engforge/solver_utils.py @@ -156,9 +156,7 @@ def ref_to_val_constraint( return ref # Make Objective - return create_constraint( - system, comp, Xrefs, contype, ref, ctx, *args, **kwargs - ) + return create_constraint(system, comp, Xrefs, contype, ref, ctx, *args, **kwargs) def create_constraint( @@ -175,9 +173,7 @@ def create_constraint( ), f"bad constraint type: {contype}" if comp.log_level < 5: - comp.debug( - f"create constraint {contype} {ref} {args} {kwargs}| {con_args}" - ) + comp.debug(f"create constraint {contype} {ref} {args} {kwargs}| {con_args}") # its a function _fun = lambda *args, **kw: ref.value(*args, **kw) @@ -320,9 +316,7 @@ def ext_str_list(extra_kw, key, default=None): ] -def combo_filter( - attr_name, var_name, solver_inst, extra_kw, combos=None -) -> bool: +def combo_filter(attr_name, var_name, solver_inst, extra_kw, combos=None) -> bool: # TODO: allow solver_inst to be None for dyn-classes # proceed to filter active items if vars / combos inputs is '*' select all, otherwise discard if not active # corresondes to problem_context.slv_dflt_options @@ -487,9 +481,7 @@ def filt_active(var, inst, extra_kw=None, dflt=False): from engforge.attr_signals import SignalInstance # not considered - if not isinstance( - inst, (SolverInstance, IntegratorInstance, SignalInstance) - ): + if not isinstance(inst, (SolverInstance, IntegratorInstance, SignalInstance)): return True activate = ext_str_list(extra_kw, "activate", []) diff --git a/engforge/system.py b/engforge/system.py index 53d24e0..67b9d1f 100644 --- a/engforge/system.py +++ b/engforge/system.py @@ -127,9 +127,7 @@ def _anything_changed(self): """looks at internal components as well as flag for anything chagned.""" if self._anything_changed_: return True - elif any( - [c.anything_changed for k, c in self.comp_references().items()] - ): + elif any([c.anything_changed for k, c in self.comp_references().items()]): return True return False diff --git a/engforge/system_reference.py b/engforge/system_reference.py index 26035fd..196c65a 100644 --- a/engforge/system_reference.py +++ b/engforge/system_reference.py @@ -40,9 +40,7 @@ def refset_input(refs, delta_dict, chk=True, fail=True, warn=True, scope="ref"): elif fail and chk and not memb: close = get_close_matches(k, keys) - raise KeyError( - f"{scope}| key {k} not in refs. did you mean {close}?" - ) + raise KeyError(f"{scope}| key {k} not in refs. did you mean {close}?") elif warn and chk and not memb: close = get_close_matches(k, keys) log.warning(f"{scope}| key {k} not in refs. did you mean {close}") diff --git a/engforge/tabulation.py b/engforge/tabulation.py index ddec3ca..9df1dd2 100644 --- a/engforge/tabulation.py +++ b/engforge/tabulation.py @@ -95,22 +95,15 @@ def dataframe(self): :rtype: pandas.DataFrame """ """""" - if hasattr(self, "last_context") and hasattr( - self.last_context, "dataframe" - ): + if hasattr(self, "last_context") and hasattr(self.last_context, "dataframe"): return self.last_context.dataframe - if ( - hasattr(self, "_patch_dataframe") - and self._patch_dataframe is not None - ): + if hasattr(self, "_patch_dataframe") and self._patch_dataframe is not None: return self._patch_dataframe return pandas.DataFrame([]) @dataframe.setter def dataframe(self, input_dataframe): - if hasattr(self, "last_context") and hasattr( - self.last_context, "dataframe" - ): + if hasattr(self, "last_context") and hasattr(self.last_context, "dataframe"): raise Exception(f"may not set dataframe on run component") self._patch_dataframe = input_dataframe @@ -205,9 +198,7 @@ def system_properties_types(self) -> list: @instance_cached def system_properties_keys(self) -> list: """Returns the table property keys""" - tabulated_properties = [ - k for k, obj in self.system_properties_def.items() - ] + tabulated_properties = [k for k, obj in self.system_properties_def.items()] return tabulated_properties @instance_cached @@ -221,9 +212,7 @@ def system_properties_description(self) -> list: @classmethod def cls_all_property_labels(cls): - return [ - obj.label for k, obj in cls.system_properties_classdef().items() - ] + return [obj.label for k, obj in cls.system_properties_classdef().items()] @classmethod def cls_all_property_keys(cls): @@ -279,9 +268,7 @@ def system_properties_classdef(cls, recache=False): if prop and isinstance(prop, system_property): __system_properties[k] = prop if log.log_level <= 3: - log.msg( - f"adding system property {mrv.__name__}.{k}" - ) + log.msg(f"adding system property {mrv.__name__}.{k}") setattr(cls, cls_key, __system_properties) @@ -290,12 +277,7 @@ def system_properties_classdef(cls, recache=False): @classmethod def pre_compile(cls): cls._anything_changed = True # set default on class - if any( - [ - v.stochastic - for k, v in cls.system_properties_classdef(True).items() - ] - ): + if any([v.stochastic for k, v in cls.system_properties_classdef(True).items()]): log.info(f"setting always save on {cls.__name__}") cls._always_save_data = True diff --git a/engforge/test/_pre_test_structures.py b/engforge/test/_pre_test_structures.py index bc3dff0..4851881 100644 --- a/engforge/test/_pre_test_structures.py +++ b/engforge/test/_pre_test_structures.py @@ -40,9 +40,7 @@ def test_beam(self): self.subtest_assert_near(self.bm.section_mass, 41.9) self.subtest_assert_near(self.bm.max_von_mises(), 27.4e6) - self.subtest_assert_near( - float(self.bm.data_dict["min_deflection_y"]), -0.0076 - ) + self.subtest_assert_near(float(self.bm.data_dict["min_deflection_y"]), -0.0076) # self.subtest_assert_near(float(self.bm.data_dict["max_shear_y"]), 3000) self.subtest_assert_near(float(self.bm.data_dict["max_shear_y"]), 3000) @@ -64,9 +62,7 @@ def test_beam(self): def subtest_assert_near(self, value, truth, pct=0.025, **kw): with self.subTest(**kw): - self.assertAlmostEqual( - value, truth, delta=max(abs(truth * pct), abs(pct)) - ) + self.assertAlmostEqual(value, truth, delta=max(abs(truth * pct), abs(pct))) class test_truss(unittest.TestCase): @@ -90,9 +86,7 @@ def setUp(self): for n1 in self.st.nodes.values(): for n2 in self.st.nodes.values(): L = numpy.sqrt( - (n1.X - n2.X) ** 2.0 - + (n1.Y - n2.Y) ** 2.0 - + (n1.Z - n2.Z) ** 2.0 + (n1.X - n2.X) ** 2.0 + (n1.Y - n2.Y) ** 2.0 + (n1.Z - n2.Z) ** 2.0 ) if ( @@ -203,9 +197,7 @@ def subtest_member(self, nodea, nodeb, result_key, truth, pct=0.025): dopasst = abs(value - truth) <= abs(truth) * pct if not dopasst: - print( - f"fails {key} {result_key}| {value:3.5f} == {truth:3.5f}?" - ) + print(f"fails {key} {result_key}| {value:3.5f} == {truth:3.5f}?") self.subtest_assert_near(value, truth, pct=pct) def subtest_assert_near(self, value, truth, pct=0.025): diff --git a/engforge/test/report_testing.py b/engforge/test/report_testing.py index 8a10737..aac0572 100644 --- a/engforge/test/report_testing.py +++ b/engforge/test/report_testing.py @@ -52,9 +52,7 @@ class OtherComponent(Component): if suprise: ones = attr.ib(default=1111) - wacky = attr.ib( - default="one hundred and 111", validator=STR_VALIDATOR() - ) + wacky = attr.ib(default="one hundred and 111", validator=STR_VALIDATOR()) more = attr.ib(default="other stuff", validator=STR_VALIDATOR()) always_save_data = True @@ -139,9 +137,7 @@ def suprise(self): tc = analysis.internal_component # 1) Ensure exists Reflect The Database - db = DBConnection( - "reports", host="localhost", user="postgres", passd="dumbpass" - ) + db = DBConnection("reports", host="localhost", user="postgres", passd="dumbpass") db.ensure_database_exists() # db.engine.echo = True diff --git a/engforge/test/test_comp_iter.py b/engforge/test/test_comp_iter.py index 7d7d628..737bff8 100644 --- a/engforge/test/test_comp_iter.py +++ b/engforge/test/test_comp_iter.py @@ -110,9 +110,7 @@ def test_keys(self): self.assertTrue(mtch, msg=f"missing keys: {should_keys - sys_key}") # save the data to table - self.system.run( - revert_last=False, revert_every=False, save_on_exit=True - ) + self.system.run(revert_last=False, revert_every=False, save_on_exit=True) df = self.system.last_context.dataframe self.assertTrue(len(df) == 1, msg=f"len: {len(df)}|\n{str(df)}") diff --git a/engforge/test/test_costs.py b/engforge/test/test_costs.py index 4a593a0..2399c51 100644 --- a/engforge/test/test_costs.py +++ b/engforge/test/test_costs.py @@ -96,8 +96,7 @@ def test_econ_array(self): df = er.dataframe tc = ( - df["econ.summary.total_cost"] - == np.array([161.0, 220.0, 305.0, 390.0]) + df["econ.summary.total_cost"] == np.array([161.0, 220.0, 305.0, 390.0]) ).all() self.assertTrue(tc) diff --git a/engforge/test/test_dynamics_spaces.py b/engforge/test/test_dynamics_spaces.py index 212abe8..3e9b6b2 100644 --- a/engforge/test/test_dynamics_spaces.py +++ b/engforge/test/test_dynamics_spaces.py @@ -28,9 +28,7 @@ def test_steady_state(self): # TODO: check dxdt=0 combo results (dynamics/rates==states) - ans = ds.run( - dxdt=0, combos="time", revert_last=False, revert_every=False - ) + ans = ds.run(dxdt=0, combos="time", revert_last=False, revert_every=False) output = ans["output"][0] self.assertTrue(output["success"]) diff --git a/engforge/test/test_four_bar.py b/engforge/test/test_four_bar.py index 44d25b2..3bdefef 100644 --- a/engforge/test/test_four_bar.py +++ b/engforge/test/test_four_bar.py @@ -56,9 +56,7 @@ def r3_x_zero(self) -> float: def r3_x(self) -> float: """length defined by gamma/theta""" return ( - self.x2 - + self.ra * numpy.cos(self.theta) - - self.r1 * numpy.cos(self.gamma) + self.x2 + self.ra * numpy.cos(self.theta) - self.r1 * numpy.cos(self.gamma) ) @system_prop @@ -70,9 +68,7 @@ def y_zero(self) -> float: def y(self) -> float: """length defined by gamma/theta""" return ( - self.x2 - + self.ra * numpy.sin(self.theta) - - self.r1 * numpy.sin(self.gamma) + self.x2 + self.ra * numpy.sin(self.theta) - self.r1 * numpy.sin(self.gamma) ) @system_prop diff --git a/engforge/test/test_performance.py b/engforge/test/test_performance.py index 480dec8..bf90ef1 100644 --- a/engforge/test/test_performance.py +++ b/engforge/test/test_performance.py @@ -77,9 +77,7 @@ def test_transient(self): parser.add_argument( "-a", "--all", action="store_true", dest="all", help="run all tests" ) - parser.add_argument( - "-b", "--base", action="store_true", help="run all tests" - ) + parser.add_argument("-b", "--base", action="store_true", help="run all tests") args = parser.parse_args() diff --git a/engforge/test/test_pipes.py b/engforge/test/test_pipes.py index 06d8e70..eae7305 100644 --- a/engforge/test/test_pipes.py +++ b/engforge/test/test_pipes.py @@ -22,32 +22,16 @@ def test_pipe_analysis(self): n4za = PipeNode(x=10, y=10, z=10) n2zoa = FlowInput(x=10, y=5, z=15, flow_in=0.6) - pipe1 = Pipe( - v=random.random(), D=0.1 * random.random(), node_s=n1, node_e=n2 - ) - pipe2 = Pipe( - v=random.random(), D=0.1 * random.random(), node_s=n1, node_e=n3 - ) + pipe1 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n1, node_e=n2) + pipe2 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n1, node_e=n3) pipe3 = Pipe(v=random.random(), D=0.1, node_s=n2, node_e=n4) - pipe4 = Pipe( - v=random.random(), D=0.1 * random.random(), node_s=n3, node_e=n4 - ) - pipe5 = Pipe( - v=random.random(), D=0.1 * random.random(), node_s=n4, node_e=n5 - ) + pipe4 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n3, node_e=n4) + pipe5 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n4, node_e=n5) - pipe5 = Pipe( - v=random.random(), D=0.1 * random.random(), node_s=n2z, node_e=n2 - ) - pipe6 = Pipe( - v=random.random(), D=0.1 * random.random(), node_s=n3, node_e=n3z - ) - pipe7 = Pipe( - v=random.random(), D=0.1 * random.random(), node_s=n4, node_e=n4z - ) - pipe8 = Pipe( - v=random.random(), D=0.1 * random.random(), node_s=n2, node_e=n4z - ) + pipe5 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n2z, node_e=n2) + pipe6 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n3, node_e=n3z) + pipe7 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n4, node_e=n4z) + pipe8 = Pipe(v=random.random(), D=0.1 * random.random(), node_s=n2, node_e=n4z) pipe9 = Pipe( v=random.random(), D=0.1 * random.random(), node_s=n4zo, node_e=n4z ) diff --git a/engforge/test/test_problem.py b/engforge/test/test_problem.py index 0109ccd..5c57223 100644 --- a/engforge/test/test_problem.py +++ b/engforge/test/test_problem.py @@ -133,9 +133,7 @@ def test_slide_crank_design(self): def test_slide_crank_design_one_match(self): sm = SliderCrank(Tg=0) - pbx = ProblemExec( - sm, {"combos": "design", "dxdt": None, "both_match": False} - ) + pbx = ProblemExec(sm, {"combos": "design", "dxdt": None, "both_match": False}) atx = pbx.ref_attrs self.assertEqual( chk(atx, "solver.var"), @@ -146,9 +144,7 @@ def test_slide_crank_design_one_match(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv", "size")), ) - self.assertEqual( - chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) - ) + self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) @@ -160,9 +156,7 @@ def test_slide_crank_design_one_match(self): def test_slide_crank_design_slv_design(self): sm = SliderCrank(Tg=0) - pbx = ProblemExec( - sm, {"combos": "design", "slv_vars": "*slv", "dxdt": None} - ) + pbx = ProblemExec(sm, {"combos": "design", "slv_vars": "*slv", "dxdt": None}) atx = pbx.ref_attrs self.assertEqual(chk(atx, "solver.var"), set()) self.assertEqual(chk(atx, "solver.obj"), set(())) @@ -198,9 +192,7 @@ def test_slide_crank_design_slv(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv")), ) - self.assertEqual( - chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) - ) + self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) @@ -232,9 +224,7 @@ def test_slide_crank_design_slv_one_match(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv")), ) - self.assertEqual( - chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) - ) + self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) diff --git a/engforge/test/test_solver.py b/engforge/test/test_solver.py index 286837d..92dfd16 100644 --- a/engforge/test/test_solver.py +++ b/engforge/test/test_solver.py @@ -44,9 +44,7 @@ def test_comps_solver_obj(self): for act in [True, False]: for cmb in [inv, inv.split(",")]: actv = acts if act else aacts - extra = dict( - combos=cmb, slv_vars="*", activate=actv, only_active=True - ) + extra = dict(combos=cmb, slv_vars="*", activate=actv, only_active=True) info = self.sc.solver_vars(**extra)["attrs"] ans = {"x", "y", "z", "comp.x", "comp.y", "comp.z"} self.assertEqual(set(info["solver.var"]), ans) @@ -175,18 +173,14 @@ def setUp(self) -> None: self.sc = CubeSystem(comp=CubeComp()) def test_exec_results(self): - extra = dict( - combos=indep_l, slv_vars=indep, activate=[], only_active=True - ) + extra = dict(combos=indep_l, slv_vars=indep, activate=[], only_active=True) with ProblemExec(self.sc, extra) as pb: o = self.sc.execute(**extra) self.assertDictEqual(o["Xstart"], o["Xans"]) def test_run_results(self): """test that inputs stay the same when no objecives present""" - extra = dict( - combos=indep, slv_vars=indep_l, activate=[], only_active=True - ) + extra = dict(combos=indep, slv_vars=indep_l, activate=[], only_active=True) scx, scy, scz = self.sc.x, self.sc.y, self.sc.z sccx, sccy, sccz = self.sc.comp.x, self.sc.comp.y, self.sc.comp.z diff --git a/engforge/test/test_tabulation.py b/engforge/test/test_tabulation.py index 30bab65..613f61d 100644 --- a/engforge/test/test_tabulation.py +++ b/engforge/test/test_tabulation.py @@ -57,9 +57,7 @@ def setUp(self): # rfil = os.path.join(self.test_dir, fil) def test_property_labels(self): - ans = set( - ["four", "test_two", "test_one", "three", "converged", "run_id"] - ) + ans = set(["four", "test_two", "test_one", "three", "converged", "run_id"]) self.assertEqual(set(self.test_config.system_properties_labels), ans) def test_property_types(self): @@ -123,9 +121,7 @@ def test_assemble_data_on_input(self): # Change Something cur_val = self.test_config.attrs_prop new_val = 6 + cur_val - self.test_config.info( - f"setting attrs prop on in {cur_val } => {new_val}" - ) + self.test_config.info(f"setting attrs prop on in {cur_val } => {new_val}") self.test_config.attrs_prop = new_val px.save_data() @@ -139,9 +135,7 @@ def test_dataframe(self, iter=5): with ProblemExec(self.test_config, {}) as px: cur_val = self.test_config.attrs_prop attr_in[i] = val = cur_val + i**2.0 - self.test_config.info( - f"setting attrs prop df {cur_val } => {val}" - ) + self.test_config.info(f"setting attrs prop df {cur_val } => {val}") self.test_config.attrs_prop = val px.save_data() px.exit_with_state() @@ -231,9 +225,7 @@ def test_input(self, num=10): with ProblemExec(self.test_config, {}) as px: for i in range(num): with ProblemExec(self.test_config, {}) as px: - self.test_config.attrs_prop = ( - i + self.test_config.attrs_prop - ) + self.test_config.attrs_prop = i + self.test_config.attrs_prop px.save_data() postdict = self.test_config.data_dict self.assertDictEqual(cur_dict, postdict) diff --git a/examples/air_filter.py b/examples/air_filter.py index 023c14c..1c98fc6 100644 --- a/examples/air_filter.py +++ b/examples/air_filter.py @@ -48,9 +48,7 @@ class Airfilter(System): pr_eq = Solver.constraint_equality("sum_dP", 0, combos="flow") - flow_curve = Plot.define( - "throttle", "w", kind="lineplot", title="Flow Curve" - ) + flow_curve = Plot.define("throttle", "w", kind="lineplot", title="Flow Curve") @system_property def dP_parasitic(self) -> float: diff --git a/examples/spring_mass.py b/examples/spring_mass.py index f2ab064..42a1660 100644 --- a/examples/spring_mass.py +++ b/examples/spring_mass.py @@ -68,7 +68,5 @@ def accl(self) -> float: if __name__ == "__main__": # Run The System, Compare damping `u`=0 & 0.1 sm = SpringMass(x=0.0) - trdf = sm.simulate( - dt=0.01, endtime=10, u=[0.0, 0.1], combos="*", slv_vars="*" - ) + trdf = sm.simulate(dt=0.01, endtime=10, u=[0.0, 0.1], combos="*", slv_vars="*") trdf.groupby("run_id").plot("time", "x") diff --git a/test/test_problem.py b/test/test_problem.py index 3b5c7f4..44ac653 100644 --- a/test/test_problem.py +++ b/test/test_problem.py @@ -133,9 +133,7 @@ def test_slide_crank_design(self): def test_slide_crank_design_one_match(self): sm = SliderCrank(Tg=0) - pbx = ProblemExec( - sm, {"combos": "design", "dxdt": None, "both_match": False} - ) + pbx = ProblemExec(sm, {"combos": "design", "dxdt": None, "both_match": False}) atx = pbx.ref_attrs self.assertEqual( chk(atx, "solver.var"), @@ -146,9 +144,7 @@ def test_slide_crank_design_one_match(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv", "size")), ) - self.assertEqual( - chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) - ) + self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) @@ -160,9 +156,7 @@ def test_slide_crank_design_one_match(self): def test_slide_crank_design_slv_design(self): sm = SliderCrank(Tg=0) - pbx = ProblemExec( - sm, {"combos": "design", "slv_vars": "*slv", "dxdt": None} - ) + pbx = ProblemExec(sm, {"combos": "design", "slv_vars": "*slv", "dxdt": None}) atx = pbx.ref_attrs self.assertEqual(chk(atx, "solver.var"), set()) self.assertEqual(chk(atx, "solver.obj"), set(())) @@ -198,9 +192,7 @@ def test_slide_crank_design_slv(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv")), ) - self.assertEqual( - chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) - ) + self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set()) @@ -232,9 +224,7 @@ def test_slide_crank_design_slv_one_match(self): chk(atx, "solver.ineq"), set(("crank_pos_slv", "gear_pos_slv", "motor_pos_slv")), ) - self.assertEqual( - chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv")) - ) + self.assertEqual(chk(atx, "solver.eq"), set(("gear_speed_slv", "range_slv"))) self.assertEqual(chk(atx, "dynamics.output"), set()) self.assertEqual(chk(atx, "dynamics.rate"), set()) self.assertEqual(chk(atx, "dynamics.state"), set())