diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8e1eddd --- /dev/null +++ b/.gitignore @@ -0,0 +1,94 @@ +.idea/ + +# cached data +*.db + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c0d7533 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Stitch Fix + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..d95a809 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# Nodebook + +[![CircleCI](https://circleci.com/gh/stitchfix/nodebook.svg?style=shield)](https://circleci.com/gh/stitchfix/nodebook) + +Nodebook is a plugin for Jupyter Notebook designed to enforce an ordered flow of cell execution. Conceptually, Nodebook notebooks operate like a script where each cell depends on the cells above it. This prevents messy and difficult to maintain out-of-order execution which frequently occurs in vanilla Jupyter notebooks where each cell modifies the global state. For more information, see [this post](http://multithreaded.stitchfix.com/blog/2017/07/26/nodebook/). + + +## Installation + +Nodebook is available on pypi and can be installed with pip: +``` +pip install nodebook +``` + +## Usage + +To use Nodebook, add the following lines to a cell in your Jupyter notebook: +``` +#pragma nodebook off +%load_ext nodebookext +%nodebook {mode} {name} +``` +Where `{mode}` is one of `memory` or `disk`, and `{name}` is a unique identifier for your notebook. + +Mode determines whether variables are stored in memory or on disk. + +For additional example usage, see [nodebook_demo.ipynb](./nodebook_demo.ipynb). Also see below for a quick demo showing the basic difference in behavior between Nodebook and standard Jupyter: + +![demo](https://user-images.githubusercontent.com/6323667/28484590-0935af6a-6e28-11e7-8bfa-f1555001bac4.gif) + +## FAQ + +#### Q: Why am I seeing "ERROR:root:Cell magic `%%execute_cell` not found."? + +Nodebook loads a javascript plugin to modify Jupyter's behavior. If you encounter this error, it means that the javascript plugin is loaded but the ipython plugin is not. This can happen when the javascript is already loaded but you have restarted the kernel and haven't run `%nodebook {mode} {name}`. The solution is either to run the `%nodebook` magic (if you want to run nodebook), or delete the cell with the `%nodebook` magic and refresh your browser to unload the javascript (if you want to turn nodebook off). + +#### Q: What are the tradeoffs between memory and disk mode? + +Nodebook serializes all cell outputs to maintain consistent state between cells. In `memory` mode, objects are serialized to an in-memory dictionary, in `disk` mode objects are serialized to a directory within your notebook's working directory. Speed can be a factor when choosing between them, but on a modern SSD, serialization time generally dominates and `memory` and `disk` mode have similar performance. The main consideration is that `disk` mode has the advantage of persisting your environment when the python kernel is restarted, but the disadvantage of leaving behind a directory on your local filesystem that you may want to manually clean up later (this can add up especially if you are working with large objects in your notebook). + +#### Q: What are the limitations of Nodebook? + +While Nodebook supports most Python operations, it has a few limitations related to the use of serialization. First, not all objects are currently serializable, most noteably generators. Second, serialization adds some extra time. This is imperceptible for small objects, but is noticable for objects larger than a few hundred MB. Instead of working directly with very large objects in Nodebook, I recommend using it to prototype your analysis on a subset of data. diff --git a/ipython/extensions/nodebookext.py b/ipython/extensions/nodebookext.py new file mode 100644 index 0000000..30dc7a5 --- /dev/null +++ b/ipython/extensions/nodebookext.py @@ -0,0 +1,99 @@ +import cPickle as pickle +import os +import sys +import errno + +from nodebook.nodebookcore import Node, Nodebook, ReferenceFinder +from nodebook.pickledict import PickleDict + +NODEBOOK_STATE = { + "cache_dir": None, + "nodebook": None, +} + +MODE_DISK = "disk" +MODE_MEMORY = "memory" +ALLOWED_MODES = [MODE_DISK, MODE_MEMORY] + + +def nodebook(line): + """ + ipython magic for initializing nodebook, expects name for nodebook database + """ + args = line.lstrip().split(' ') + + try: + mode = args[0] + assert mode in ALLOWED_MODES + except (IndexError, AssertionError): + raise SyntaxError("Must specify mode as %s" % str(ALLOWED_MODES)) + + if mode == MODE_MEMORY: + persist = False + else: + persist = True + + if persist: + NODEBOOK_STATE['cache_dir'] = 'nodebook_cache/' + try: + NODEBOOK_STATE['cache_dir'] += args[1] + except IndexError: + NODEBOOK_STATE['cache_dir'] += 'default' + + try: + os.makedirs(NODEBOOK_STATE['cache_dir']) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(NODEBOOK_STATE['cache_dir']): + pass + else: + raise + try: + with open(os.path.join(NODEBOOK_STATE['cache_dir'], 'nodebook.p'), 'rb') as f: + NODEBOOK_STATE['nodebook'] = pickle.load(f) + except IOError: + var_store = PickleDict(NODEBOOK_STATE['cache_dir']) + NODEBOOK_STATE['nodebook'] = Nodebook(var_store) + else: + var_store = PickleDict() + NODEBOOK_STATE['nodebook'] = Nodebook(var_store) + + if len(NODEBOOK_STATE['nodebook'].nodes) > 0: + NODEBOOK_STATE['nodebook'].update_all_prompts(get_ipython().payload_manager) + + +def execute_cell(line, cell): + """ + ipython magic for executing nodebook cell, expects cell id and parent id inline, followed by code + """ + assert NODEBOOK_STATE['nodebook'] is not None, "Nodebook not initialized, please use %nodebook {nodebook_name}" + cell_id, parent_id = line.lstrip().split(' ') + + # make sure cell exists and is in the right position + NODEBOOK_STATE['nodebook'].insert_node_after(cell_id, parent_id) + + # update code and run + NODEBOOK_STATE['nodebook'].update_code(cell_id, cell) + res, objs = NODEBOOK_STATE['nodebook'].run_node(cell_id) + + # update prompts + NODEBOOK_STATE['nodebook'].update_all_prompts(get_ipython().payload_manager) + + # update cache if needed + if NODEBOOK_STATE['cache_dir'] is not None: + with open(os.path.join(NODEBOOK_STATE['cache_dir'], 'nodebook.p'), 'wb') as f: + pickle.dump(NODEBOOK_STATE['nodebook'], f, protocol=2) + + # UGLY HACK - inject outputs into global environment for autocomplete support + # TODO: find a better way to handle autocomplete + sys._getframe(2).f_globals.update(objs) + return res + + +def load_ipython_extension(ipython): + ipython.register_magic_function(nodebook, magic_kind='line') + ipython.register_magic_function(execute_cell, magic_kind='cell') + ipython.run_cell_magic('javascript', '', "Jupyter.utils.load_extensions('nodebookext')") + + +def unload_ipython_extension(ipython): + pass diff --git a/ipython/nbextensions/nodebookext.js b/ipython/nbextensions/nodebookext.js new file mode 100644 index 0000000..6cc03dc --- /dev/null +++ b/ipython/nbextensions/nodebookext.js @@ -0,0 +1,149 @@ +define([ + 'base/js/namespace', + 'notebook/js/codecell', + 'notebook/js/notebook' + ], + function( + IPython, + codecell, + notebook + ){ + var CodeCell = codecell.CodeCell; + var Notebook = notebook.Notebook; + + function _on_load(){ + console.info('[Nodebook] Extension loaded') + // currently we reinitialze all cells on import + // TODO: allow recovery of previously defined state + var cells = IPython.notebook.get_cells(); + var name_count = 0; + for(var i in cells){ + var cell = cells[i]; + if ((cell instanceof IPython.CodeCell)) { + if ("node_name" in cell.metadata) { + cell.cell_id = cell.metadata["node_name"] + } else { + cell.metadata["node_exists"] = false; + cell.metadata["node_name"] = cell.cell_id; + } + } + } + + function handle_test_payload(payload) { + var cells = this.notebook.get_cells(); + var cell_by_id = cells.find(function(cell, index){return cell.cell_id == payload['cell_id']}); + + try { + cell_by_id.set_input_prompt(payload['prompt']) + } catch(err) { + console.log(err) + } + } + + function patch_CodeCell_execute () { + console.info('[Nodebook] Patching cell execute') + CodeCell.prototype.execute = function (stop_on_error) { + if (!this.kernel) { + console.log("Can't execute cell since kernel is not set."); + return; + } + + if (stop_on_error === undefined) { + stop_on_error = true; + } + + this.output_area.clear_output(false, true); + var old_msg_id = this.last_msg_id; + if (old_msg_id) { + this.kernel.clear_callbacks_for_msg(old_msg_id); + delete CodeCell.msg_cells[old_msg_id]; + this.last_msg_id = null; + } + if (this.get_text().trim().length === 0) { + // nothing to do + this.set_input_prompt(null); + return; + } + this.set_input_prompt('*'); + this.element.addClass("running"); + + + ////// MODIFIED SECTION + var callbacks = this.get_callbacks(); + + // add in an extra callback here -- probably not the best place though + callbacks['shell']['payload']['set_prompt'] = $.proxy(handle_test_payload, this); + + + // first get any pragmas + var my_code = this.get_text(); + if (my_code.substr(0, 16) == "#pragma nodebook") { + var pragma_arg = my_code.split("\n")[0].substr(17); + } else { + var pramga_arg = null; + } + + if (pragma_arg == "off") { + console.log("Running outside nodebook"); + var exec_text = my_code + this.metadata["node_exists"] = false; + } else { + // find id of this cell and the previous code cell + var node_name = this.cell_id; + var index = this.notebook.find_cell_index(this); + + // find the first previous non-empty code cell + do { + index--; + parent = this.notebook.get_cell(index); + is_valid = (parent instanceof IPython.CodeCell) && (parent.metadata['node_exists'] == true) + } + while ((index > 0) && !is_valid); + var parent_name = parent.cell_id; + if (index == 0 && !is_valid) { + // no valid parent found + parent_name = ""; + } + + // put ipython cell magics first (mostly for %%time) + if (my_code.startsWith('%%')) { + var split_code = my_code.split('\n'); + var external_magic = split_code.shift(); + external_magic = external_magic.concat('\n'); + my_code = split_code.join('\n'); + } else { + var external_magic = ""; + } + + var nodebook_magic = "%%execute_cell ".concat(node_name, " ", parent_name, "\n"); + var exec_text = nodebook_magic.concat(my_code); + exec_text = external_magic.concat(exec_text); + + // mark as existing + this.metadata["node_name"] = this.cell_id; + this.metadata["node_exists"] = true; + } + this.last_msg_id = this.kernel.execute(exec_text, callbacks, {silent: false, store_history: true, + stop_on_error : stop_on_error}); + ////// END MODIFIED SECTION + + CodeCell.msg_cells[this.last_msg_id] = this; + this.render(); + this.events.trigger('execute.CodeCell', {cell: this}); + var that = this; + function handleFinished(evt, data) { + if (that.kernel.id === data.kernel.id && that.last_msg_id === data.msg_id) { + that.events.trigger('finished_execute.CodeCell', {cell: that}); + that.events.off('finished_iopub.Kernel', handleFinished); + } + } + this.events.on('finished_iopub.Kernel', handleFinished); + }; + } + + patch_CodeCell_execute() + } + + + return {load_ipython_extension: _on_load }; +}) \ No newline at end of file diff --git a/nodebook/__init__.py b/nodebook/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nodebook/nodebookcore.py b/nodebook/nodebookcore.py new file mode 100644 index 0000000..a12eda5 --- /dev/null +++ b/nodebook/nodebookcore.py @@ -0,0 +1,346 @@ +from . import pickledict +import ast +import __builtin__ + +INDENT = ' ' # an indent is canonically 4 spaces ;) + + +class ReferenceFinder(ast.NodeVisitor): + def __init__(self): + self.locals = set() + self.inputs = set() + self.imports = set() + + def visit_Assign(self, node): + # we need to visit "value" before "targets" + self.visit(node.value) + for target in node.targets: + self.visit(target) + + def generic_comp(self, node): + # we need to visit generators before elt + for generator in node.generators: + self.visit(generator) + self.visit(node.elt) + + def visit_ListComp(self, node): + return self.generic_comp(node) + + def visit_SetComp(self, node): + return self.generic_comp(node) + + def visit_GeneratorExp(self, node): + return self.generic_comp(node) + + def visit_DictComp(self, node): + # we need to visit generators before key/value + for generator in node.generators: + self.visit(generator) + self.visit(node.value) + self.visit(node.key) + + def visit_FunctionDef(self, node): + self.locals.add(node.name) + self.generic_visit(node) + + def visit_AugAssign(self, node): + target = node.target + while (type(target) is ast.Subscript): + target = target.value + if target.id not in self.locals: + self.inputs.add(target.id) + self.generic_visit(node) + + def visit_Name(self, node): + if type(node.ctx) in {ast.Store, ast.Param}: + self.locals.add(node.id) + elif type(node.ctx) is ast.Load: + if node.id not in self.locals: + self.inputs.add(node.id) + + def visit_alias(self, node): + self.imports.add(node.name) + if node.asname is not None: + self.locals.add(node.asname) + else: + self.locals.add(node.name) + + +class Nodebook(object): + """ + Nodebook maintains a variable store for accessing variables and a pointer to the head node in the list + """ + + def __init__(self, variable_store): + self.variables = variable_store + self.refcount = {} + self.head = None + self.nodes = {} + + def add_ref(self, val_hash): + """ + Increment reference count for value hash + """ + self.refcount[val_hash] = self.refcount.get(val_hash, 0) + 1 + + def remove_ref(self, val_hash): + """ + Decrement reference count for value hash + """ + self.refcount[val_hash] -= 1 + if self.refcount[val_hash] == 0: + del self.variables[val_hash] + + def update_code(self, node_id, code): + """ + Update target node's code + """ + node = self.nodes[node_id] + node.update_code(code) + + def run_node(self, node_id): + """ + Run target node and retrieve expression result and modified objects + """ + # get node inputs + node = self.nodes[node_id] + input_objs = {} + input_hashes = {} + for var in node.inputs.keys(): + val_hash = self._find_latest_output(node.parent, var) + if val_hash is not None: + input_objs[var] = self.variables[val_hash] + input_hashes[var] = val_hash + + # run node + res, output_objs, output_hashes = node.run(input_objs, input_hashes) + + # update node outputs + for var, val in output_objs.iteritems(): + self.variables[output_hashes[var]] = val + self._update_output_hashes(node, output_hashes) + + return res, output_objs + + def _find_latest_output(self, node, var): + """ + Find the most recent output hash for a variable starting from node + Fails if undefined unless var is a builtin + """ + # base case + if node is None: + if var in __builtin__.__dict__: + return None + else: + raise KeyError("name '%s' is not defined" % var) + + if var in node.outputs: + # found it, but make sure we're valid + if node.valid: + return node.outputs[var] + else: + # re-run the parent if it wasn't valid + # TODO: synchronize output with frontend javascript + print "auto-running invalidated node N_%s (%s)" % (node.get_index() + 1, node.name) + self.run_node(node.name) + return self._find_latest_output(node, var) + else: + # check next parent + return self._find_latest_output(node.parent, var) + + def _update_output_hashes(self, node, outputs): + """ + Update node's output hashes and invalid downstream nodes that depended on their previous values + """ + # invalidate any any children relying on specific hash-versions of old outputs that aren't in the new outputs + invalidated_outputs = set(node.outputs.iteritems()) - set(outputs.iteritems()) + invalidated_outputs = {k: v for k, v in invalidated_outputs} + + # also invalidate any children that rely on any version of a brand-new output, regardless of hash + # TODO this is potentially overly restrictive, if, eg, a value is blindly over-written again later + # TODO(con't) we should try to account for this to avoid invalidating excessively many cells + new_outputs = set(outputs.iteritems()) - set(node.outputs.iteritems()) + new_outputs = {k: v for k, v in new_outputs} + + # update reference counts + for val_hash in new_outputs.itervalues(): + self.add_ref(val_hash) + for val_hash in invalidated_outputs.itervalues(): + self.remove_ref(val_hash) + + # invalidate changed outputs + invalidated_outputs.update({k: None for k, _ in new_outputs.iteritems()}) + node.outputs = outputs + node.invalidate_children(invalidated_outputs) + + def insert_node_after(self, node_id, parent_id): + """ + Create node with id node_id if it doesn't exist, then move it to a position after parent_id + """ + # get the node by id or make a new node + if node_id in self.nodes: + node = self.nodes[node_id] + else: + node = Node(node_id) + self.nodes[node_id] = node + + # get the parent by id or leave empty + parent = self.nodes.get(parent_id, None) + + if parent is None: + # no parent, node is head + if self.head != node: + node.child = self.head + self.head = node + elif parent == node.parent: + # node is already in the right place, don't need to do anything + pass + else: + # first, extract node from its current position + old_parent = node.parent + old_child = node.child + if old_parent is not None: + old_parent.child = old_child + if old_child is not None: + old_child.parent = old_parent + + # next, insert node to its new location + node.parent = parent + if parent is not None: + # put node in between target parent and parent's old child + node.child = parent.child + parent.child = node + # if node now has a child, set it as child's parent + if node.child is not None: + node.child.parent = node + + def update_all_prompts(self, ipython_payload_manager): + """ + Update prompts for all nodes based on their position in list + """ + node = self.head + index = 1 + while node is not None: + if not node.valid: + prompt = "X" + else: + prompt = "N_%d" % index + self.update_prompt(node, prompt, ipython_payload_manager) + node = node.child + index += 1 + + def update_prompt(self, node, prompt, ipython_payload_manager): + """ + Use ipython payload manager to update prompt for target node + """ + payload = { + "source": "set_prompt", + "cell_id": node.name, + "prompt": prompt, + } + ipython_payload_manager.write_payload(payload, single=False) + + +class Node(object): + def __init__(self, name): + self.name = name + self.parent = None + self.child = None + self.valid = True + self.inputs = {} + self.outputs = {} + self.imports = set() + self.code = '' + + def update_code(self, code): + """ + Parse a block of python code for its inputs and assign to this node + """ + self.code = code + tree = ast.parse(code) + rf = ReferenceFinder() + rf.visit(tree) + self.inputs = {x: None for x in rf.inputs} + self.imports = rf.imports + self.valid = False # not valid until executed + + def run(self, input_objs, input_hashes): + """ + Execute this node in the provided environment given hashes of inputs + """ + env = input_objs + + # if code ends in an expression, execute it as an expression, otherwise execute whole block + block = ast.parse(self.code) + if len(block.body) > 0 and type(block.body[-1]) is ast.Expr: + last = ast.Expression(block.body.pop().value) + exec compile(block, '', mode='exec') in env + res = eval(compile(last, '', mode='eval'), env) + else: + exec compile(block, '', mode='exec') in env + res = None + + # find outputs which have changed from input hashes + self.inputs = input_hashes + output_objs = {} + output_hashes = {} + for var in [k for k in env.keys() if k != '__builtins__']: + val = env[var] + val_hash = pickledict.hash(val) + if self.inputs.get(var, 0) != val_hash: + output_hashes[var] = val_hash + output_objs[var] = val + self.valid = True + return res, output_objs, output_hashes + + def get_index(self): + """ + Return index of this node + """ + if self.parent is None: + return 0 + else: + return 1 + self.parent.get_index() + + def invalidate_children(self, outputs): + """ + Invalidate any children of this node that depend on the provided outputs + """ + if len(outputs) == 0: + return # no action for empty outputs + + child = self.child + while child is not None: + child.invalidate(outputs) + child = child.child + + def invalidate(self, inputs=None): + """ + Invalidate node and any children that depend on its outputs + Optionally conditional on this node using a specific input variable / hash + """ + # only need to take action if this node _was_ valid + if not self.valid: + return + + # don't invalidate if input var is specified and is not one of our inputs + if inputs is not None: + shared_inputs = set(inputs).intersection(self.inputs) + if len(shared_inputs) == 0: + return + + # also don't invalidate if no specified input hashes match + input_match = False + for input in shared_inputs: + if inputs[input] is None or inputs[input] == self.inputs[input]: + input_match = True + if not input_match: + return + + self.valid = False + + # also invalidate children that rely on our outputs + self.invalidate_children(self.outputs) + + def __str__(self): + return self.name diff --git a/nodebook/pickledict.py b/nodebook/pickledict.py new file mode 100644 index 0000000..6e8c4be --- /dev/null +++ b/nodebook/pickledict.py @@ -0,0 +1,126 @@ +import io +import os +from functools import partial +import hashlib +import pandas as pd +import msgpack +import inspect + +# using dill instead of pickle for more complete serialization +import dill + +# Use cStringIO if available. +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +import UserDict + +PANDAS_CODE = 1 +DILL_CODE = 2 + + +def msgpack_serialize(obj): + if type(obj) is pd.DataFrame or type(obj) is pd.Series: + try: + return msgpack.ExtType(PANDAS_CODE, obj.to_msgpack()) + except: + # pandas msgpack support is experimental and sometimes fails + return msgpack.ExtType(DILL_CODE, dill.dumps(obj, recurse=True)) + else: + if inspect.isclass(obj): + # dynamically defined classes default to __builtin__ but are only serializable in __main__ + if obj.__module__ == '__builtin__': + obj.__module__ = '__main__' + return msgpack.ExtType(DILL_CODE, dill.dumps(obj, recurse=True)) + + +def msgpack_deserialize(code, data): + if code == PANDAS_CODE: + return pd.read_msgpack(data) + elif code == DILL_CODE: + return dill.loads(data) + else: + return msgpack.ExtType(code, data) + + +class PickleDict(object, UserDict.DictMixin): + """ + Dictionary with immutable elements using pickle(dill), optionally supporting persisting to disk + """ + + def __init__(self, persist_path=None): + """ + persist_path: if provided, perform serialization to/from disk to this path + """ + self.persist_path = persist_path + self.dump = partial(msgpack.dump, default=msgpack_serialize) + self.load = partial(msgpack.load, ext_hook=msgpack_deserialize) + + self.dict = {} + + def keys(self): + return self.dict.keys() + + def __len__(self): + return len(self.dict) + + def has_key(self, key): + return key in self.dict + + def __contains__(self, key): + return key in self.dict + + def get(self, key, default=None): + if key in self.dict: + return self[key] + return default + + def __getitem__(self, key): + if self.persist_path is not None: + path = self.dict[key] + with open(path, 'rb') as f: + value = self.load(f) + else: + f = StringIO(self.dict[key]) + value = self.load(f) + return value + + def __setitem__(self, key, value): + if self.persist_path is not None: + path = os.path.join(self.persist_path, '%s.pak' % key) + with open(path, 'wb') as f: + self.dump(value, f) + self.dict[key] = path + else: + f = StringIO() + self.dump(value, f) + serialized = f.getvalue() + self.dict[key] = serialized + + def __delitem__(self, key): + if self.persist_path is not None: + os.remove(self.dict[key]) + del self.dict[key] + + +def hash(obj, hash_name='md5'): + """ + get a hash of a python object based on its serialized data + """ + # TODO: avoid the double-serialization of pickling both for hashing and storage -- can accomplish by refactoring a hash&store method into pickledict + # TODO OR find a faster way to hash? + stream = io.BytesIO() + dump = partial(msgpack.dump, default=msgpack_serialize) + hasher = hashlib.new(hash_name) + + try: + dump(obj, stream) + except dill.PicklingError as e: + e.args += ('PicklingError while hashing %r: %r' % (obj, e),) + raise + + dumps = stream.getvalue() + hasher.update(dumps) + return hasher.hexdigest() diff --git a/nodebook/utils.py b/nodebook/utils.py new file mode 100644 index 0000000..5bffb80 --- /dev/null +++ b/nodebook/utils.py @@ -0,0 +1,52 @@ +import json +from nodebookcore import INDENT + + +def output_to_function(output_node, main_closing_statement, args): + def add_dependencies(node_inputs, dep_set): + dep = {(k, v) for k, v in node_inputs.iteritems() if k not in args and v is not None} + return dep_set.union(dep) + + depends = add_dependencies(output_node.inputs, set()) + + avail = set() + funcs = [output_node.extract_function()] + n = output_node + while not depends.issubset(avail) and n.parent is not None: + n = n.parent + if len(depends.intersection(n.outputs.iteritems())) != 0: + avail.update(n.outputs.iteritems()) + depends = add_dependencies(n.inputs, depends) + funcs.append(n.extract_function()) + + if n is None: + raise KeyError('Could not find input dependencies: %s' % str(depends - avail)) + + funcs = funcs[::-1] + defs = "\n\n".join((f[0] for f in funcs)) + main = 'def main({}):'.format(','.join(args)) + for f in funcs: + main += '\n{}{}'.format(INDENT, f[1]) + main += '\n{}{}'.format(INDENT, main_closing_statement) + + code = '{}\n\n{}\n'.format(defs, main) + return code + + +def create_module(node, export_statement, input_dict): + """ + create a python module to execute a given node + """ + body = output_to_function(node, export_statement, input_dict.keys()) + + imports = '\n'.join([ + 'import json', + ]) + + deser = '' + for k, v in input_dict.iteritems(): + deser += "\n{} = json.loads('{}')".format(k, json.dumps(v)) + deser += '\nmain({})'.format(','.join(input_dict.iterkeys())) + + code = '{}\n\n{}\n\n{}\n'.format(imports, body, deser) + return code diff --git a/nodebook_demo.ipynb b/nodebook_demo.ipynb new file mode 100644 index 0000000..e4f6d02 --- /dev/null +++ b/nodebook_demo.ipynb @@ -0,0 +1,648 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Nodebook \n", + "Repeatable jupyter notebooks" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Notebook configuration\n", + "\n", + "`#pragma nodebook off` instructs the cell to run in the global environment, this is required for ipython magics" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true, + "node_exists": false, + "node_name": "A5DD3DDCC9474F27B9FF8E7E88B10652", + "node_parent": 0, + "run_control": { + "state": "n" + } + }, + "outputs": [], + "source": [ + "#pragma nodebook off\n", + "# (optional) inline matplotlib figures\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "node_exists": false, + "node_name": "0B7F6A76C7834459B0790F244E4F44C4" + }, + "outputs": [ + { + "data": { + "application/javascript": [ + "Jupyter.utils.load_extensions('nodebookext')" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#pragma nodebook off\n", + "# here we load the nodebook extension and initialize the nodebook\n", + "%load_ext nodebookext\n", + "%nodebook memory test_nb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Example 1\n", + "### Chain of execution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ordinary notebook behavior" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true, + "node_exists": false, + "node_name": "F1A1FED924194E9EA016979D3B23BEA5" + }, + "outputs": [], + "source": [ + "#pragma nodebook off\n", + "x = (10+4) % 10" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "node_exists": false, + "node_name": "FF84458F275649F389C48E8F41DEB536" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "34" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#pragma nodebook off\n", + "x" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true, + "node_exists": false, + "node_name": "2A50187EBDA5452A984EDF8E7F90501C" + }, + "outputs": [], + "source": [ + "#pragma nodebook off\n", + "x += 10" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "node_exists": false, + "node_name": "6E357C9D129E4E869A76889BD8EE8AFD" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "14" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#pragma nodebook off\n", + "x" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Nodebook behavior" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "node_exists": true, + "node_name": "EB910D7B09F2443081D464E1B36C9392", + "node_parent": 7, + "run_control": { + "state": "n" + } + }, + "outputs": [], + "source": [ + "x = 4" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "node_exists": true, + "node_name": "4B8AB43B29EC4524AE9DBBA8485F9B23", + "node_parent": 8, + "run_control": { + "state": "n" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "4" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "node_exists": true, + "node_name": "17C98397C28445D68406D9AA38FFFDB4", + "node_parent": 9, + "run_control": { + "state": "n" + } + }, + "outputs": [], + "source": [ + "x += 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "node_exists": true, + "node_name": "6E21FE582C4643F6BDA61DB040294A5C", + "node_parent": 10, + "run_control": { + "state": "n" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "14" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Example 2\n", + "### Inspecting intermediate data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "node_exists": true, + "node_name": "0E269658505D4EE5914DD1CD33236788" + }, + "outputs": [], + "source": [ + "# first we'll load some common imports\n", + "import pandas as pd\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "node_exists": true, + "node_name": "1846537E19A44D308AF5E23F0620B54D", + "node_parent": 16, + "run_control": { + "state": "n" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
x
0-9
1-8
2-7
3-6
4-5
5-4
6-3
7-2
8-1
90
101
112
123
134
145
156
167
178
189
\n", + "
" + ], + "text/plain": [ + " x\n", + "0 -9\n", + "1 -8\n", + "2 -7\n", + "3 -6\n", + "4 -5\n", + "5 -4\n", + "6 -3\n", + "7 -2\n", + "8 -1\n", + "9 0\n", + "10 1\n", + "11 2\n", + "12 3\n", + "13 4\n", + "14 5\n", + "15 6\n", + "16 7\n", + "17 8\n", + "18 9" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# define a dataframe\n", + "df = pd.DataFrame({'x':np.arange(-9,10)})\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "node_exists": true, + "node_name": "0423E3B5977F4518A3F9EF885313A3C5", + "node_parent": 17, + "run_control": { + "state": "n" + } + }, + "outputs": [], + "source": [ + "# transform the data\n", + "df **= 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "node_exists": true, + "node_name": "13F16C6E3B3D4E4687B5CDD12B06076E", + "node_parent": 19, + "run_control": { + "state": "n" + } + }, + "outputs": [], + "source": [ + "# additional transformation\n", + "df = -df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "node_exists": true, + "node_name": "44138EFE05A3418099F3E6FB697B2A28" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VNX5x/HPk50kLElIgGwkgQQIW4QQWV0ABRVJ3dG6\noxRLq6K1lZ/WtbV1qVqqVVFwrwqKirgjLiAiBGRL2EISSNgSEghk387vjww2xWgSZiZ3JvO8X695\nMXNn7r3fhMkzZ8699xwxxqCUUsqzeFkdQCmlVPvT4q+UUh5Ii79SSnkgLf5KKeWBtPgrpZQH0uKv\nlFIeSIu/Ukp5IC3+SinlgbT4K6WUB/KxOkBT3bt3N3FxcVbHUEopt7Ju3bpDxpjwtqzjUsU/Li6O\njIwMq2MopZRbEZHdbV1Hu32UUsoDafFXSikPpMVfKaU8kEv1+SullFVqa2spKCigqqrK6ig/KyAg\ngOjoaHx9fe3elhZ/pZQCCgoK6Ny5M3FxcYiI1XF+whhDcXExBQUFxMfH2709p3f7iMhkEdkuItki\ncqez96eUUiejqqqKsLAwlyz8ACJCWFiYw76ZOLX4i4g38DRwDpAMXC4iyc7cp1JKnSxXLfzHOTKf\ns7t90oBsY0wOgIi8CaQDWU7er1J2aWgwHDxWRX5JJXtKKjhQWklokD+xoYHEhHYislsnfL31fAnl\nvpxd/KOA/CaPC4BTm75ARGYAMwBiY2OdHEep/zpaVcue4goKDlewp6Tix0KfX1JBweFKauobfnZd\nL4HIbp2ICQn88QMhJvT4/UDCgvxcvhWpPJvlB3yNMfOAeQCpqak6m7xymLr6BgoOV5JvK+57Sioo\nOF7gD1dwpKL2f17ftZMvsaGB9O/VmbMG9mhS2APp1TWA4vIa8ksqfvyAOH5/+fZCio5V/8+2Av28\niQ0NJLrJh0Nskw+HAF/v9vxVKPUTzi7+e4GYJo+jbcuUcpqcojLeWpvPO+sLOFRW8+NyP28vokMa\nW+hDY7o2FuKQxmIcExpI106/fPpcVLdORHXrxMiEsJ88V1lT/+M3iKbfIgoOV7Bq1yEqaup/fG0n\nX2+mDOnFtLRYhsV2028ICoC1a9cyffp01qxZQ319PWlpabz11lsMGjTIKftzdvFfCySKSDyNRX8a\ncIWT96k8UFVtPZ9sOcAba/bwfW4J3l7ChP4RTEzuQe/QQGLDAunROQAvL+cU2k5+3iT26Exij84/\nec4YQ0l5zY8fDN/tKmbJxn0sWldAUo9gpo2I5cJhUXQL9HNKNtV293+QSda+ow7dZnJkF+49f+DP\nPj9ixAimTp3K3XffTWVlJVdeeaXTCj84ufgbY+pE5HfAp4A3sMAYk+nMfSrPsu3AUd5ck8+7P+yl\ntLKW2NBA7pjUj0uGRxPRJcDqeIDtFL1gf8KC/TklNoT0lCjunpLMBxv38eaaPTywNIu/f7KNyQN7\nMi0thlEJrnu6oXKue+65hxEjRhAQEMDcuXOdui+n9/kbYz4CPnL2fpTnKK+uY+mmfbyxJp8N+Ufw\n8/Zi0qCeXD4ihpEJYU5r3TtSsL8Pl6fFcnlaLFn7jvLW2j28+8NelmzcR1xYIJeNiOXi4dGEd/a3\nOqpH+qUWujMVFxdTVlZGbW0tVVVVBAUFOW1fYozrHGNNTU01OqSzao4xhk0Fpby5Np8lG/ZSXlNP\n34hgpo2I4cJh0YQGuX+XSVVtPR9t3s+ba/JZk1eCj5cwcUAPLkuL4bTEcLzd4EPNnW3dupUBAwZY\nmmHq1KlMmzaN3Nxc9u/fz1NPPfWT1zSXU0TWGWNS27Ivy8/2UeqXlFbW8v6GvbyxJp+t+48S4OvF\nlCGRXJ4Ww7DYkA7VPRLg682Fw6K5cFg02YVlLMzI5+11BXySeYCobp24JDWaS1NjiOzWyeqoygle\neeUVfH19ueKKK6ivr2f06NEsX76c8ePHO2V/2vJXLscYw9q8w7y5Zg8fbt5PdV0DAyO7MC0tlvSU\nSLoE2D+olbuoqWvg86yDvLl2Dyt2HsJL4PSkcKalxTK+f4ReaOZArtDybw1t+asO6fOsg/z9463s\nKion2N+Hi4dHc3laLIOiulodzRJ+Pl6cN6QX5w3pRX5JBQsz8lmYkc9vXl1HeGd/fntGH64eFadd\nQqrNtPgrl1BSXsN9SzJZsnEfST2CeeTiIUwZ0otAP32LHhcTGsjtZ/fjlgmJfLW9iBdX5XL/B1ks\n3bSfhy8aQt+IYKsjKjeif1nKUsYYPty8n3vfz+RoVS2zJyZx0xl98PPR7oyf4+PtxcTkHkwYEMG7\nP+zl/g+yOHfuCmZPTOLGcfH4aFfQSTPGuPRxJEd202vxV5YpPFrFn9/fwqeZBxka3ZVHLh5Jv54/\nvUhKNU9EuHBYNGMTu3PPe5k8/Mk2Ptq8n0cvGUL/nl2sjud2AgICKC4udtlhnY+P5x8Q4JjrV/SA\nr2p3xhgWr9/LA0uzqKyt5/azkpg+Vlus9vpo837+/N4WjlbVMuvMvvz2jL76DaoN3Hkmr5M54KvF\nX7WrfUcq+b93N/PV9iJSe4fw8MVD6BOufdWOUlJewwMfZPLehn3079mZRy8eyuBozzxY7km0+CuX\nZYzhjTX5PPTRVuobDH+a3I+rR8W5xdW47mhZ1kHuem8zh8pqmHFaArdMSNSRRDswPdVTuaQ9xRXc\nuXgTq3YVM7pPGH+/cAixYYFWx+rQJib3YER8KA99uJVnvtrFp5kHePTiIQzvHWp1NOUitOWvnKa+\nwfDyqjwe/XQ73l7CXecNYNqIGJc8mNaRrdhZxJ3vbGZfaSXXjY7nD5OS9BTaDkZb/splZBeW8ad3\nNrFu92HO7BfOXy8YrMMSWGRcYjifzj6NRz7ZxoJvc1m29SB/v2gwo/t0tzqaspCeCqAcqq6+gWe+\n2sW5c1eQXVjG45cOZcG1I7TwWyzY34cH0gfx1oyReAlc8fz33PXuZo5V1ba8suqQtOWvHGbbgaPc\nsWgTm/eWMnlgTx741UAiOrvGmPqq0akJYXx8y2k8/vl25q/M5ctthTx04WDO6BdhdTTVzuxq+YvI\nJSKSKSINIpJ6wnNzRCRbRLaLyCT7YipXZozhqeU7Of9fK9l3pJJ//3oYz141XAu/i+rk581d5yXz\nzk2jCfL34doX1/KHRRupbDLVpOr47G35bwEuBJ5rulBEkmmcsnEgEAksE5EkY4y+uzqYuvoG5ize\nzKJ1BZw/NJL7pw7sEGPre4JTYkNYevNY/vVFNk9/lc3u4nJeuGZEi3MZq47Brpa/MWarMWZ7M0+l\nA28aY6qNMblANpBmz76U66mqreem19ezaF0Bt05MZO60FC38bsbfx5s/TOrHU5cPY0P+ES577jsK\nj7ruFa7KcZx1wDcKyG/yuMC2THUQR6tquXrBGpZtPcgD6QO5dWKSnsLpxs4b0osXr01jT0kFFz27\nirxD5VZHUk7WYvEXkWUisqWZW7ojAojIDBHJEJGMoqIiR2xSOVnRsWqmPbea9bsP8+RlKVw9Ks7q\nSMoBxiZ2540bR1JWVcfFz35H5r5SqyMpJ2qx+BtjJhpjBjVze/8XVtsLxDR5HG1b1tz25xljUo0x\nqeHh4W1Lr9pdfkkFlzy7itxD5cy/dgTpKfqFriMZGtONRTNH4+ctTHtuNd/nFFsdSTmJs7p9lgDT\nRMRfROKBRGCNk/al2sm2A0e56JlVHK6o5fUbT+X0JP2w7oj6RgTz9k2jiejiz9UL1vB51kGrIykn\nsPdUzwtEpAAYBXwoIp8CGGMygYVAFvAJMEvP9HFva/NKuPTZ7/ASYdHMUQyLDbE6knKiyG6dWDRz\nNP17dWHma+tYlJHf8krKrejYPqpFy7cd5KbX1hPVrROvTE8jOkQHZfMU5dV1zHxtHSt2HmLOOf35\nzel9rI6kmnEyY/vo8A7qFy1eX8CNr6wjqUdnFs0cpYXfwwT5+/DCNamcN6QXf/t4G3/7eKtDpxJU\n1tHhHdTPmr8ylweXZjG6Txjzrk4l2F/fLp7I38ebudNOISTQl+e+zuFweQ0PXTBYZ15zc/rXrH7C\nGMNjn23n6S93MXlgT56clqITgXg4by/hwfRBhAb5M/eLnRypqGXu5afo+8KN6Ue3+h/1DYb/e3cL\nT3+5i8vTYnj618P0D1wBjRPG33ZWEvedn8xnWQe5ZsEajuqooG5Li7/6UXVdPb/7z3reWLOHWWf2\n4aELBuOt0yyqE1w7Jp5/Tkth3e7DTHtuNUXHqq2OpE6CFn8FQFl1Hde9uJaPtxzg7vMGcMek/jpc\ng/pZ6SlRPH9NKjmHyrjk2VXkl1RYHUm1kRZ/RXFZNZfPW833uSX845Kh3DAuwepIyg2c2S+C128Y\nyeGKWi56ZhXbDhy1OpJqAy3+Hq7gcAWXPPsdOw4eY95Vw7loeLTVkZQbGd47hEUzRyEClz77HRl5\nJVZHUq2kxd+DZReWcfEz31FUVs1rN5zKhAE9rI6k3FBSj868PXM0YcH+XDn/e77cXmh1JNUKWvw9\nVNGxaq5ZsIa6BsPC34xiRFyo1ZGUG4sJDWTRzFH0CQ/mptfWsangiNWRVAu0+Hugqtp6ZryaQXF5\nNS9eO4IBvbpYHUl1AN2D/XnpujTCgvyZ/nIG+45UWh1J/QIt/h7GGMMdb2/ihz1HePKyFAZHd7U6\nkupAwjv7s+DaEVTW1HPDyxmUV9dZHUn9DC3+HuaJZTv5YOM+/jS5P5MH9bI6juqA+vXszFNXnMK2\nA0e55c0fqG/QsYBckRZ/D/LeD3uZ+8VOLhkezczT9XRO5Txn9Ivg3vMHsmxrIX/7aKvVcVQzdGwf\nD5GRV8If397EqfGh/PWCwXoBl3K6a0bHsauojBdW5pIQHswVp8ZaHUk1Ye9kLo+KyDYR2SQi74pI\ntybPzRGRbBHZLiKT7I+qTtae4gpmvLqOqJBOPHvlcPx89Aufah/3TEnm9KRw7nl/C99mH7I6jmrC\n3irwOTDIGDME2AHMARCRZGAaMBCYDPxbRHR0MAscrapl+strqW8wzL8mlZAgP6sjKQ/i4+3Fv644\nhYTwIGa+to7swjKrIykbu4q/MeYzY8zxw/mraZyoHSAdeNMYU22MyQWygTR79qXarq6+gVmvryf3\nUDnPXDmMhPBgqyMpD9QlwJf514zA38eL619aS0l5jdWRFI494Hs98LHtfhTQdNLPAtsy1U6MMdz3\nQSYrdh7ioQsGM7pPd6sjKQ8WExrIc1elcuBoFTNfXUd1nU7pbbUWi7+ILBORLc3c0pu85i6gDni9\nrQFEZIaIZIhIRlFRUVtXVz/jxW/zeG31Hn5zegKXjoixOo5SDO8dwmOXDGVNXglzFm/W6SAt1uLZ\nPsaYib/0vIhcC0wBJpj//m/uBZpWnGjbsua2Pw+YB40TuLccWbVk+baD/OXDLM5O7sGfJvW3Oo5S\nP5o6NJKcojKeXLaTPuHBzDqzr9WRPJa9Z/tMBv4ITDXGNB3QewkwTUT8RSQeSATW2LMv1Tpb9x/l\n9//5geTILjw5LQUvnYxFuZhbJiSSnhLJo59u58NN+62O47HsPc//KcAf+Nx23vhqY8xMY0ymiCwE\nsmjsDppljNFOPicrPFbF9JfWEhzgwwtXjyDQTy/jUK5HRHj4oiEUHK7ktoUbiA7pxNCYbi2vqBxK\nXKnfLTU11WRkZFgdwy1V1dZz2bzV7DhwjEUzRzEoSsfsUa7tUFk1v3r6W6rrGnhv1hiiunWyOpLb\nEpF1xpjUtqyjV/t0AA0NhtsXbmRTwRGenJaihV+5he7BjYPAVdXUM/2ltZTpIHDtSot/B/DEsh18\nuHk/d07uz6SBPa2Oo1SrJfXozFO/HsbOwjJufkMHgWtPWvzd3OL1BfxreTaXpcYw4zQdrE25n9OT\nwrnv/GSWbyvkIR0Ert3oEUE3tjavhDvf2cyohDAe/NUgHaxNua2rRsWxq6ic+StzSQgP4ten9rY6\nUoenxd9N7S4uZ8YrGUTrYG2qg/jzlGR2F5dzz/uZxIYGMi4x3OpIHZpWDDdUWlnL9S+txQALrh1B\n10BfqyMpZTdvL+FfVwwjMSKY376+nuzCY1ZH6tC0+LuZ44O17Smp4NkrhxPXPcjqSEo5TLC/Dy9c\nk4q/jzfX6SBwTqXF380889UuVmYf4q+/GszIhDCr4yjlcNEhgTx/9XAOllYzZ/EmHQPISbT4u5HN\nBaX884udTB0aqYO1qQ7tlNgQ/jApiU8zD/LO+maHBVN20uLvJqpq67n1rR/oHuzPg+mDrI6jlNNN\nH5tAWnwo9y3JJL+kouUVVJto8XcTD3+yjV1F5Tx6yRA9wKs8greX8I9LhgJw+6KNegGYg2nxdwMr\ndx7ixW/zuHZ0nJ7+pjxKTGgg956fzJrcEuavzLE6Toeixd/FlVbWcsfbG+kTHsSfJuvY/MrzXDw8\nmkkDe/DYpzvYduCo1XE6DC3+Lu7e97dQdKyaJy5LoZOft9VxlGp3IsJDFwymSydfZr+1UaeAdBAt\n/i5s6aZ9vLdhH78fn8iQaB3vXHmusGB/Hr5oMFv3H+WJz3daHadD0OLvog6UVnHXu1sYGtONWWf2\nsTqOUpabMKAHl6fF8Nw3u1ibV2J1HLdn7zSOD4rIJhHZICKfiUikbbmIyFwRybY9P8wxcT2DMYY/\nvrOJ6rp6nrh0KD7e+hmtFMDd5yUTExLIbQs36Pj/drK3qjxqjBlijEkBlgL32JafQ+O8vYnADOAZ\nO/fjUV5bvZtvdhRx17kDSAgPtjqOUi4jyN+Hxy8dyt7DlTz4QZbVcdyaXcXfGNP00HsQcPxE3HTg\nFdNoNdBNRHrZsy9PkVNUxl8/2sppSeFcOVKHtVXqRKlxocw8vQ9vZeTzedZBq+O4Lbv7E0TkryKS\nD/ya/7b8o4D8Ji8rsC1rbv0ZIpIhIhlFRUX2xnFrdfUNzF64kQBfbx69eIiOz6/Uz7h1YhLJvbpw\n5zubOFRWbXUct9Ri8ReRZSKypZlbOoAx5i5jTAzwOvC7tgYwxswzxqQaY1LDwz37Aqanv9zFxvwj\n/OVXg+jRJcDqOEq5LD8fL564LIVjVXXMWbxZB387CS0Wf2PMRGPMoGZu75/w0teBi2z39wJNRx6L\nti1TP2Nj/hHmLt/Jr1IimTIk0uo4Srm8fj0788fJ/fg86yCL1hVYHcft2Hu2T2KTh+nANtv9JcDV\ntrN+RgKlxpj99uyrI6usqWf2wg1EdPbnfh20TalWu35MPCMTQrlfB39rM3v7/P9u6wLaBJwN3GJb\n/hGQA2QDzwO/tXM/HdrDn2wjp6icxy4ZStdOOmibUq3l5SU8dslQvES4faEO/tYWds3ha4y56GeW\nG2CWPdv2FCt2FvHSqjyuGxPHmL7drY6jlNuJDgnkvqkDuX3RRp5fkcPM0/WiyNbQq4csVFpRyx2L\nNtE3IlgHbVPKDhcOi2LywJ48/tkOtu7Xwd9aQ4u/hf78/hYOlVXzxKUpBPjqoG1KnSwR4aELjw/+\ntkEHf2sFLf4WWbJxH0s27uOWCYkMju5qdRyl3F5okB+PXDyYbQeO8fhnO6yO4/K0+FvgQGkVd7+7\nmVNiu3HTGdo/qZSjjO/fg8vTYpm3Iofvc4qtjuPStPi3s4YGwx1vb6S23vD4pSk6aJtSDnb3eQOI\nDQ3ktoUbOVZVa3Ucl6WVp529uno3K3Ye4q7zBhDfPcjqOEp1OI2Dv6Wwv7SSB3Twt5+lxb8dZReW\n8bePt3JGv3B+fWqs1XGU6rCG9w7hpjP6sGhdAZ9mHrA6jkvS4t9OausbuG3hBgJ8vXnkIh20TSln\nu2VCEgMjuzBn8WaKjungbyfS4t9Onv4ym00FpTx0wWAidNA2pZzOz8eLJy9LoaxaB39rjhb/drC7\nuJx/f7mLqUMjOXewTmugVHtJ7NGZP5ydxLKtB/lia6HVcVyKFv928ODSrfh6C3edN8DqKEp5nOvG\nxNM3IpgHlmZRVasXfx2nxd/JvtpeyLKtB/n9hEQdo18pC/h6e3Hv+cnsKalg/spcq+O4DC3+TlRT\n18ADH2QR3z2I68bEWR1HKY81LjGcSQN78NTybPaXVlodxyVo8Xeil1blknOonHumJOPvo2P3KGWl\nu89Lpt4Y/vbRtpZf7AG0+DtJ4dEq/rlsJ+P7R3Bm/wir4yjl8WJCA5l5WgJLNu5jTW6J1XEs55Di\nLyK3i4gRke62xyIic0UkW0Q2icgwR+zHnTz8yXZq6w1/npJsdRSllM1NZ/QlsmsA9y7J9PiJX+wu\n/iISQ+MsXnuaLD4HSLTdZgDP2Lsfd7J+z2HeWV/A9HHxOoSDUi6kk583d52XzNb9R3ljzZ6WV+jA\nHNHyfwL4I9D0YzQdeMU0Wg10ExGPOMG9ocFw35JMenTx53dn9rU6jlLqBOcO7snIhFAe+2w7Rypq\nrI5jGXsncE8H9hpjNp7wVBSQ3+RxgW1Zh/f2ugI2FZQy55wBBPnbNUumUsoJRIT7pg7kaGUtj3/u\nueP+t1idRGQZ0LOZp+4C/o/GLp+TJiIzaOwaIjbWvQc7K62s5eFPtjG8dwjpKZFWx1FK/Yz+Pbtw\n1cjevLp6N5enxTKgVxerI7W7Flv+xpiJxphBJ96AHCAe2CgieUA0sF5EegJ7gZgmm4m2LWtu+/OM\nManGmNTw8HB7fx5Lzf1iJyUVNdw/daAO3KaUi5t9VhJdO/ly35JMjxz356S7fYwxm40xEcaYOGNM\nHI1dO8OMMQeAJcDVtrN+RgKlxpj9jonsmnYePMbLq/KYNiKWQVE6LaNSrq5boB9/mNSP73NL+HBz\nhy5PzXLWef4f0fjNIBt4Hvitk/bjEowx3P9BFoF+3vzh7CSr4yilWmnaiFiSe3XhoQ+3UlFTZ3Wc\nduWw4m/7BnDIdt8YY2YZY/oYYwYbYzIctR9X9FnWQVZmH+K2s5IIC/a3Oo5SqpW8vYT70weyr7SK\nZ7/aZXWcdqVX+NqpqraeB5dmkdQjmCtH9rY6jlKqjUbEhZKeEsmz3+Swp7jC6jjtRou/neZ9k0PB\n4UrumzpQJ2NXyk3NOWcAPl7CXz70nDl/tVrZYe+RSv79VTbnDe7F6D7drY6jlDpJPbsG8Lvxffks\n6yDf7CiyOk670OJvh4c+2grAnHP7W5xEKWWv6WPj6R0WyP0fZFJb32B1HKfT4n+SvttVzIeb9nPT\n6X2JDgm0Oo5Syk7+Pt7cMyWZXUXlvLwqz+o4TqfF/yTU1Tdw/weZRHXrxG9OT7A6jlLKQcb3j+CM\nfuH8c9lOio5VWx3HqbT4n4T/rNnDtgPH+POUAQT46iQtSnUUIsKfpyRTVVfPo5927ElftPi3UUl5\nDf/4bAdj+oYxaWBzQx4ppdxZn/Bgrh8Tz8KMAjbkH7E6jtNo8W+jf3y2nbLqOu49X8fvUaqj+t34\nvoR39ue+JZk0dNBJX7T4t8GWvaX8Z80erh7Vm6Qena2Oo5Ryks4Bvtw5uT8b8o+w+Idmx6R0e1r8\nW6lx/J5MQgL9uHWijt+jVEd3wSlRnBLbjb9/vI1jVbVWx3E4Lf6ttGTjPtbmHeaPk/rRtZOv1XGU\nUk7m5SXcd/5Aisur+dfybKvjOJwW/1Yor67jbx9tY3BUVy5JjWl5BaVUhzA0phuXDo9hwcpcsgvL\nrI7jUFr8W+HpL7M5cLSK+6YOxNtLD/Iq5UnumNyPTn7ePLA0q0NN+qLFvwV5h8p5YUUuFw6LYnjv\nEKvjKKXaWfdgf2ZPTOKbHUUs21podRyH0eLfgr98mIWvt3DnZB2/RylPddWo3iRGBPPg0iyqauut\njuMQdhV/EblPRPaKyAbb7dwmz80RkWwR2S4ik+yP2v5W7Gz8pL95QiIRXQKsjqOUsoivtxf3nj+Q\nPSUVLPg21+o4DuGIlv8TxpgU2+0jABFJBqYBA4HJwL9FxK3GQTDG8NhnO4jq1olrx8RZHUcpZbGx\nid0Z3z+C577O6RCnfjqr2ycdeNMYU22MyaVxLt80J+3LKb7cXsjG/CPcPKEv/j5u9bmllHKS285K\norSylhe/zbM6it0cUfx/JyKbRGSBiBw/IhoF5Dd5TYFt2U+IyAwRyRCRjKIi15hEwRjD45/vIDY0\nkAuHRVsdRynlIgZFdeXs5B48vyKH0kr3bv23WPxFZJmIbGnmlg48A/QBUoD9wD/aGsAYM88Yk2qM\nSQ0PD2/zD+AMn2cdZMveo9w8IRFfnZpRKdXErROTOFZVx/wVOVZHsYtPSy8wxkxszYZE5Hlgqe3h\nXqDp1VDRtmUur6HB8MSyncR3D+JXKZFWx1FKuZjkyC6cO7gnC77N47ox8YQE+Vkd6aTYe7ZPryYP\nLwC22O4vAaaJiL+IxAOJwBp79tVePsk8wNb9R7llQqJOyK6UatYtE5Ior6njeTdu/dtb3R4Rkc0i\nsgk4E5gNYIzJBBYCWcAnwCxjjMufHFvfYHji8x30CQ/i/KHa6ldKNa9fz85MGRLJS6vyKC5zzxm/\n7Cr+xpirjDGDjTFDjDFTjTH7mzz3V2NMH2NMP2PMx/ZHdb4PN+9nZ2EZt05M0mEclFK/6JYJiVTV\n1jPvG/ds/Wu/hk19g+HJZTtI6hHMeYN7tbyCUsqj9Y0IJj0lipe/y3PL+X61+Nss2biXnKJyZk9M\nwktb/UqpVrh5QiK19YZnv95ldZQ20+IP1NU38M9lOxnQq4vOy6uUarX47kFccEoUr63ezcGjVVbH\naRMt/sC7P+wlr7iC2RMTtdWvlGqTm8cnUt9geOYr92r9e3zxr61vYO7ynQyO6spZyT2sjqOUcjOx\nYYFckhrNf77fw/7SSqvjtJrHF/931hWQX1LJbWclIaKtfqVU2806sy8Gw1NuNN2jRxf/mroG/rU8\nm5SYbpzRzzWGllBKuZ/okEAuGxHDwox88ksqrI7TKh5d/Bdm5LP3iLb6lVL2m3VmXwTh6S/do/Xv\nscW/qraep7/MJrV3COMSu1sdRynl5np17cQVp8ayaF0Bu4vLrY7TIo8t/m+tzWd/aZW2+pVSDnPT\nGX3w8RL+5QZ9/x5Z/I+3+k+ND2VUnzCr4yilOogeXQK4cmRvFq8vIPeQa7f+PbL4v7Z6N4XHqpmt\nrX6llINOkRuSAAAQG0lEQVTNPL0Pfj5ezP1ip9VRfpHHFf+Kmjqe/XoXY/qGMTJBW/1KKccK7+zP\nNaPieH/DXrILj1kd52d5XPF/9bvdHCqrYfbEJKujKKU6qBmnJRDg680/v3Ddvn+PKv5l1Y2t/tOS\nwkmNC7U6jlKqgwoL9ufa0XEs3bSP7Qdcs/Vvd/EXkd+LyDYRyRSRR5osnyMi2SKyXUQm2bsfR3h5\nVR6HK2qZPTHR6ihKqQ7uxnEJBPn58OSyHVZHaVaLc/j+EhE5E0gHhhpjqkUkwrY8GZgGDAQigWUi\nkmTlbF7HqmqZ900O4/tHcEpsiFUxlFIeIiTIj+vHxDF3eTaZ+0oZGNnV6kj/w96W/03A340x1QDG\nmELb8nTgTWNMtTEmF8gG0uzcl11e/DaP0spabjtL+/qVUu1j+rgEOgf48OQy1zvzx97inwSME5Hv\nReRrERlhWx4F5Dd5XYFtmSVKK2t5fkUOZyf3YFCUa336KqU6rq6dfLlxXAKfZx1kc0Gp1XH+R4vF\nX0SWiciWZm7pNHYbhQIjgTuAhdLGE+dFZIaIZIhIRlFR0Un9EC2ZvzKXY1V13Kpn+Cil2tl1Y+Lo\n2smXJ1ys77/F4m+MmWiMGdTM7X0aW/SLTaM1QAPQHdgLxDTZTLRtWXPbn2eMSTXGpIaHO35kzSMV\nNSxYmcu5g3uSHNnF4dtXSqlf0jnAlxmnJbB8WyE/7DlsdZwf2dvt8x5wJoCIJAF+wCFgCTBNRPxF\nJB5IBNbYua+T8vyKHMpr6rhlgrb6lVLWuGZ0HKFBfjzhQn3/9hb/BUCCiGwB3gSusX0LyAQWAlnA\nJ8AsK870KSmv4cVv85gyJJJ+PTu39+6VUgqAYH8ffnNaAt/sKCIjr8TqOICdxd8YU2OMudLWDTTM\nGLO8yXN/Ncb0Mcb0M8Z8bH/Utnvum11U1dZzywQ9r18pZa2rRvWme7Cfy/T9d9grfIuOVfPKqt2k\np0TRNyLY6jhKKQ8X6OfDzNP78G12Matziq2O03GL/7Nf76KmvoGbtdWvlHIRV47sTXhnfx7/fAfG\nGEuzdMjif/BoFa+t3s0Fp0QR3z3I6jhKKQVAgK83s87ow5rcElbtsrb13yGL/zNf7aKuwXDzeG31\nK6Vcy7S0WHp2CbC89d/hiv+hsmr+s2YPFw+LJjYs0Oo4Sin1PwJ8vZk1vi/rdh/mOwv7/jtc8X/l\nu93U1jfwm9MTrI6ilFLNumR4NGFBfrywIteyDB2q+FfV1vPa6t1M6N+DhHA9w0cp5ZoCfL25alRv\nlm8rtGy2rw5V/N9ZX0BJeQ03jIu3OopSSv2iK0f2xs/Hi/kr8yzZf4cp/g0Nhvkrcxkc1ZVT43WW\nLqWUa+se7M9Fw6JYvL6A4rLqdt9/hyn+X24vJKeonBvGxdPGgUWVUsoS08cmUF3XwKurd7f7vjtM\n8X9+RQ6RXQM4d3Avq6MopVSr9I0IZnz/CF79bjdVte07/FmHKP5b9payOqeEa8fE4evdIX4kpZSH\nuGFsPMXlNbz7Q7Oj3jtNh6iUL6zIIdjfh2lpsVZHUUqpNhnVJ4zkXl2YvzKXhob2u+jL7Yv//tJK\nlm7az2UjYugS4Gt1HKWUahMR4cbT4skuLOPrHc6ZzbA5bl/8X/o2jwZjuG5MnNVRlFLqpJw3OJKe\nXQJ4fkVOu+3TruIvIm+JyAbbLU9ENjR5bo6IZIvIdhGZZH/UnyqrruM/a/ZwzuBeRIfoUA5KKffk\n5+PFNaPjWLWrmMx97TPRu72TuVxmjEkxxqQA7wCLAUQkGZgGDAQmA/8WEW97w57orbX5HKuq48Zx\nOpSDUsq9XZEWS6CfN/PbacgHh3T7SOOJ9ZcCb9gWpQNvGmOqjTG5QDaQ5oh9HVdX38CL3+YyIi6E\nlJhujty0Ukq1u66BvlyaGsOSjfs4UFrl9P05qs9/HHDQGHN8duIoIL/J8wW2ZQ7zaeZBCg5XcoO2\n+pVSHcT0sfE0GMNLq/Kcvq8Wi7+ILBORLc3c0pu87HL+2+pvExGZISIZIpJRVNS6I93GGJ5fkUNc\nWCATB/Q4md0qpZTLiQkNZPKgnvzn+92UV9c5dV8tFn9jzETbBO0n3t4HEBEf4ELgrSar7QVimjyO\nti1rbvvzjDGpxpjU8PDwVoVet/swG/KPcP3YeLy9dCgHpVTHMX1sAker6liUkd/yi+3giG6ficA2\nY0xBk2VLgGki4i8i8UAisMYB+wIah3Lo2smXi4dHO2qTSinlEob3DmFYbDcWfJtHvRMv+nJE8Z/G\nCV0+xphMYCGQBXwCzDLGOGTgit3F5XyWdZArR8YS6OfjiE0qpZRLuXFcAntKKvgs84DT9mF38TfG\nXGuMebaZ5X81xvQxxvQzxnxs736OW7AyFx8v4ZpRcY7apFJKuZSzB/YkJrSTUy/6cqsrfI9U1LAw\no4CpQ6OI6BJgdRyllHIKby/h+jHxrN9zhHW7DztlH25V/F//fg+VtfU6U5dSqsO7NDWGLgE+zF/p\nnNa/2xT/mroGXl6Vx7jE7gzo1cXqOEop5VRB/j5ccWpvPtlygPySCodv322K/wcb91F4rFov6lJK\neYxrR8fhJcL8lY4f8sEtiv/xi7qSegRzWmJ3q+MopVS76Nk1gKlDI1mYkU9pRa1Dt+0Wxf/b7GK2\nHTjGDWMTdH5epZRHmT4unoqaet5Yu8eh23WL4v/8ihy6B/uTfkqk1VGUUqpdDYzsyug+Ybz0bR41\ndQ0O267LF/8dB4/x9Y4irhnVG38fh48KrZRSLu/GcQkcOFrFh5v3OWybLl/856/IJcDXi1+P7G11\nFKWUssTpSeH0jQjm+W9yMcYxQz64dPEvOlbNuz/s5aJh0YQG+VkdRymlLOHlJUwfG0/W/qN8l1Ps\nmG06ZCtO8up3edQ2NDB9rF7UpZTybBecEkVYkB8vOGimL5ct/pU19by6ejcT+vcgITzY6jhKKWWp\nAF9vrhrVm+XbCskuPGb39ly2+C/+oYDDFbXcqEM5KKUUAFeN7I2/j5dDLvpyyeLf0GCYvyKXIdFd\nSYsPtTqOUkq5hLBgfy4cFs076/dSXFZt17Zcsvgv31ZIzqFypo+N14u6lFKqielj46mpa+DV1bvt\n2o5LFv/nV+QQ2TWAcwf3sjqKUkq5lL4RwYzvH8Gr3+2mqvbk58iyq/iLSIqIrBaRDbZJ2NNsy0VE\n5opItohsEpFhrd3m5oJSvs8t4box8fh6u+Rnk1JKWeqGcfEUl9fw7g/NTo3eKvZW10eA+40xKcA9\ntscA59A4b28iMAN4prUbfGFlDsH+PlyWFtPyi5VSygONSghjYGQXXliRQ8NJzvNrb/E3wPHB9bsC\nx689TgdeMY1WA91EpMU+nNr6BpZu2s9lI2LoEuBrZzSllOqYRIQbxsWzq6icr3cUndQ27C3+twKP\nikg+8Bgwx7Y8Cshv8roC27KfEJEZti6jjPyiIwBcNybOzlhKKdWxTRkSSc8uASc9z2+LxV9ElonI\nlmZu6cBNwGxjTAwwG5jf1gDGmHnGmFRjTGplgw/nDOpJdEhg238SpZTyIL7eXlw7Jo5Vu05uuAef\nll5gjJn4c8+JyCvALbaHi4AXbPf3Ak077aNty35RgzE6U5dSSrXS5WmxzP1i50mta2+3zz7gdNv9\n8cDxFEuAq21n/YwESo0x+1vaWLdAX1JiutkZSSmlPEPXTr7ceJIN5hZb/i24EfiniPgAVTSe2QPw\nEXAukA1UANe1ZmMx2t2jlFJtMvusJG47ifXsKv7GmJXA8GaWG2CWPdtWSinlPHoVlVJKeSAt/kop\n5YG0+CullAfS4q+UUh5Ii79SSnkgLf5KKeWBtPgrpZQHksZT8l2DiBwDtlud4wTdgUNWh2iGK+bS\nTK2jmVrPFXO5YqZ+xpjObVnB3it8HW27MSbV6hBNiUiGq2UC18ylmVpHM7WeK+Zy1UxtXUe7fZRS\nygNp8VdKKQ/kasV/ntUBmuGKmcA1c2mm1tFMreeKuTpEJpc64KuUUqp9uFrLXymlVDtwmeIvIpNF\nZLuIZIvInS6QJ0ZEvhSRLBHJFJFbWl6rfYiIt4j8ICJLrc4CICLdRORtEdkmIltFZJTVmQBEZLbt\n/26LiLwhIgEWZFggIoUisqXJslAR+VxEdtr+DXGBTI/a/v82ici7ItKusyo1l6nJc7eLiBGR7u2Z\n6Zdyicjvbb+vTBF5xOpMIpIiIqtFZINtTvS0lrbjEsVfRLyBp4FzgGTgchFJtjYVdcDtxphkYCQw\nywUyHXcLsNXqEE38E/jEGNMfGIoLZBORKOBmINUYMwjwBqZZEOUlYPIJy+4EvjDGJAJf2B5bnelz\nYJAxZgiwA5jjApkQkRjgbGBPO+c57iVOyCUiZwLpwFBjzEDgMaszAY8A9xtjUoB7bI9/kUsUfyAN\nyDbG5BhjaoA3afzlWsYYs98Ys952/xiNBS3KykwAIhINnMd/50u2lIh0BU4D5gMYY2qMMUesTfUj\nH6CTbaa5QBqnHW1XxphvgJITFqcDL9vuvwz8yupMxpjPjDF1toeraZx329JMNk8AfwQsOTj5M7lu\nAv5ujKm2vabQBTIZoIvtflda8V53leIfBeQ3eVyACxTa40QkDjgF+N7aJAA8SeMfQ4PVQWzigSLg\nRVtX1AsiEmR1KGPMXhpbZHuA/TTOI/2Ztal+1KPJnNYHgB5WhmnG9cDHVocQkXRgrzFmo9VZTpAE\njBOR70XkaxEZYXUg4FbgURHJp/F93+I3N1cp/i5LRIKBd4BbjTFHLc4yBSg0xqyzMscJfIBhwDPG\nmFOActq/G+MnbP3o6TR+OEUCQSJypbWpfso25anLnHInInfR2OX5usU5AoH/o7ELw9X4AKE0dgff\nASwUEbE2EjcBs40xMcBsbN/Ef4mrFP+9QEyTx9G2ZZYSEV8aC//rxpjFVucBxgBTRSSPxq6x8SLy\nmrWRKAAKjDHHvxW9TeOHgdUmArnGmCJjTC2wGBhtcabjDopILwDbv+3abfBzRORaYArwa2P9OeB9\naPzg3mh7v0cD60Wkp6WpGhUAi02jNTR+C2/3g9EnuIbG9zjAIhq70n+RqxT/tUCiiMSLiB+NB+aW\nWBnI9kk+H9hqjHncyizHGWPmGGOijTFxNP6OlhtjLG3NGmMOAPki0s+2aAKQZWGk4/YAI0Uk0PZ/\nOQEXOBBts4TGP1Zs/75vYRag8Ww7GrsTpxpjKqzOY4zZbIyJMMbE2d7vBcAw2/vNau8BZwKISBLg\nh/UDve0DTrfdHw/sbHENY4xL3IBzaTzLYBdwlwvkGUvj1/FNwAbb7VyrczXJdwaw1OoctiwpQIbt\nd/UeEGJ1Jluu+4FtwBbgVcDfggxv0HjMoZbGAjYdCKPxLJ+dwDIg1AUyZdN43O34e/1ZqzOd8Hwe\n0N1F/v/8gNds76v1wHgXyDQWWAdspPHY5PCWtqNX+CqllAdylW4fpZRS7UiLv1JKeSAt/kop5YG0\n+CullAfS4q+UUh5Ii79SSnkgLf5KKeWBtPgrpZQH+n9oYFs+gksERwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# inspect the data\n", + "# try moving this cell up!\n", + "df.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "node_exists": true, + "node_name": "DCFFAB46E4064FAD88DE491CD466A0F2", + "node_parent": 21, + "run_control": { + "state": "n" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yz
001
111
221
331
441
551
661
771
881
991
\n", + "
" + ], + "text/plain": [ + " y z\n", + "0 0 1\n", + "1 1 1\n", + "2 2 1\n", + "3 3 1\n", + "4 4 1\n", + "5 5 1\n", + "6 6 1\n", + "7 7 1\n", + "8 8 1\n", + "9 9 1" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# overwrite df with something else\n", + "df = pd.DataFrame({'y':np.arange(10),'z':1})\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "node_exists": false, + "node_name": "174D7423CAC546B3881D1DE8A50BC0A4" + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b7e4789 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test=pytest diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ac9d916 --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +from setuptools import setup, find_packages +import os +import sys + +# we don't support wheels because they won't let us install the nbextension (a bit hacky) +if 'bdist_wheel' in sys.argv: + raise RuntimeError("Nodebook does not support wheels, coercing to pip fallback behavior (non-fatal error)") + +setup( + name='nodebook', + version='0.1.0', + author='Kevin Zielnicki', + author_email='kzielnicki@stitchfix.com', + license='Stitch Fix 2017', + description='Nodebook Jupyter Extension', + packages=find_packages(), + long_description='Nodebook Jupyter Extension', + url='https://github.com/stitchfix/nodebook', + install_requires=[ + 'ipython<=5.3', # newer versions of ipython do not support 2.7 + 'jupyter', + 'click', + 'dill', + 'msgpack-python', + 'pandas', + 'pytest-runner', + ], + tests_require=['pytest'], + data_files=[ + (os.path.expanduser('~/.ipython/nbextensions'), ['ipython/nbextensions/nodebookext.js']), + (os.path.expanduser('~/.ipython/extensions'), ['ipython/extensions/nodebookext.py']), + ], +) diff --git a/tests/test_nodebookcore.py b/tests/test_nodebookcore.py new file mode 100644 index 0000000..dcb2579 --- /dev/null +++ b/tests/test_nodebookcore.py @@ -0,0 +1,100 @@ +import pandas as pd +import pytest +from nodebook.nodebookcore import ReferenceFinder, Nodebook, Node +from nodebook.pickledict import PickleDict +import ast + + +class TestReferenceFinder(object): + @pytest.fixture() + def rf(self): + return ReferenceFinder() + + def test_assign(self, rf): + code_tree = ast.parse("x = 3") + rf.visit(code_tree) + assert rf.inputs == set() + assert rf.imports == set() + assert rf.locals == {'x'} + + def test_augassign(self, rf): + code_tree = ast.parse("x += 3") + rf.visit(code_tree) + assert rf.inputs == {'x'} + assert rf.imports == set() + assert rf.locals == {'x'} + + def test_import(self, rf): + code_tree = ast.parse("import numpy as np") + rf.visit(code_tree) + assert rf.inputs == set() + assert rf.imports == {'numpy'} + assert rf.locals == {'np'} + + def test_multiline(self, rf): + code_tree = ast.parse( + "import pandas as pd\n" + "y = pd.Series(x)\n" + ) + rf.visit(code_tree) + assert rf.inputs == {'x'} + assert rf.locals == {'pd', 'y'} + assert rf.imports == {'pandas'} + + +class TestNodebook(object): + @pytest.fixture() + def nb(self): + var_store = PickleDict() + return Nodebook(var_store) + + def test_single_node(self, nb): + node_id = '111' + nb.insert_node_after(node_id, None) + nb.update_code(node_id, "x = 42\nx") + res, objs = nb.run_node(node_id) + assert res == 42 + assert objs == {'x': 42} + + def test_node_chain(self, nb): + # first node sets x to 42 + node_id1 = '111' + nb.insert_node_after(node_id1, None) + nb.update_code(node_id1, "x = 42") + res, objs = nb.run_node(node_id1) + assert res == None + assert objs == {'x': 42} + + # second node increments x to 52 + node_id2 = '222' + nb.insert_node_after(node_id2, node_id1) + nb.update_code(node_id2, "x += 10") + res, objs = nb.run_node(node_id2) + assert res == None + assert objs == {'x': 52} + + # running second node again should give same results + res, objs = nb.run_node(node_id2) + assert res == None + assert objs == {'x': 52} + + # third node evaluates to x + node_id3 = '333' + nb.insert_node_after(node_id3, node_id2) + nb.update_code(node_id3, "x") + res, objs = nb.run_node(node_id3) + assert res == 52 + assert objs == {} + + # fourth node inserted between first and second node, changing x to 1 + node_id4 = '444' + nb.insert_node_after(node_id4, node_id1) + nb.update_code(node_id4, "x=1") + res, objs = nb.run_node(node_id4) + assert res == None + assert objs == {'x': 1} + + # re-running third node should now evaluate to 11 + res, objs = nb.run_node(node_id3) + assert res == 11 + assert objs == {} diff --git a/tests/test_pickledict.py b/tests/test_pickledict.py new file mode 100644 index 0000000..b6d773c --- /dev/null +++ b/tests/test_pickledict.py @@ -0,0 +1,35 @@ +import pandas as pd +import pytest +from nodebook.pickledict import PickleDict + + +@pytest.fixture(params=[None, 'tmpdir'], ids=['mode_memory', 'mode_disk']) +def mydict(request, tmpdir): + if request.param == 'tmpdir': + persist_path = tmpdir.strpath + print persist_path + else: + persist_path = None + return PickleDict(persist_path=persist_path) + + +class TestPickleDict(object): + def test_int(self, mydict): + mydict['test_int'] = 42 + assert mydict['test_int'] == 42 + + def test_string(self, mydict): + mydict['test_string'] = 'foo' + assert mydict['test_string'] == 'foo' + + def test_df(self, mydict): + df = pd.DataFrame({'a': [0, 1, 2], 'b': ['foo', 'bar', 'baz']}) + mydict['test_df'] = df + assert mydict['test_df'].equals(df) + + def test_immutability(self, mydict): + l = [1, 2, 3] + mydict['test_mut'] = l + assert mydict['test_mut'] == l + l.append(42) + assert not mydict['test_mut'] == l