ability to disable recursively evaluation of jinja and yaql expressions.

CHANGELOG.rst

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+1.5.1
+-----
+Added
+~~~~~
+
+* Ability to disable the recursively evaluation of jinja and yaql expressions, by setting the environment variable ``ENABLE_RECURSIVELY_EVALUATION`` to ``false``.
+  Contributed by @moradf90
+
+
 1.5.0
 -----
 
@@ -138,7 +147,7 @@ Added
 ~~~~~
 
 * Add flake8 extension to restrict import alias. (improvement)
-* Add developer docs on getting started, testing, and StackStorm integration. (improvement) 
+* Add developer docs on getting started, testing, and StackStorm integration. (improvement)
 
 Changed
 ~~~~~~~
@@ -168,7 +177,7 @@ Added
 Fixed
 ~~~~~
 
-* Add sleep in while loop for composing execution graph to spread out cpu spike. (improvement) 
+* Add sleep in while loop for composing execution graph to spread out cpu spike. (improvement)
 * Value in quotes in shorthand publish should be evaluated as string type. Fixes #130 (bug fix)
 * Fix interpretation of boolean value in shorthand format of publish. Fixes #119 (bug fix)
 * Update YAQL section in docs on use of "=>" for named parameters in function calls. Closes #124

orquesta/__init__.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.5.0"
+__version__ = "1.5.1"

orquesta/expressions/base.py

@@ -15,6 +15,7 @@
 import abc
 import inspect
 import logging
+import os
 import re
 import six
 import threading
@@ -37,6 +38,11 @@ class Evaluator(object):
     _type = "unspecified"
     _delimiter = None
 
+    @classmethod
+    def enable_recursively_evaluation(cls):
+        env_value = os.environ.get("ENABLE_RECURSIVELY_EVALUATION")
+        return not (env_value is not None and str(env_value).lower() == "false")
+
     @classmethod
     def get_type(cls):
         return cls._type
@@ -131,7 +137,10 @@ def validate(statement):
 
 def evaluate(statement, data=None):
     if isinstance(statement, dict):
-        return {evaluate(k, data=data): evaluate(v, data=data) for k, v in six.iteritems(statement)}
+        return {
+            evaluate(k, data=data): evaluate(v, data=data)
+            for k, v in six.iteritems(statement)
+        }
 
     elif isinstance(statement, list):
         return [evaluate(item, data=data) for item in statement]
@@ -171,7 +180,9 @@ def extract_vars(statement):
 
 def func_has_ctx_arg(func):
     getargspec = (
-        inspect.getargspec if six.PY2 else inspect.getfullargspec  # pylint: disable=no-member
+        inspect.getargspec
+        if six.PY2
+        else inspect.getfullargspec  # pylint: disable=no-member
     )
 
     return "context" in getargspec(func).args

orquesta/expressions/jinja.py

@@ -62,7 +62,9 @@ class JinjaEvaluator(expr_base.Evaluator):
     #   word boundary ctx().*
     #   word boundary ctx(*)*
     #   word boundary ctx(*).*
-    _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_ctx_ref_pattern)
+    _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(
+        _regex_ctx_ref_pattern
+    )
     _regex_ctx_var_parser = re.compile(_regex_ctx_pattern)
 
     _regex_var = r"[a-zA-Z0-9_-]+"
@@ -96,7 +98,11 @@ def contextualize(cls, data):
             ctx["__current_item"] = ctx["__vars"].get("__current_item")
 
         for name, func in six.iteritems(cls._custom_functions):
-            ctx[name] = functools.partial(func, ctx) if expr_base.func_has_ctx_arg(func) else func
+            ctx[name] = (
+                functools.partial(func, ctx)
+                if expr_base.func_has_ctx_arg(func)
+                else func
+            )
 
         return ctx
 
@@ -136,7 +142,9 @@ def validate(cls, text):
 
             try:
                 parser = jinja2.parser.Parser(
-                    cls._jinja_env.overlay(), cls.strip_delimiter(expr), state="variable"
+                    cls._jinja_env.overlay(),
+                    cls.strip_delimiter(expr),
+                    state="variable",
                 )
 
                 parser.parse_expression()
@@ -161,7 +169,8 @@ def _evaluate_and_expand(cls, text, data=None):
 
                 # Traverse and evaulate again in case additional inline epxressions are
                 # introduced after the jinja block is evaluated.
-                output = cls._evaluate_and_expand(output, data)
+                if cls.enable_recursively_evaluation():
+                    output = cls._evaluate_and_expand(output, data)
             else:
                 # The output will first be the original text and the expressions
                 # will be substituted by the evaluated value.
@@ -176,7 +185,10 @@ def _evaluate_and_expand(cls, text, data=None):
                     if inspect.isgenerator(result):
                         result = list(result)
 
-                    if isinstance(result, six.string_types):
+                    if (
+                        isinstance(result, six.string_types)
+                        and cls.enable_recursively_evaluation()
+                    ):
                         result = cls._evaluate_and_expand(result, data)
 
                     # For StrictUndefined values, UndefinedError only gets raised when the value is
@@ -185,7 +197,9 @@ def _evaluate_and_expand(cls, text, data=None):
                     # raise an exception with error description.
                     if not isinstance(result, jinja2.runtime.StrictUndefined):
                         if len(exprs) > 1 or block_exprs or len(output) > len(expr):
-                            output = output.replace(expr, str_util.unicode(result, force=True))
+                            output = output.replace(
+                                expr, str_util.unicode(result, force=True)
+                            )
                         else:
                             output = str_util.unicode(result)
 
@@ -216,17 +230,21 @@ def evaluate(cls, text, data=None):
         output = cls._evaluate_and_expand(text, data=data)
 
         if isinstance(output, six.string_types):
-            exprs = [cls.strip_delimiter(expr) for expr in cls._regex_parser.findall(output)]
+            exprs = [
+                cls.strip_delimiter(expr) for expr in cls._regex_parser.findall(output)
+            ]
 
-            if exprs:
+            if exprs and cls.enable_recursively_evaluation():
                 raise JinjaEvaluationException(
                     "There are unresolved variables: %s" % ", ".join(exprs)
                 )
 
         if isinstance(output, six.string_types) and raw_blocks:
             # Put raw blocks back into the expression.
             for i in range(0, len(raw_blocks)):
-                output = output.replace("{%s}" % str(i), raw_blocks[i])  # pylint: disable=E1101
+                output = output.replace(
+                    "{%s}" % str(i), raw_blocks[i]
+                )  # pylint: disable=E1101
 
             # Evaluate the raw blocks.
             ctx = cls.contextualize(data)

orquesta/expressions/yql.py

@@ -62,7 +62,9 @@ class YAQLEvaluator(expr_base.Evaluator):
     #   word boundary ctx().*
     #   word boundary ctx(*)*
     #   word boundary ctx(*).*
-    _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_ctx_ref_pattern)
+    _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(
+        _regex_ctx_ref_pattern
+    )
     _regex_ctx_var_parser = re.compile(_regex_ctx_pattern)
 
     _regex_var = r"[a-zA-Z0-9_-]+"
@@ -84,7 +86,9 @@ def contextualize(cls, data):
         # Some yaql expressions (e.g. distinct()) refer to hash value of variable.
         # But some built-in Python type values (e.g. list and dict) don't have __hash__() method.
         # The convert_input_data method parses specified variable and convert it to hashable one.
-        if isinstance(data, yaql_utils.SequenceType) or isinstance(data, yaql_utils.MappingType):
+        if isinstance(data, yaql_utils.SequenceType) or isinstance(
+            data, yaql_utils.MappingType
+        ):
             ctx["__vars"] = yaql_utils.convert_input_data(data)
         else:
             ctx["__vars"] = data or {}
@@ -144,7 +148,10 @@ def evaluate(cls, text, data=None):
                 if inspect.isgenerator(result):
                     result = list(result)
 
-                if isinstance(result, six.string_types):
+                if (
+                    isinstance(result, six.string_types)
+                    and cls.enable_recursively_evaluation()
+                ):
                     result = cls.evaluate(result, data)
 
                 if len(exprs) > 1 or len(output) > len(expr):
@@ -158,7 +165,9 @@ def evaluate(cls, text, data=None):
             raise YaqlEvaluationException(msg % (error, expr))
         except (yaql_exc.YaqlException, ValueError, TypeError) as e:
             msg = "Unable to evaluate expression '%s'. %s: %s"
-            raise YaqlEvaluationException(msg % (expr, e.__class__.__name__, str(e).strip("'")))
+            raise YaqlEvaluationException(
+                msg % (expr, e.__class__.__name__, str(e).strip("'"))
+            )
         except Exception as e:
             msg = "Unable to evaluate expression '%s'. %s: %s"
             raise YaqlEvaluationException(msg % (expr, e.__class__.__name__, str(e)))

orquesta/tests/unit/expressions/test_facade_jinja_evaluate.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import os
 import unittest
 
 from orquesta import exceptions as exc
@@ -30,7 +30,9 @@ def test_basic_eval_undefined(self):
 
         data = {}
 
-        self.assertRaises(exc.ExpressionEvaluationException, expr_base.evaluate, expr, data)
+        self.assertRaises(
+            exc.ExpressionEvaluationException, expr_base.evaluate, expr, data
+        )
 
     def test_dict_eval(self):
         expr = "{{ ctx().nested.foo }}"
@@ -58,12 +60,28 @@ def test_eval_recursive(self):
 
         self.assertEqual("fee-fi-fo-fum", expr_base.evaluate(expr, data))
 
+        # when the recursively evaluation is disable
+        os.environ.setdefault("ENABLE_RECURSIVELY_EVALUATION", "false")
+
+        self.assertEqual("{{ ctx().fi }}", expr_base.evaluate(expr, data))
+
     def test_eval_recursive_undefined(self):
         expr = "{{ ctx().fee }}"
 
-        data = {"fee": "{{ ctx().fi }}", "fi": "{{ ctx().fo }}", "fo": "{{ ctx().fum }}"}
+        data = {
+            "fee": "{{ ctx().fi }}",
+            "fi": "{{ ctx().fo }}",
+            "fo": "{{ ctx().fum }}",
+        }
+
+        self.assertRaises(
+            exc.ExpressionEvaluationException, expr_base.evaluate, expr, data
+        )
+
+        # when the recursively evaluation is disable
+        os.environ.setdefault("ENABLE_RECURSIVELY_EVALUATION", "false")
 
-        self.assertRaises(exc.ExpressionEvaluationException, expr_base.evaluate, expr, data)
+        self.assertEqual("{{ ctx().fi }}", expr_base.evaluate(expr, data))
 
     def test_multi_eval_recursive(self):
         expr = "{{ ctx().fee }} {{ ctx().im }}"
@@ -79,6 +97,13 @@ def test_multi_eval_recursive(self):
 
         self.assertEqual("fee-fi-fo-fum! i'm hungry!", expr_base.evaluate(expr, data))
 
+        # when the recursively evaluation is disable
+        os.environ.setdefault("ENABLE_RECURSIVELY_EVALUATION", "false")
+
+        self.assertEqual(
+            "{{ ctx().fi }} {{ ctx().hungry }}", expr_base.evaluate(expr, data)
+        )
+
     def test_eval_list(self):
         expr = ["{{ ctx().foo }}", "{{ ctx().marco }}", "foo{{ ctx().foo }}"]
 
@@ -136,7 +161,14 @@ def test_eval_dict_of_list(self):
         self.assertDictEqual(expected, expr_base.evaluate(expr, data))
 
     def test_type_preservation(self):
-        data = {"k1": 101, "k2": 1.999, "k3": True, "k4": [1, 2], "k5": {"k": "v"}, "k6": None}
+        data = {
+            "k1": 101,
+            "k2": 1.999,
+            "k3": True,
+            "k4": [1, 2],
+            "k5": {"k": "v"},
+            "k6": None,
+        }
 
         self.assertEqual(data["k1"], expr_base.evaluate("{{ ctx().k1 }}", data))
 
@@ -179,7 +211,9 @@ def test_block_eval_undefined(self):
 
         data = {"x": ["a", "b", "c"]}
 
-        self.assertRaises(exc.ExpressionEvaluationException, expr_base.evaluate, expr, data)
+        self.assertRaises(
+            exc.ExpressionEvaluationException, expr_base.evaluate, expr, data
+        )
 
     def test_block_eval_recursive(self):
         expr = "{% for i in ctx().x %}{{ i }}{% endfor %}"
@@ -200,7 +234,8 @@ def test_block_eval_recursive(self):
 
     def test_multi_block_eval(self):
         expr = (
-            "{% for i in ctx().x %}{{ i }}{% endfor %}" "{% for i in ctx().y %}{{ i }}{% endfor %}"
+            "{% for i in ctx().x %}{{ i }}{% endfor %}"
+            "{% for i in ctx().y %}{{ i }}{% endfor %}"
         )
 
         data = {"x": ["a", "b", "c"], "y": ["d", "e", "f"]}