diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4fbad2a0..9dc972b3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Changelog ========= +----- +Added +~~~~~ + +* Ability to disable the recursively evaluation of jinja and yaql expressions, by setting the environment variable ``ENABLE_RECURSIVELY_EVALUATION`` to ``false``. + Contributed by @moradf90 + + Unreleased ---------- @@ -38,6 +46,7 @@ Fixed * Update jsonschema requirements to allow 3.2 (security fix) Contributed by @james-bellamy + 1.5.0 ----- diff --git a/orquesta/__init__.py b/orquesta/__init__.py index f8ed314e..4b540ee9 100644 --- a/orquesta/__init__.py +++ b/orquesta/__init__.py @@ -13,3 +13,4 @@ # limitations under the License. __version__ = "1.6.0" + diff --git a/orquesta/expressions/base.py b/orquesta/expressions/base.py index 4e62c454..281cea94 100644 --- a/orquesta/expressions/base.py +++ b/orquesta/expressions/base.py @@ -15,6 +15,7 @@ import abc import inspect import logging +import os import re import six import threading @@ -37,6 +38,11 @@ class Evaluator(object): _type = "unspecified" _delimiter = None + @classmethod + def enable_recursively_evaluation(cls): + env_value = os.environ.get("ENABLE_RECURSIVELY_EVALUATION") + return not (env_value is not None and str(env_value).lower() == "false") + @classmethod def get_type(cls): return cls._type @@ -131,7 +137,10 @@ def validate(statement): def evaluate(statement, data=None): if isinstance(statement, dict): - return {evaluate(k, data=data): evaluate(v, data=data) for k, v in six.iteritems(statement)} + return { + evaluate(k, data=data): evaluate(v, data=data) + for k, v in six.iteritems(statement) + } elif isinstance(statement, list): return [evaluate(item, data=data) for item in statement] @@ -171,7 +180,9 @@ def extract_vars(statement): def func_has_ctx_arg(func): getargspec = ( - inspect.getargspec if six.PY2 else inspect.getfullargspec # pylint: disable=no-member + inspect.getargspec + if six.PY2 + else inspect.getfullargspec # pylint: disable=no-member ) return "context" in getargspec(func).args diff --git a/orquesta/expressions/jinja.py b/orquesta/expressions/jinja.py index 431a7132..4367a345 100644 --- a/orquesta/expressions/jinja.py +++ b/orquesta/expressions/jinja.py @@ -62,7 +62,9 @@ class JinjaEvaluator(expr_base.Evaluator): # word boundary ctx().* # word boundary ctx(*)* # word boundary ctx(*).* - _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_ctx_ref_pattern) + _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format( + _regex_ctx_ref_pattern + ) _regex_ctx_var_parser = re.compile(_regex_ctx_pattern) _regex_var = r"[a-zA-Z0-9_-]+" @@ -96,7 +98,11 @@ def contextualize(cls, data): ctx["__current_item"] = ctx["__vars"].get("__current_item") for name, func in six.iteritems(cls._custom_functions): - ctx[name] = functools.partial(func, ctx) if expr_base.func_has_ctx_arg(func) else func + ctx[name] = ( + functools.partial(func, ctx) + if expr_base.func_has_ctx_arg(func) + else func + ) return ctx @@ -136,7 +142,9 @@ def validate(cls, text): try: parser = jinja2.parser.Parser( - cls._jinja_env.overlay(), cls.strip_delimiter(expr), state="variable" + cls._jinja_env.overlay(), + cls.strip_delimiter(expr), + state="variable", ) parser.parse_expression() @@ -161,7 +169,8 @@ def _evaluate_and_expand(cls, text, data=None): # Traverse and evaulate again in case additional inline epxressions are # introduced after the jinja block is evaluated. - output = cls._evaluate_and_expand(output, data) + if cls.enable_recursively_evaluation(): + output = cls._evaluate_and_expand(output, data) else: # The output will first be the original text and the expressions # will be substituted by the evaluated value. @@ -176,7 +185,10 @@ def _evaluate_and_expand(cls, text, data=None): if inspect.isgenerator(result): result = list(result) - if isinstance(result, six.string_types): + if ( + isinstance(result, six.string_types) + and cls.enable_recursively_evaluation() + ): result = cls._evaluate_and_expand(result, data) # For StrictUndefined values, UndefinedError only gets raised when the value is @@ -185,7 +197,9 @@ def _evaluate_and_expand(cls, text, data=None): # raise an exception with error description. if not isinstance(result, jinja2.runtime.StrictUndefined): if len(exprs) > 1 or block_exprs or len(output) > len(expr): - output = output.replace(expr, str_util.unicode(result, force=True)) + output = output.replace( + expr, str_util.unicode(result, force=True) + ) else: output = str_util.unicode(result) @@ -216,9 +230,11 @@ def evaluate(cls, text, data=None): output = cls._evaluate_and_expand(text, data=data) if isinstance(output, six.string_types): - exprs = [cls.strip_delimiter(expr) for expr in cls._regex_parser.findall(output)] + exprs = [ + cls.strip_delimiter(expr) for expr in cls._regex_parser.findall(output) + ] - if exprs: + if exprs and cls.enable_recursively_evaluation(): raise JinjaEvaluationException( "There are unresolved variables: %s" % ", ".join(exprs) ) @@ -226,7 +242,9 @@ def evaluate(cls, text, data=None): if isinstance(output, six.string_types) and raw_blocks: # Put raw blocks back into the expression. for i in range(0, len(raw_blocks)): - output = output.replace("{%s}" % str(i), raw_blocks[i]) # pylint: disable=E1101 + output = output.replace( + "{%s}" % str(i), raw_blocks[i] + ) # pylint: disable=E1101 # Evaluate the raw blocks. ctx = cls.contextualize(data) diff --git a/orquesta/expressions/yql.py b/orquesta/expressions/yql.py index 7ebcbdfb..baf44743 100644 --- a/orquesta/expressions/yql.py +++ b/orquesta/expressions/yql.py @@ -62,7 +62,9 @@ class YAQLEvaluator(expr_base.Evaluator): # word boundary ctx().* # word boundary ctx(*)* # word boundary ctx(*).* - _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_ctx_ref_pattern) + _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format( + _regex_ctx_ref_pattern + ) _regex_ctx_var_parser = re.compile(_regex_ctx_pattern) _regex_var = r"[a-zA-Z0-9_-]+" @@ -84,7 +86,9 @@ def contextualize(cls, data): # Some yaql expressions (e.g. distinct()) refer to hash value of variable. # But some built-in Python type values (e.g. list and dict) don't have __hash__() method. # The convert_input_data method parses specified variable and convert it to hashable one. - if isinstance(data, yaql_utils.SequenceType) or isinstance(data, yaql_utils.MappingType): + if isinstance(data, yaql_utils.SequenceType) or isinstance( + data, yaql_utils.MappingType + ): ctx["__vars"] = yaql_utils.convert_input_data(data) else: ctx["__vars"] = data or {} @@ -144,7 +148,10 @@ def evaluate(cls, text, data=None): if inspect.isgenerator(result): result = list(result) - if isinstance(result, six.string_types): + if ( + isinstance(result, six.string_types) + and cls.enable_recursively_evaluation() + ): result = cls.evaluate(result, data) if len(exprs) > 1 or len(output) > len(expr): @@ -158,7 +165,9 @@ def evaluate(cls, text, data=None): raise YaqlEvaluationException(msg % (error, expr)) except (yaql_exc.YaqlException, ValueError, TypeError) as e: msg = "Unable to evaluate expression '%s'. %s: %s" - raise YaqlEvaluationException(msg % (expr, e.__class__.__name__, str(e).strip("'"))) + raise YaqlEvaluationException( + msg % (expr, e.__class__.__name__, str(e).strip("'")) + ) except Exception as e: msg = "Unable to evaluate expression '%s'. %s: %s" raise YaqlEvaluationException(msg % (expr, e.__class__.__name__, str(e))) diff --git a/orquesta/tests/unit/expressions/test_facade_jinja_evaluate.py b/orquesta/tests/unit/expressions/test_facade_jinja_evaluate.py index f189d046..9730c2d9 100644 --- a/orquesta/tests/unit/expressions/test_facade_jinja_evaluate.py +++ b/orquesta/tests/unit/expressions/test_facade_jinja_evaluate.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest from orquesta import exceptions as exc @@ -30,7 +30,9 @@ def test_basic_eval_undefined(self): data = {} - self.assertRaises(exc.ExpressionEvaluationException, expr_base.evaluate, expr, data) + self.assertRaises( + exc.ExpressionEvaluationException, expr_base.evaluate, expr, data + ) def test_dict_eval(self): expr = "{{ ctx().nested.foo }}" @@ -58,12 +60,28 @@ def test_eval_recursive(self): self.assertEqual("fee-fi-fo-fum", expr_base.evaluate(expr, data)) + # when the recursively evaluation is disable + os.environ.setdefault("ENABLE_RECURSIVELY_EVALUATION", "false") + + self.assertEqual("{{ ctx().fi }}", expr_base.evaluate(expr, data)) + def test_eval_recursive_undefined(self): expr = "{{ ctx().fee }}" - data = {"fee": "{{ ctx().fi }}", "fi": "{{ ctx().fo }}", "fo": "{{ ctx().fum }}"} + data = { + "fee": "{{ ctx().fi }}", + "fi": "{{ ctx().fo }}", + "fo": "{{ ctx().fum }}", + } + + self.assertRaises( + exc.ExpressionEvaluationException, expr_base.evaluate, expr, data + ) + + # when the recursively evaluation is disable + os.environ.setdefault("ENABLE_RECURSIVELY_EVALUATION", "false") - self.assertRaises(exc.ExpressionEvaluationException, expr_base.evaluate, expr, data) + self.assertEqual("{{ ctx().fi }}", expr_base.evaluate(expr, data)) def test_multi_eval_recursive(self): expr = "{{ ctx().fee }} {{ ctx().im }}" @@ -79,6 +97,13 @@ def test_multi_eval_recursive(self): self.assertEqual("fee-fi-fo-fum! i'm hungry!", expr_base.evaluate(expr, data)) + # when the recursively evaluation is disable + os.environ.setdefault("ENABLE_RECURSIVELY_EVALUATION", "false") + + self.assertEqual( + "{{ ctx().fi }} {{ ctx().hungry }}", expr_base.evaluate(expr, data) + ) + def test_eval_list(self): expr = ["{{ ctx().foo }}", "{{ ctx().marco }}", "foo{{ ctx().foo }}"] @@ -136,7 +161,14 @@ def test_eval_dict_of_list(self): self.assertDictEqual(expected, expr_base.evaluate(expr, data)) def test_type_preservation(self): - data = {"k1": 101, "k2": 1.999, "k3": True, "k4": [1, 2], "k5": {"k": "v"}, "k6": None} + data = { + "k1": 101, + "k2": 1.999, + "k3": True, + "k4": [1, 2], + "k5": {"k": "v"}, + "k6": None, + } self.assertEqual(data["k1"], expr_base.evaluate("{{ ctx().k1 }}", data)) @@ -179,7 +211,9 @@ def test_block_eval_undefined(self): data = {"x": ["a", "b", "c"]} - self.assertRaises(exc.ExpressionEvaluationException, expr_base.evaluate, expr, data) + self.assertRaises( + exc.ExpressionEvaluationException, expr_base.evaluate, expr, data + ) def test_block_eval_recursive(self): expr = "{% for i in ctx().x %}{{ i }}{% endfor %}" @@ -200,7 +234,8 @@ def test_block_eval_recursive(self): def test_multi_block_eval(self): expr = ( - "{% for i in ctx().x %}{{ i }}{% endfor %}" "{% for i in ctx().y %}{{ i }}{% endfor %}" + "{% for i in ctx().x %}{{ i }}{% endfor %}" + "{% for i in ctx().y %}{{ i }}{% endfor %}" ) data = {"x": ["a", "b", "c"], "y": ["d", "e", "f"]}