From 861102e88937c5a2000b8bf02362dee45371bcdd Mon Sep 17 00:00:00 2001 From: Tom Weaver <43210711+TomWeaver18@users.noreply.github.com> Date: Mon, 22 Jul 2019 17:14:35 +0100 Subject: [PATCH] Add DexLabel support. (#33) * Add DexLabel Command * Unresolved DexLabels in DexExpectWatchValue now appear in an error message. use "isinstance" for label checking rather than trying to cast to int. fixed line length issue in ParseCommand.py. * rework get_label_args to use list comprehension.\nany string parameter is now considered a valid label * resolve issues introduced by master merge * fix comment indentation issues in ParseCommands.py --- dex/command/CommandBase.py | 6 +++ dex/command/ParseCommand.py | 41 ++++++++++++------- dex/command/__init__.py | 2 +- dex/command/commands/DexExpectWatchValue.py | 16 ++++++++ .../commands/DexLabel.py} | 30 ++++++++++---- dex/debugger/DebuggerBase.py | 11 +++-- dex/debugger/Debuggers.py | 7 +--- dex/debugger/visualstudio/VisualStudio.py | 5 ++- dex/dextIR/__init__.py | 1 - dex/heuristic/Heuristic.py | 15 +++---- dexter.py | 1 - 11 files changed, 87 insertions(+), 48 deletions(-) rename dex/{dextIR/CommandIR.py => command/commands/DexLabel.py} (64%) diff --git a/dex/command/CommandBase.py b/dex/command/CommandBase.py index 87a8360..d9c24c0 100644 --- a/dex/command/CommandBase.py +++ b/dex/command/CommandBase.py @@ -33,6 +33,12 @@ def __init__(self): self.path = None self.lineno = None + def get_label_args(self): + return list() + + def has_labels(self): + return False + @staticmethod @abc.abstractstaticmethod def get_name(): diff --git a/dex/command/ParseCommand.py b/dex/command/ParseCommand.py index dbcfb40..99a4b93 100644 --- a/dex/command/ParseCommand.py +++ b/dex/command/ParseCommand.py @@ -28,13 +28,13 @@ from collections import defaultdict from dex.utils.Exceptions import CommandParseError -from dex.dextIR import CommandIR from dex.command.CommandBase import CommandBase from dex.command.commands.DexExpectProgramState import DexExpectProgramState from dex.command.commands.DexExpectStepKind import DexExpectStepKind from dex.command.commands.DexExpectStepOrder import DexExpectStepOrder from dex.command.commands.DexExpectWatchValue import DexExpectWatchValue +from dex.command.commands.DexLabel import DexLabel from dex.command.commands.DexUnreachable import DexUnreachable from dex.command.commands.DexWatch import DexWatch @@ -45,6 +45,7 @@ def _get_valid_commands(): DexExpectStepKind.get_name() : DexExpectStepKind, DexExpectStepOrder.get_name() : DexExpectStepOrder, DexExpectWatchValue.get_name() : DexExpectWatchValue, + DexLabel.get_name() : DexLabel, DexUnreachable.get_name() : DexUnreachable, DexWatch.get_name() : DexWatch } @@ -52,8 +53,8 @@ def _get_valid_commands(): def _get_command_name(command_raw: str) -> str: """Return command name by splitting up DExTer command contained in - command_raw on the first opening paranthesis and further stripping - any potential leading or trailing whitespace. + command_raw on the first opening paranthesis and further stripping + any potential leading or trailing whitespace. """ return command_raw.split('(', 1)[0].rstrip() @@ -77,14 +78,25 @@ def _eval_command(command_raw: str, valid_commands: dict) -> CommandBase: return command -def get_command_object(commandIR: CommandIR): - """Externally visible version of _safe_eval. Only returns the Command - object itself. - """ - command = _eval_command(commandIR.raw_text, _get_valid_commands()) - command.path = commandIR.loc.path - command.lineno = commandIR.loc.lineno - return command +def resolve_labels(command: CommandBase, commands: dict): + """Attempt to resolve any labels in command""" + dex_labels = commands['DexLabel'] + command_label_args = command.get_label_args() + for command_arg in command_label_args: + for dex_label in list(dex_labels.values()): + if (dex_label.path == command.path and + dex_label.eval() == command_arg): + command.resolve_label(dex_label.get_as_pair()) + # labels for command should be resolved by this point. + if command.has_labels(): + syntax_error = SyntaxError() + syntax_error.filename = command.path + syntax_error.lineno = command.lineno + syntax_error.offset = 0 + syntax_error.msg = 'Unresolved labels' + for label in command.get_label_args(): + syntax_error.msg += ' \'' + label + '\'' + raise syntax_error def _find_start_of_command(line, valid_commands) -> int: @@ -133,7 +145,6 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): continue command_name = _get_command_name(line[start:]) - command_path = path command_lineno = lineno command_column = start + 1 # Column numbers start at 1. cmd_text_list = [command_name] @@ -152,9 +163,11 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): try: raw_text = "".join(cmd_text_list) command = _eval_command(raw_text, valid_commands) - command.path = command_path - command.lineno = command_lineno + command_name = _get_command_name(raw_text) + command.path = path + command.lineno = lineno command.raw_text = raw_text + resolve_labels(command, commands) assert (path, lineno) not in commands[command_name], ( command_name, commands[command_name]) commands[command_name][path, lineno] = command diff --git a/dex/command/__init__.py b/dex/command/__init__.py index 9ce1d4b..9937568 100644 --- a/dex/command/__init__.py +++ b/dex/command/__init__.py @@ -21,4 +21,4 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from dex.command.ParseCommand import find_all_commands, get_command_object +from dex.command.ParseCommand import find_all_commands diff --git a/dex/command/commands/DexExpectWatchValue.py b/dex/command/commands/DexExpectWatchValue.py index ebf664b..6c8e9ed 100644 --- a/dex/command/commands/DexExpectWatchValue.py +++ b/dex/command/commands/DexExpectWatchValue.py @@ -145,6 +145,22 @@ def missing_values(self): def encountered_values(self): return sorted(list(set(self.values) - self._missing_values)) + + def resolve_label(self, label_line_pair): + # from_line and to_line could have the same label. + label, lineno = label_line_pair + if self._to_line == label: + self._to_line = lineno + if self._from_line == label: + self._from_line = lineno + + def has_labels(self): + return len(self.get_label_args()) > 0 + + def get_label_args(self): + return [label for label in (self._from_line, self._to_line) + if isinstance(label, str)] + def _handle_watch(self, step, watch): self.times_encountered += 1 diff --git a/dex/dextIR/CommandIR.py b/dex/command/commands/DexLabel.py similarity index 64% rename from dex/dextIR/CommandIR.py rename to dex/command/commands/DexLabel.py index 0010428..cc6ad39 100644 --- a/dex/dextIR/CommandIR.py +++ b/dex/command/commands/DexLabel.py @@ -1,7 +1,7 @@ # DExTer : Debugging Experience Tester # ~~~~~~ ~ ~~ ~ ~~ # -# Copyright (c) 2018 by SN Systems Ltd., Sony Interactive Entertainment Inc. +# Copyright (c) 2019 by SN Systems Ltd., Sony Interactive Entertainment Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -20,14 +20,28 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -"""Serialization of a DExTer command embedded within one of the files under -test. +"""Command used to give a line in a test a named psuedonym. Every DexLabel has + a line number and Label string component. """ -from dex.dextIR.LocIR import LocIR +from dex.command.CommandBase import CommandBase -class CommandIR: - def __init__(self, raw_text: str, loc: LocIR): - self.raw_text = raw_text - self.loc = loc +class DexLabel(CommandBase): + def __init__(self, label): + + if not isinstance(label, str): + raise TypeError('invalid argument type') + + self._label = label + super(DexLabel, self).__init__() + + def get_as_pair(self): + return (self._label, self.lineno) + + @staticmethod + def get_name(): + return __class__.__name__ + + def eval(self): + return self._label diff --git a/dex/debugger/DebuggerBase.py b/dex/debugger/DebuggerBase.py index 41a9cc1..f654476 100644 --- a/dex/debugger/DebuggerBase.py +++ b/dex/debugger/DebuggerBase.py @@ -28,7 +28,7 @@ import time import traceback -from dex.command import get_command_object + from dex.dextIR import DebuggerIR, ValueIR from dex.utils.Exceptions import DebuggerException from dex.utils.Exceptions import NotYetLoadedDebuggerException @@ -118,9 +118,9 @@ def _update_step_watches(self, step_info): try: # Iterate over all watches of the types named in watch_cmds for watch in towatch: - if (watch.loc.path == loc.path - and watch.loc.lineno == loc.lineno): - result = get_command_object(watch).eval(self) + if (watch.path == loc.path + and watch.lineno == loc.lineno): + result = watch.eval(self) step_info.watches.update(result) break except KeyError: @@ -137,8 +137,7 @@ def start(self): self.steps.clear_steps() self.launch() - for command in chain.from_iterable(self.steps.commands.values()): - command_obj = get_command_object(command) + for command_obj in chain.from_iterable(self.steps.commands.values()): self.watches.update(command_obj.get_watches()) max_steps = self.context.options.max_steps diff --git a/dex/debugger/Debuggers.py b/dex/debugger/Debuggers.py index af1bcca..62b7056 100644 --- a/dex/debugger/Debuggers.py +++ b/dex/debugger/Debuggers.py @@ -30,7 +30,7 @@ from tempfile import NamedTemporaryFile from dex.command import find_all_commands -from dex.dextIR import CommandIR, DextIR, LocIR +from dex.dextIR import DextIR from dex.utils import get_root_directory, Timer from dex.utils.Environment import is_native_windows from dex.utils.Exceptions import CommandParseError, DebuggerException @@ -150,10 +150,7 @@ def _get_command_infos(context): for command in commands[command_type].values(): if command_type not in command_infos: command_infos[command_type] = [] - - loc = LocIR(path=command.path, lineno=command.lineno, column=None) - command_infos[command_type].append( - CommandIR(loc=loc, raw_text=command.raw_text)) + command_infos[command_type].append(command) return OrderedDict(command_infos) diff --git a/dex/debugger/visualstudio/VisualStudio.py b/dex/debugger/visualstudio/VisualStudio.py index e200cc1..4e0463d 100644 --- a/dex/debugger/visualstudio/VisualStudio.py +++ b/dex/debugger/visualstudio/VisualStudio.py @@ -179,8 +179,9 @@ def get_step_info(self): frames.append(frame) loc = LocIR(**self._location) - frames[0].loc = loc - state_frames[0].location = SourceLocation(**self._location) + if frames: + frames[0].loc = loc + state_frames[0].location = SourceLocation(**self._location) reason = StopReason.BREAKPOINT if loc.path is None: # pylint: disable=no-member diff --git a/dex/dextIR/__init__.py b/dex/dextIR/__init__.py index c2626e0..8b1c614 100644 --- a/dex/dextIR/__init__.py +++ b/dex/dextIR/__init__.py @@ -30,5 +30,4 @@ from dex.dextIR.LocIR import LocIR from dex.dextIR.StepIR import StepIR, StepKind, StopReason from dex.dextIR.ValueIR import ValueIR -from dex.dextIR.CommandIR import CommandIR from dex.dextIR.ProgramState import ProgramState, SourceLocation, StackFrame diff --git a/dex/heuristic/Heuristic.py b/dex/heuristic/Heuristic.py index 58b5f46..103854f 100644 --- a/dex/heuristic/Heuristic.py +++ b/dex/heuristic/Heuristic.py @@ -31,7 +31,6 @@ import os from itertools import groupby from dex.command.commands.DexExpectWatchValue import StepValueInfo -from dex.command.ParseCommand import get_command_object PenaltyCommand = namedtuple('PenaltyCommand', ['pen_dict', 'max_penalty']) @@ -128,8 +127,7 @@ def __init__(self, context, steps): # Get DexExpectWatchValue results. try: - for watch in steps.commands["DexExpectWatchValue"]: - command = get_command_object(watch) + for command in steps.commands['DexExpectWatchValue']: command.eval(steps) maximum_possible_penalty = min(3, len( command.values)) * worst_penalty @@ -143,8 +141,7 @@ def __init__(self, context, steps): try: penalties = defaultdict(list) maximum_possible_penalty_all = 0 - for command in steps.commands["DexExpectProgramState"]: - expect_state = get_command_object(command) + for expect_state in steps.commands['DexExpectProgramState']: success = expect_state.eval(steps) p = 0 if success else self.penalty_incorrect_program_state @@ -175,8 +172,7 @@ def __init__(self, context, steps): penalties = defaultdict(list) maximum_possible_penalty_all = 0 try: - for step_kind in steps.commands['DexExpectStepKind']: - command = get_command_object(step_kind) + for command in steps.commands['DexExpectStepKind']: command.eval() # Cap the penalty at 2 * expected count or else 1 maximum_possible_penalty = max(command.count * 2, 1) @@ -219,11 +215,10 @@ def __init__(self, context, steps): if 'DexExpectStepOrder' in steps.commands: cmds = steps.commands['DexExpectStepOrder'] - cmds = [(c, get_command_object(c)) for c in cmds] # Form a list of which line/cmd we _should_ have seen - cmd_num_lst = [(x, c.loc.lineno) for c, co in cmds - for x in co.sequence] + cmd_num_lst = [(x, c.lineno) for c in cmds + for x in c.sequence] # Order them by the sequence number cmd_num_lst.sort(key=lambda t: t[0]) # Strip out sequence key diff --git a/dexter.py b/dexter.py index c66343d..a3bbc55 100755 --- a/dexter.py +++ b/dexter.py @@ -27,7 +27,6 @@ import sys from dex.tools import main -from dex.utils.ReturnCode import ReturnCode if __name__ == '__main__': return_code = main()