diff --git a/fixture_tests/README.md b/fixture_tests/README.md new file mode 100644 index 000000000..9a33c2234 --- /dev/null +++ b/fixture_tests/README.md @@ -0,0 +1,44 @@ +# GCode Fixture Tests +Basic tests sent to ESP32 hardware, to validate firmware behavior. + +The `run_fixture` command is used to exercise a fixture on real hardware. Fixtures contain a list of +commands to send to the ESP32, and a list of expected responses. The test runner will send the +commands to the ESP32, and compare the responses to the expected responses. + +Install the tool's dependencies with pip: +```bash +pip install -r requirements.txt +``` + +Supported operations: +- `# ...`: Comment +- `->`: Send a command to the ESP32 +- `<-`: Expect a response from the ESP32 +- `<~`: Expect an optional message from the ESP32, but on mismatch, continue the test +- `<|`: Expect one of the following responses from the ESP32 + +The tool can be ran with either a directory, or a single file. If a directory is provided, the tool +will run all the files ending in `.nc` in the directory. + +Example, checking alarm state: +```bash +./run_fixture /dev/cu.usbserial-31320 fixtures/alarms.nc +-> $X +<~ [MSG:INFO: Caution: Unlocked] +<- ok +-> $Alarm/Send=10 +<- ok +<- [MSG:INFO: ALARM: Spindle Control] +Fixture fixtures/alarms.nc passed +``` + +Example, checking idle status reporting: +```bash +./run_fixture /dev/cu.usbserial-31320 fixtures/idle_status.nc +-> $X +<~ [MSG:INFO: Caution: Unlocked] +<- ok +-> ?? +<| +Fixture fixtures/idle_status.nc passed +``` diff --git a/fixture_tests/fixtures/alarms.nc b/fixture_tests/fixtures/alarms.nc new file mode 100644 index 000000000..19ee47e28 --- /dev/null +++ b/fixture_tests/fixtures/alarms.nc @@ -0,0 +1,6 @@ +-> $X +<~ [MSG:INFO: Caution: Unlocked] +<- ok +-> $Alarm/Send=10 +<- ok +<- [MSG:INFO: ALARM: Spindle Control] diff --git a/fixture_tests/fixtures/idle_status.nc b/fixture_tests/fixtures/idle_status.nc new file mode 100644 index 000000000..d651bfefa --- /dev/null +++ b/fixture_tests/fixtures/idle_status.nc @@ -0,0 +1,6 @@ +-> $X +<~ [MSG:INFO: Caution: Unlocked] +<- ok +-> ?? +<| +<| diff --git a/fixture_tests/requirements.txt b/fixture_tests/requirements.txt new file mode 100644 index 000000000..c08c3c561 --- /dev/null +++ b/fixture_tests/requirements.txt @@ -0,0 +1,2 @@ +pyserial==3.5 +termcolor==2.4.0 diff --git a/fixture_tests/run_fixture b/fixture_tests/run_fixture new file mode 100755 index 000000000..3cda67701 --- /dev/null +++ b/fixture_tests/run_fixture @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 -u +# runs python unbuffered + +from termcolor import colored +import argparse +import os +import serial + +parser = argparse.ArgumentParser() +parser.add_argument("device") +parser.add_argument("fixture_file") +parser.add_argument("-b", "--baudrate", type=int, default=115200) +args = parser.parse_args() + +OPS = [ + # send to controller + "->", + # expect from controller + "<-", + # expect from controller, but optional + "<~", + # expect one of + "<|", +] + +fixture_files = [] + +# check if fixture_file is a directory +if os.path.isdir(args.fixture_file): + for file in os.listdir(args.fixture_file): + if file.endswith(".nc"): + fixture_files.append(os.path.join(args.fixture_file, file)) +else: + fixture_files.append(args.fixture_file) + + +def parse_fixture_lines(fixture_file): + # fixture_lines is a list of tuples: + # (op, match, lineno) + + # Read the fixture file + with open(fixture_file, "r") as f: + fixture_lines = [] + fixture_file = f.read() + for lineno, line in enumerate(fixture_file.splitlines()): + if line.startswith("#"): + # skip comment lines + continue + + for op in OPS: + if line.startswith(op + " "): + line = line[len(op) + 1 :] + if op == "<|": + if len(fixture_lines) > 0 and fixture_lines[-1][0] == "<|": + # append to previous group of matches + fixture_lines[-1][1].append(line) + else: + # new group of matches + fixture_lines.append((op, [line], lineno + 1)) + else: + fixture_lines.append((op, line, lineno + 1)) + break + else: + raise ValueError( + f"Invalid line {lineno} in fixture file {fixture_file}: {line}" + ) + return fixture_lines + + +def run_fixture(fixture_file): + fixture_lines = parse_fixture_lines(fixture_file) + controller = serial.Serial(args.device, args.baudrate, timeout=1) + try: + # last line read from the controller + line = None + + for op, fixture_line, lineno in fixture_lines: + if op == "->": + # send the fixture line to the controller + print( + colored(f"{op} ", "dark_grey") + + colored(fixture_line, "green", attrs=["dark"]) + ) + controller.write(fixture_line.encode("utf-8") + b"\n") + elif op == "<-" or op == "<~" or op == "<|": + is_optional = op == "<~" + + # read a line, and wait for the expected response + if line is None: + line = controller.readline().decode("utf-8").strip() + + if op == "<|": # match any one of + if line in fixture_line: + print( + colored(f"{op} ", "dark_grey") + + colored(line, "green", attrs=["dark", "bold"]) + ) + line = None + else: + print(f"Test failed at line {colored(str(lineno), 'red')}") + print(f"Expected one of:") + for fline in fixture_line: + print(f" `{colored(fline, 'red')}'") + print(f"Actual: `{colored(line, 'red')}'") + exit(1) + elif line == fixture_line: # exact match + print( + colored(f"{op} ", "dark_grey") + + colored(line, "green", attrs=["dark", "bold"]) + ) + line = None + else: # match failed + if is_optional: # but that's okay if it's an optional line + print( + colored(f"{op} Did not get optional line ", "dark_grey") + + colored(fixture_line, "dark_grey", attrs=["bold"]) + ) + # do not clear line, so we can try to match it again on + # the next op + else: + print(f"Test failed at line {colored(str(lineno), 'red')}") + print(f"Expected: `{colored(fixture_line, 'red')}'") + print(f"Actual: `{colored(line, 'red')}'") + exit(1) + + except KeyboardInterrupt: + print("Interrupt") + except TimeoutError as e: + print("Timeout waiting for response, line: " + e.args[0]) + finally: + controller.close() + + print( + colored(f"Fixture ", "green") + + colored(fixture_file, "green", attrs=["bold"]) + + colored(" passed", "green") + ) + + +for fixture_file in fixture_files: + run_fixture(fixture_file)