jq: merge eval_ast/macroexpand into EVAL. Add DEBUG-EVAL

Original issue describing the change and converting the first set of
implementations: kanaka#592

Tracking issue for other implementations: kanaka#657

All normal tests pass, but REGRESS and self-hosting fail.

Steps:
display the results from jq without python
simplify/improve quasiquote
simplify replenv construction

Cosmetic:
Update the interpreter from latest Debian/Ubuntu.
move first core functions from steps4-A to core.jq
simplify interprocess communication between run and utils.jq
merge run and rts.py, simplify it

impls/jq/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:bionic
+FROM ubuntu:24.04
 MAINTAINER Joel Martin <[email protected]>
 
 ##########################################################
@@ -9,10 +9,8 @@ MAINTAINER Joel Martin <[email protected]>
 RUN apt-get -y update
 
 # Required for running tests
-RUN apt-get -y install make python
-
-# Some typical implementation and test requirements
-RUN apt-get -y install curl libreadline-dev libedit-dev libpcre3-dev
+RUN apt-get -y install make python3
+RUN ln -fs /usr/bin/python3 /usr/local/bin/python
 
 RUN mkdir -p /mal
 WORKDIR /mal
@@ -21,12 +19,4 @@ WORKDIR /mal
 # Specific implementation requirements
 #########################################################
 
-RUN apt-get -y install python3.8 wget
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10
-
-# grab jq 1.6 from github releases
-RUN wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
-
-RUN chmod +x jq-linux64
-# a bit ugly, but it'll do?
-RUN mv jq-linux64 /usr/bin/jq
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install jq

impls/jq/Makefile

@@ -1,3 +1,11 @@
 all:
 
-.PHONY: clean
+clean:
+	rm -fr .mypy_cache/
+
+check:
+	flake8 run
+	pylint run
+	mypy run
+
+.PHONY: all clean check

impls/jq/core.jq

@@ -4,6 +4,31 @@ include "reader";
 
 def core_identify:
     {
+        "+": {
+            kind: "fn", # native function
+            inputs: 2,
+            function: "number_add"
+        },
+        "-": {
+            kind: "fn", # native function
+            inputs: 2,
+            function: "number_sub"
+        },
+        "*": {
+            kind: "fn", # native function
+            inputs: 2,
+            function: "number_mul"
+        },
+        "/": {
+            kind: "fn", # native function
+            inputs: 2,
+            function: "number_div"
+        },
+        "eval": {
+            kind: "fn",
+            inputs: 1,
+            function: "eval"
+        },
         "env": {
             kind: "fn",
             function: "env",
@@ -369,9 +394,9 @@ def core_interp(arguments; env):
     ) // (
         select(.function == ">=") | null | wrap(arguments[0].value >= arguments[1].value | tostring)
     ) // (
-        select(.function == "slurp") | arguments | map(.value) | issue_extern("read") | wrap("string")
+        select(.function == "slurp") | arguments[0].value | slurp | wrap("string")
     ) // (
-        select(.function == "read-string") | arguments | first.value | read_str | read_form.value
+        select(.function == "read-string") | arguments | first.value | read_form
     ) // (
         select(.function == "atom?") | null | wrap(arguments | first.kind == "atom" | tostring)
     ) // (

impls/jq/env.jq

@@ -257,7 +257,7 @@ def addToEnv(envexp; name):
 def _env_remove_references(refs):
     if . != null then
         if .environment == null then
-            _debug("This one broke the rules, officer: \(.)")
+            debug("This one broke the rules, officer: \(.)")
         else
             {
                 environment: (.environment | to_entries | map(select(.key as $key | refs | contains([$key]) | not)) | from_entries),

impls/jq/interp.jq

@@ -71,7 +71,7 @@ def addFrees(newEnv; frees):
 def interpret(arguments; env; _eval):
     extractReplEnv(env) as $replEnv |
     extractAtoms(env) as $envAtoms |
-    (if $DEBUG then _debug("INTERP: \(. | pr_str(env))") else . end) |
+    (if $DEBUG then debug("INTERP: \(pr_str(env))") end) |
     (select(.kind == "fn") |
         arg_check(arguments) | 
             (select(.function == "eval") | 

impls/jq/reader.jq

@@ -308,4 +308,4 @@ def read_form_(depth):
         end end end end end end end end end end);
 
 def read_form:
-    {tokens: .} | read_form_(0);
+    ({tokens: read_str} | read_form_(0).value) // {kind: "nil"};

impls/jq/run

@@ -1,3 +1,52 @@
-#!/bin/sh
+#!/usr/bin/python3
+"""Spawn a jq subprocess and wrap some IO interactions for it.
 
-exec python rts.py "${@}"
+jq seems unable to
+ - open an arbitrary file (slurp)
+ - emit a string on stdout without new line (readline)
+"""
+from json import JSONDecodeError, dumps, loads
+from os import environ
+from os.path import dirname, join, realpath
+from subprocess import PIPE, Popen
+from sys import argv
+
+rundir = dirname(realpath(__file__))
+with Popen(args=['/usr/bin/jq',
+                 '--argjson', 'DEBUG', 'false',
+                 '-nrM',  # --null-input --raw-output --monochrome-output
+                 '-L', rundir,
+                 '-f', join(rundir, environ.get('STEP', 'stepA_mal') + '.jq'),
+                 '--args'] + argv[1:],
+           stdin=PIPE, stderr=PIPE, encoding='utf-8',
+           ) as proc:
+    assert proc.stderr is not None  # for mypy
+    for received in proc.stderr:
+        try:
+            as_json = loads(received)
+        except JSONDecodeError:
+            print(f'JQ STDERR: {received}', end=None)
+        else:
+            match as_json:
+                case ['DEBUG:', ['display', str(message)]]:
+                    # While at it, provide a way to immediately print to
+                    # stdin for DEBUG-EVAL, println and prn (jq is able to
+                    # output to stderr, but *we* are already piping it).
+                    print(message)
+                    # Jq waits for this signal to go on, so that its own
+                    # output is not mixed with our one.
+                    print('null', file=proc.stdin, flush=True)
+                case ['DEBUG:', ['readline', str(prompt)]]:
+                    try:
+                        data = input(prompt)
+                    except EOFError:
+                        break                   # Expected end of this script
+                    print(dumps(data), file=proc.stdin, flush=True)
+                case ['DEBUG:', ['slurp', str(fname)]]:
+                    with open(fname, 'r', encoding='utf-8') as file_handler:
+                        data = file_handler.read()
+                    print(dumps(data), file=proc.stdin, flush=True)
+                case _:
+                    # Allow normal debugging information for other purposes.
+                    print(f'JQ STDERR: {received}', end=None)
+print()

impls/jq/step0_repl.jq

@@ -1,10 +1,5 @@
 include "utils";
 
-def read_line:
-    . as $in
-    | label $top
-    | _readline;
-
 def READ:
     .;
 
@@ -14,14 +9,10 @@ def EVAL:
 def PRINT:
     .;
 
-def rep:
-    READ | EVAL | PRINT | _display;
-
-def repl_:
-    ("user> " | _print) |
-    (read_line | rep);
-
 def repl:
-    while(true; repl_);
+    # Infinite generator, interrupted by ./run.
+   "user> " | __readline |
+   READ | EVAL |
+   PRINT, repl;
 
 repl

impls/jq/step1_read_print.jq

@@ -2,41 +2,25 @@ include "reader";
 include "printer";
 include "utils";
 
-def read_line:
-    . as $in
-    | label $top
-    | _readline;
-
 def READ:
-    read_str | read_form | .value;
+    read_form;
 
 def EVAL:
     .;
 
 def PRINT:
     pr_str;
 
-def rep:
-    READ | EVAL |
-        if . != null then
-            PRINT
-        else
-            null
-        end;
-
-def repl_:
-    ("user> " | _print) |
-    (read_line | rep);
-
 def repl:
-    {continue: true} | while(
-        .continue;
-        try {value: repl_, continue: true}
-        catch
-            if is_jqmal_error then
-                {value: "Error: \(.)", continue: true}
-            else
-                {value: ., continue: false}
-            end) | if .value then .value|_display else empty end;
+    # Infinite generator, interrupted by an exception or ./run.
+    "user> " | __readline |
+    try (
+        READ | EVAL |
+        PRINT, repl
+    ) catch if is_jqmal_error then
+        ., repl
+    else
+        halt_error
+    end;
 
 repl