Merge eval-ast and macro expansion into EVAL, add DEBUG-EVAL

See issue kanaka#587.
* Merge eval-ast and eval into a single conditional.
* Expand macros during the apply phase, removing lots of duplicate
  tests, and increasing the overall consistency by allowing the macro
  to be computed instead of referenced by name (`((defmacro! cond
  (...)))` is currently illegal for example).
* Print "EVAL: $ast" at the top of EVAL if DEBUG-EVAL exists in the
  MAL environment.
* Remove macroexpand and quasiquoteexpand special forms.
* Use pattern-matching style in process/step*.txt.

Unresolved issues:
c.2: unable to reproduce with gcc 11.12.0.
elm: the directory is unchanged.
groovy: sometimes fail, but not on each rebuild.
nasm: fails some new soft tests, but the issue is unreproducible when
  running the interpreter manually.
objpascal: unreproducible with fpc 3.2.2.
ocaml: unreproducible with 4.11.1.
perl6: unreproducible with rakudo 2021.09.

Unrelated fixes:
c: fix defmacro! so that it does not mutate functions.
dart: fix recent errors and warnings
logo: prevent macros from mutating functions
python.2: fix argv in step9 and add with-meta reader macro.
ocaml: remove metadata from symbols.
ruby.2: step9 was failing with REGRESS=1, the fix is now backported.
vb: prevent macro from mutating functions, allow (keyword :a).

Improve the logo implementation.
Encapsulate all representation in types.lg and env.lg, unwrap numbers.
Replace some manual iterations with logo control structures.
Reduce the diff between steps.
Use native iteration in env_get and env_map
Rewrite the reader with less temporary strings.
Reduce the number of temporary lists (for example, reverse iteration
with butlast requires O(n^2) allocations).
It seems possible to remove a few exceptions: GC settings
(Dockerfile), NO_SELF_HOSTING (IMPLS.yml) and step5_EXCLUDES
(Makefile.impls) .

IMPLS.yml

@@ -49,7 +49,7 @@ IMPL:
   - {IMPL: julia}
   - {IMPL: kotlin}
   - {IMPL: livescript}
-  - {IMPL: logo, NO_SELF_HOST: 1}  # step4 timeout
+  - {IMPL: logo}
   - {IMPL: lua}
   - {IMPL: make, NO_SELF_HOST: 1}  # step4 timeout
   - {IMPL: mal, MAL_IMPL: js,      BUILD_IMPL: js,  NO_SELF_HOST: 1}

Makefile.impls

@@ -43,7 +43,6 @@ IMPLS = ada ada.2 awk bash basic bbc-basic c c.2 chuck clojure coffee common-lis
 
 step5_EXCLUDES += bash        # never completes at 10,000
 step5_EXCLUDES += basic       # too slow, and limited to ints of 2^16
-step5_EXCLUDES += logo        # too slow for 10,000
 step5_EXCLUDES += make        # no TCO capability (iteration or recursion)
 step5_EXCLUDES += mal         # host impl dependent
 step5_EXCLUDES += matlab      # never completes at 10,000

impls/ada.2/envs.adb

@@ -53,6 +53,25 @@ package body Envs is
       return HM.Element (Position);
    end Get;
 
+   function Get_Or_Nil (Env : Instance;
+                        Key : Types.String_Ptr) return Types.T is
+      Position : HM.Cursor := Env.Data.Find (Key);
+      Ref      : Link;
+   begin
+      if not HM.Has_Element (Position) then
+         Ref := Env.Outer;
+         loop
+            if Ref = null then
+               return Types.Nil;
+            end if;
+            Position := Ref.all.Data.Find (Key);
+            exit when HM.Has_Element (Position);
+            Ref := Ref.all.Outer;
+         end loop;
+      end if;
+      return HM.Element (Position);
+   end Get_Or_Nil;
+
    procedure Keep_References (Object : in out Instance) is
    begin
       for Position in Object.Data.Iterate loop

impls/ada.2/envs.ads

@@ -27,6 +27,9 @@ package Envs is
    function Get (Env : in Instance;
                  Key : in Types.String_Ptr) return Types.T;
 
+   function Get_Or_Nil (Env : Instance;
+                        Key : Types.String_Ptr) return Types.T;
+
    procedure Set (Env      : in out Instance;
                   Key      : in     Types.T;
                   New_Item : in     Types.T) with Inline;

impls/ada.2/step2_eval.adb

@@ -1,4 +1,3 @@
-with Ada.Environment_Variables;
 with Ada.Containers.Indefinite_Hashed_Maps;
 with Ada.Strings.Hash;
 with Ada.Text_IO.Unbounded_IO;
@@ -14,8 +13,6 @@ with Types.Strings;
 
 procedure Step2_Eval is
 
-   Dbgeval : constant Boolean := Ada.Environment_Variables.Exists ("dbgeval");
-
    use type Types.T;
    use all type Types.Kind_Type;
 
@@ -52,12 +49,8 @@ procedure Step2_Eval is
    is
       First          : Types.T;
    begin
-      if Dbgeval then
-         Ada.Text_IO.New_Line;
-         Ada.Text_IO.Put ("EVAL: ");
-         Print (Ast);
-      end if;
-
+      --  Ada.Text_IO.Put ("EVAL: ");
+      --  Print (Ast);
       case Ast.Kind is
       when Kind_Nil | Kind_Atom | Kind_Boolean | Kind_Number | Types.Kind_Key
         | Kind_Macro | Types.Kind_Function =>

impls/ada.2/step3_env.adb

@@ -1,4 +1,3 @@
-with Ada.Environment_Variables;
 with Ada.Text_IO.Unbounded_IO;
 
 with Envs;
@@ -13,7 +12,7 @@ with Types.Strings;
 
 procedure Step3_Env is
 
-   Dbgeval : constant Boolean := Ada.Environment_Variables.Exists ("dbgeval");
+   Dbgeval : constant Types.String_Ptr := Types.Strings.Alloc ("DEBUG-EVAL");
 
    use type Types.T;
    use all type Types.Kind_Type;
@@ -45,8 +44,7 @@ procedure Step3_Env is
    is
       First          : Types.T;
    begin
-      if Dbgeval then
-         Ada.Text_IO.New_Line;
+      if Types.To_Boolean (Env.all.Get_Or_Nil (Dbgeval)) then
          Ada.Text_IO.Put ("EVAL: ");
          Print (Ast);
          Envs.Dump_Stack (Env.all);
@@ -209,6 +207,7 @@ begin
       --  Collect garbage.
       Err.Data := Types.Nil;
       Repl.all.Keep;
+      Dbgeval.Keep;
       Garbage_Collected.Clean;
    end loop;
    Ada.Text_IO.New_Line;

impls/ada.2/step4_if_fn_do.adb

@@ -1,4 +1,3 @@
-with Ada.Environment_Variables;
 with Ada.Text_IO.Unbounded_IO;
 
 with Core;
@@ -15,7 +14,7 @@ with Types.Strings;
 
 procedure Step4_If_Fn_Do is
 
-   Dbgeval : constant Boolean := Ada.Environment_Variables.Exists ("dbgeval");
+   Dbgeval : constant Types.String_Ptr := Types.Strings.Alloc ("DEBUG-EVAL");
 
    use type Types.T;
    use all type Types.Kind_Type;
@@ -47,8 +46,7 @@ procedure Step4_If_Fn_Do is
    is
       First          : Types.T;
    begin
-      if Dbgeval then
-         Ada.Text_IO.New_Line;
+      if Types.To_Boolean (Env.all.Get_Or_Nil (Dbgeval)) then
          Ada.Text_IO.Put ("EVAL: ");
          Print (Ast);
          Envs.Dump_Stack (Env.all);
@@ -81,17 +79,13 @@ procedure Step4_If_Fn_Do is
          if First.Str.all = "if" then
             Err.Check (Ast.Sequence.all.Length in 3 .. 4,
                        "expected 2 or 3 parameters");
-            declare
-               Tst : constant Types.T := Eval (Ast.Sequence.all.Data (2), Env);
-            begin
-               if Tst /= Types.Nil and Tst /= (Kind_Boolean, False) then
-                  return Eval (Ast.Sequence.all.Data (3), Env);
-               elsif Ast.Sequence.all.Length = 3 then
-                  return Types.Nil;
-               else
-                  return Eval (Ast.Sequence.all.Data (4), Env);
-               end if;
-            end;
+            if Types.To_Boolean (Eval (Ast.Sequence.all.Data (2), Env)) then
+               return Eval (Ast.Sequence.all.Data (3), Env);
+            elsif Ast.Sequence.all.Length = 3 then
+               return Types.Nil;
+            else
+               return Eval (Ast.Sequence.all.Data (4), Env);
+            end if;
          elsif First.Str.all = "let*" then
             Err.Check (Ast.Sequence.all.Length = 3
                and then Ast.Sequence.all.Data (2).Kind in Types.Kind_Sequence,
@@ -251,6 +245,7 @@ begin
       --  Collect garbage.
       Err.Data := Types.Nil;
       Repl.all.Keep;
+      Dbgeval.Keep;
       Garbage_Collected.Clean;
    end loop;
    Ada.Text_IO.New_Line;

impls/ada.2/step5_tco.adb

@@ -1,4 +1,3 @@
-with Ada.Environment_Variables;
 with Ada.Text_IO.Unbounded_IO;
 
 with Core;
@@ -15,7 +14,7 @@ with Types.Strings;
 
 procedure Step5_Tco is
 
-   Dbgeval : constant Boolean := Ada.Environment_Variables.Exists ("dbgeval");
+   Dbgeval : constant Types.String_Ptr := Types.Strings.Alloc ("DEBUG-EVAL");
 
    use type Types.T;
    use all type Types.Kind_Type;
@@ -56,8 +55,7 @@ procedure Step5_Tco is
       First          : Types.T;
    begin
       <<Restart>>
-      if Dbgeval then
-         Ada.Text_IO.New_Line;
+      if Types.To_Boolean (Env.all.Get_Or_Nil (Dbgeval)) then
          Ada.Text_IO.Put ("EVAL: ");
          Print (Ast);
          Envs.Dump_Stack (Env.all);
@@ -90,19 +88,15 @@ procedure Step5_Tco is
          if First.Str.all = "if" then
             Err.Check (Ast.Sequence.all.Length in 3 .. 4,
                        "expected 2 or 3 parameters");
-            declare
-               Tst : constant Types.T := Eval (Ast.Sequence.all.Data (2), Env);
-            begin
-               if Tst /= Types.Nil and Tst /= (Kind_Boolean, False) then
-                  Ast := Ast.Sequence.all.Data (3);
-                  goto Restart;
-               elsif Ast.Sequence.all.Length = 3 then
-                  return Types.Nil;
-               else
-                  Ast := Ast.Sequence.all.Data (4);
-                  goto Restart;
-               end if;
-            end;
+            if Types.To_Boolean (Eval (Ast.Sequence.all.Data (2), Env)) then
+               Ast := Ast.Sequence.all.Data (3);
+               goto Restart;
+            elsif Ast.Sequence.all.Length = 3 then
+               return Types.Nil;
+            else
+               Ast := Ast.Sequence.all.Data (4);
+               goto Restart;
+            end if;
          elsif First.Str.all = "let*" then
             Err.Check (Ast.Sequence.all.Length = 3
                and then Ast.Sequence.all.Data (2).Kind in Types.Kind_Sequence,
@@ -284,6 +278,7 @@ begin
       --  Collect garbage.
       Err.Data := Types.Nil;
       Repl.all.Keep;
+      Dbgeval.Keep;
       Garbage_Collected.Clean;
    end loop;
    Ada.Text_IO.New_Line;

impls/ada.2/step6_file.adb

@@ -1,5 +1,4 @@
 with Ada.Command_Line;
-with Ada.Environment_Variables;
 with Ada.Text_IO.Unbounded_IO;
 
 with Core;
@@ -16,7 +15,7 @@ with Types.Strings;
 
 procedure Step6_File is
 
-   Dbgeval : constant Boolean := Ada.Environment_Variables.Exists ("dbgeval");
+   Dbgeval : constant Types.String_Ptr := Types.Strings.Alloc ("DEBUG-EVAL");
 
    use type Types.T;
    use all type Types.Kind_Type;
@@ -60,8 +59,7 @@ procedure Step6_File is
       First          : Types.T;
    begin
       <<Restart>>
-      if Dbgeval then
-         Ada.Text_IO.New_Line;
+      if Types.To_Boolean (Env.all.Get_Or_Nil (Dbgeval)) then
          Ada.Text_IO.Put ("EVAL: ");
          Print (Ast);
          Envs.Dump_Stack (Env.all);
@@ -94,19 +92,15 @@ procedure Step6_File is
          if First.Str.all = "if" then
             Err.Check (Ast.Sequence.all.Length in 3 .. 4,
                        "expected 2 or 3 parameters");
-            declare
-               Tst : constant Types.T := Eval (Ast.Sequence.all.Data (2), Env);
-            begin
-               if Tst /= Types.Nil and Tst /= (Kind_Boolean, False) then
-                  Ast := Ast.Sequence.all.Data (3);
-                  goto Restart;
-               elsif Ast.Sequence.all.Length = 3 then
-                  return Types.Nil;
-               else
-                  Ast := Ast.Sequence.all.Data (4);
-                  goto Restart;
-               end if;
-            end;
+            if Types.To_Boolean (Eval (Ast.Sequence.all.Data (2), Env)) then
+               Ast := Ast.Sequence.all.Data (3);
+               goto Restart;
+            elsif Ast.Sequence.all.Length = 3 then
+               return Types.Nil;
+            else
+               Ast := Ast.Sequence.all.Data (4);
+               goto Restart;
+            end if;
          elsif First.Str.all = "let*" then
             Err.Check (Ast.Sequence.all.Length = 3
                and then Ast.Sequence.all.Data (2).Kind in Types.Kind_Sequence,
@@ -310,6 +304,7 @@ begin
          --  Collect garbage.
          Err.Data := Types.Nil;
          Repl.all.Keep;
+         Dbgeval.Keep;
          Garbage_Collected.Clean;
       end loop;
       Ada.Text_IO.New_Line;

impls/ada.2/step7_quote.adb

@@ -1,5 +1,4 @@
 with Ada.Command_Line;
-with Ada.Environment_Variables;
 with Ada.Text_IO.Unbounded_IO;
 
 with Core;
@@ -16,7 +15,7 @@ with Types.Strings;
 
 procedure Step7_Quote is
 
-   Dbgeval : constant Boolean := Ada.Environment_Variables.Exists ("dbgeval");
+   Dbgeval : constant Types.String_Ptr := Types.Strings.Alloc ("DEBUG-EVAL");
 
    use type Types.T;
    use all type Types.Kind_Type;
@@ -62,8 +61,7 @@ procedure Step7_Quote is
       First          : Types.T;
    begin
       <<Restart>>
-      if Dbgeval then
-         Ada.Text_IO.New_Line;
+      if Types.To_Boolean (Env.all.Get_Or_Nil (Dbgeval)) then
          Ada.Text_IO.Put ("EVAL: ");
          Print (Ast);
          Envs.Dump_Stack (Env.all);
@@ -96,19 +94,15 @@ procedure Step7_Quote is
          if First.Str.all = "if" then
             Err.Check (Ast.Sequence.all.Length in 3 .. 4,
                        "expected 2 or 3 parameters");
-            declare
-               Tst : constant Types.T := Eval (Ast.Sequence.all.Data (2), Env);
-            begin
-               if Tst /= Types.Nil and Tst /= (Kind_Boolean, False) then
-                  Ast := Ast.Sequence.all.Data (3);
-                  goto Restart;
-               elsif Ast.Sequence.all.Length = 3 then
-                  return Types.Nil;
-               else
-                  Ast := Ast.Sequence.all.Data (4);
-                  goto Restart;
-               end if;
-            end;
+            if Types.To_Boolean (Eval (Ast.Sequence.all.Data (2), Env)) then
+               Ast := Ast.Sequence.all.Data (3);
+               goto Restart;
+            elsif Ast.Sequence.all.Length = 3 then
+               return Types.Nil;
+            else
+               Ast := Ast.Sequence.all.Data (4);
+               goto Restart;
+            end if;
          elsif First.Str.all = "let*" then
             Err.Check (Ast.Sequence.all.Length = 3
                and then Ast.Sequence.all.Data (2).Kind in Types.Kind_Sequence,
@@ -167,9 +161,6 @@ procedure Step7_Quote is
                   Ast    => Ast.Sequence.all.Data (3),
                   Env    => Env));
             end;
-         elsif First.Str.all = "quasiquoteexpand" then
-            Err.Check (Ast.Sequence.all.Length = 2, "expected 1 parameter");
-            return Quasiquote (Ast.Sequence.all.Data (2));
          elsif First.Str.all = "quasiquote" then
             Err.Check (Ast.Sequence.all.Length = 2, "expected 1 parameter");
             Ast := Quasiquote (Ast.Sequence.all.Data (2));
@@ -379,6 +370,7 @@ begin
          --  Collect garbage.
          Err.Data := Types.Nil;
          Repl.all.Keep;
+         Dbgeval.Keep;
          Garbage_Collected.Clean;
       end loop;
       Ada.Text_IO.New_Line;