Merge pull request #5323 from OceanOak/grammar-updates-4

Update the grammar to support characters
darklang · Mar 13, 2024 · b0b7d65 · b0b7d65
2 parents 0a7fcff + 56ee486
commit b0b7d65
Show file tree

Hide file tree

Showing 12 changed files with 4,209 additions and 3,437 deletions.
diff --git a/backend/testfiles/execution/stdlib/parser.dark b/backend/testfiles/execution/stdlib/parser.dark
@@ -264,6 +264,11 @@ module TextToTextRoundtripping =
     ("\"hello\"" |> roundtripCliScript) = "\"hello\""
     ("\"hello\\tworld\"" |> roundtripCliScript) = "\"hello\\tworld\""
 
+    // char literals
+    ("'a'" |> roundtripCliScript) = "'a'"
+    ("'\\n'" |> roundtripCliScript) = "'\\n'"
+    ("'\t'" |> roundtripCliScript) = "'\t'"
+
     // variables and let bindings
     ("assumedlyAVariableName" |> roundtripCliScript) = "assumedlyAVariableName"
     // TODO: this is ugly

diff --git a/packages/darklang/languageTools/parser.dark b/packages/darklang/languageTools/parser.dark
@@ -580,6 +580,39 @@ module Darklang =
             (WrittenTypes.Unparseable { source = node })
             |> Stdlib.Result.Result.Error
 
+        let parseCharLiteral
+          (node: ParsedNode)
+          : Stdlib.Result.Result<WrittenTypes.Expr, WrittenTypes.Unparseable> =
+          if node.typ == "char_literal" then
+            let openQuoteNode =
+              (findNodeByFieldName node "symbol_open_single_quote")
+              |> Stdlib.Option.toResult
+                "No symbol_open_single_quote node found in char_literal"
+
+            let charNode =
+              match findNodeByFieldName node "content" with
+              | Some charPart ->
+                Stdlib.Option.Option.Some((charPart.sourceRange, charPart.text))
+              | None -> Stdlib.Option.Option.None
+
+            let closeQuoteNode =
+              (findNodeByFieldName node "symbol_close_single_quote")
+              |> Stdlib.Option.toResult
+                "No symbol_close_single_quote node found in char_literal"
+
+            match openQuoteNode, closeQuoteNode with
+            | Ok openQuoteNode, Ok closeQuoteNode ->
+              (WrittenTypes.Expr.EChar(
+                node.sourceRange,
+                charNode,
+                openQuoteNode.sourceRange,
+                closeQuoteNode.sourceRange
+              ))
+              |> Stdlib.Result.Result.Ok
+
+            | _ ->
+              (WrittenTypes.Unparseable { source = node })
+              |> Stdlib.Result.Result.Error
 
         let parseLetExpr
           (node: ParsedNode)
@@ -804,6 +837,7 @@ module Darklang =
           | "uint128_literal" -> parseIntLiteral node
           | "float_literal" -> parseFloatLiteral node
           | "string_literal" -> parseStringLiteral node
+          | "char_literal" -> parseCharLiteral node
 
           // assigning and accessing variables
           | "let_expression" -> parseLetExpr node

diff --git a/packages/darklang/languageTools/semanticTokens.dark b/packages/darklang/languageTools/semanticTokens.dark
@@ -231,6 +231,18 @@ module Darklang =
               [ makeToken symbolCloseQuote TokenType.Symbol ] ]
             |> Stdlib.List.flatten
 
+          // 'a'
+          | EChar(range, _contentsMaybe, symbolOpenQuote, symbolCloseQuote) ->
+            [ // '
+              [ makeToken symbolOpenQuote TokenType.Symbol ]
+
+              // a
+              [ makeToken range TokenType.String ]
+
+              // '
+              [ makeToken symbolCloseQuote TokenType.Symbol ] ]
+            |> Stdlib.List.flatten
+
           // let x = 2
           // x + 1
           | ELet(range, lp, expr, body, keywordLet, symbolEquals) ->

diff --git a/packages/darklang/languageTools/writtenTypes.dark b/packages/darklang/languageTools/writtenTypes.dark
@@ -147,6 +147,11 @@ module Darklang =
           contents: Stdlib.Option.Option<SourceRange * String> *
           symbolOpenQuote: SourceRange *
           symbolCloseQuote: SourceRange
+        | EChar of
+          SourceRange *
+          contents: Stdlib.Option.Option<SourceRange * String> *
+          symbolOpenQuote: SourceRange *
+          symbolCloseQuote: SourceRange
 
         | ELet of
           SourceRange *

diff --git a/packages/darklang/languageTools/writtenTypesToProgramTypes.dark b/packages/darklang/languageTools/writtenTypesToProgramTypes.dark
@@ -206,6 +206,10 @@ module Darklang =
                 gid (),
                 [ ProgramTypes.StringSegment.StringText s ]
               )
+          | EChar(_, c, _, _) ->
+            match c with
+            | None -> ProgramTypes.Expr.EChar(gid (), "")
+            | Some((_, c)) -> ProgramTypes.Expr.EChar(gid (), c)
 
           // declaring and accessing variables
           | ELet(_, pat, rhs, body, _, _) ->

diff --git a/packages/darklang/prettyPrinter/programTypes.dark b/packages/darklang/prettyPrinter/programTypes.dark
@@ -718,7 +718,7 @@ module Darklang =
           $"{signPart}{whole}.{remainderPart}"
 
 
-        | EChar(_id, c) -> "\"" ++ c ++ "\""
+        | EChar(_id, c) -> "'" ++ c ++ "'"
 
         | EString(_id, segments) ->
           match segments with

diff --git a/tree-sitter-darklang/grammar.js b/tree-sitter-darklang/grammar.js
@@ -82,6 +82,7 @@ module.exports = grammar({
         $.uint128_literal,
         $.float_literal,
         $.string_literal,
+        $.char_literal,
 
         $.let_expression,
         $.variable_identifier,
@@ -129,6 +130,7 @@ module.exports = grammar({
 
     //
     // Strings
+    // TODO: maybe add support for multiline strings (""")
     string_literal: $ =>
       choice(
         seq(
@@ -146,12 +148,30 @@ module.exports = grammar({
         choice(
           // higher precedence than escape sequences
           token.immediate(prec(1, /[^\\"\n]+/)),
-          $.string_escape_sequence,
+          $.char_or_string_escape_sequence,
         ),
       ),
-    string_escape_sequence: _ =>
+
+    //
+    // Characters
+    char_literal: $ =>
+      seq(
+        field("symbol_open_single_quote", alias("'", $.symbol)),
+        field("content", $.character),
+        field("symbol_close_single_quote", alias("'", $.symbol)),
+      ),
+    character: $ =>
+      choice(
+        // higher precedence than escape sequences
+        token.immediate(prec(1, /[^'\\\n]/)),
+        $.char_or_string_escape_sequence,
+      ),
+
+    char_or_string_escape_sequence: _ =>
       token.immediate(seq("\\", /(\"|\\|\/|b|f|n|r|t|u)/)),
 
+    //
+    // Infix operations
     infix_operation: $ =>
       // given `1 + 2 * 3`, this will parse as `1 + (2 * 3)`
       choice(
@@ -214,6 +234,8 @@ module.exports = grammar({
         ),
       ),
 
+    //
+    // Integers
     //CLEANUP: we are using .NET suffixes for integers (e.g. `1L` for Int64) temporarily until we remove the old parser.
     int8_literal: $ =>
       seq(field("digits", $.digits), field("suffix", alias("y", $.symbol))),
@@ -260,11 +282,14 @@ module.exports = grammar({
         field("suffix", alias("Z", $.symbol)),
       ),
 
-    float_literal: $ => /[+-]?[0-9]+\.[0-9]+/,
     digits: $ => choice($.positive_digits, $.negative_digits),
     negative_digits: $ => /-\d+/,
     positive_digits: $ => /\d+/,
 
+    //
+    // Floats
+    float_literal: $ => /[+-]?[0-9]+\.[0-9]+/,
+
     //
     // Common
     type_reference: $ => choice($.builtin_type, $.qualified_type_name),

diff --git a/tree-sitter-darklang/src/grammar.json b/tree-sitter-darklang/src/grammar.json
@@ -291,6 +291,10 @@
           "type": "SYMBOL",
           "name": "string_literal"
         },
+        {
+          "type": "SYMBOL",
+          "name": "char_literal"
+        },
         {
           "type": "SYMBOL",
           "name": "let_expression"
@@ -563,12 +567,71 @@
           },
           {
             "type": "SYMBOL",
-            "name": "string_escape_sequence"
+            "name": "char_or_string_escape_sequence"
           }
         ]
       }
     },
-    "string_escape_sequence": {
+    "char_literal": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "FIELD",
+          "name": "symbol_open_single_quote",
+          "content": {
+            "type": "ALIAS",
+            "content": {
+              "type": "STRING",
+              "value": "'"
+            },
+            "named": true,
+            "value": "symbol"
+          }
+        },
+        {
+          "type": "FIELD",
+          "name": "content",
+          "content": {
+            "type": "SYMBOL",
+            "name": "character"
+          }
+        },
+        {
+          "type": "FIELD",
+          "name": "symbol_close_single_quote",
+          "content": {
+            "type": "ALIAS",
+            "content": {
+              "type": "STRING",
+              "value": "'"
+            },
+            "named": true,
+            "value": "symbol"
+          }
+        }
+      ]
+    },
+    "character": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "IMMEDIATE_TOKEN",
+          "content": {
+            "type": "PREC",
+            "value": 1,
+            "content": {
+              "type": "PATTERN",
+              "value": "[^'\\\\\\n]"
+            }
+          }
+        },
+        {
+          "type": "SYMBOL",
+          "name": "char_or_string_escape_sequence"
+        }
+      ]
+    },
+    "char_or_string_escape_sequence": {
       "type": "IMMEDIATE_TOKEN",
       "content": {
         "type": "SEQ",
@@ -1142,10 +1205,6 @@
         }
       ]
     },
-    "float_literal": {
-      "type": "PATTERN",
-      "value": "[+-]?[0-9]+\\.[0-9]+"
-    },
     "digits": {
       "type": "CHOICE",
       "members": [
@@ -1167,6 +1226,10 @@
       "type": "PATTERN",
       "value": "\\d+"
     },
+    "float_literal": {
+      "type": "PATTERN",
+      "value": "[+-]?[0-9]+\\.[0-9]+"
+    },
     "type_reference": {
       "type": "CHOICE",
       "members": [

diff --git a/tree-sitter-darklang/src/node-types.json b/tree-sitter-darklang/src/node-types.json
@@ -9,6 +9,57 @@
     "named": true,
     "fields": {}
   },
+  {
+    "type": "char_literal",
+    "named": true,
+    "fields": {
+      "content": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "character",
+            "named": true
+          }
+        ]
+      },
+      "symbol_close_single_quote": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "symbol",
+            "named": true
+          }
+        ]
+      },
+      "symbol_open_single_quote": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "symbol",
+            "named": true
+          }
+        ]
+      }
+    }
+  },
+  {
+    "type": "character",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": false,
+      "types": [
+        {
+          "type": "char_or_string_escape_sequence",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "digits",
     "named": true,
@@ -40,6 +91,10 @@
           "type": "bool_literal",
           "named": true
         },
+        {
+          "type": "char_literal",
+          "named": true
+        },
         {
           "type": "float_literal",
           "named": true
@@ -658,7 +713,7 @@
       "required": false,
       "types": [
         {
-          "type": "string_escape_sequence",
+          "type": "char_or_string_escape_sequence",
           "named": true
         }
       ]
@@ -909,6 +964,10 @@
     "type": "\n",
     "named": false
   },
+  {
+    "type": "char_or_string_escape_sequence",
+    "named": true
+  },
   {
     "type": "float_literal",
     "named": true
@@ -929,10 +988,6 @@
     "type": "positive_digits",
     "named": true
   },
-  {
-    "type": "string_escape_sequence",
-    "named": true
-  },
   {
     "type": "symbol",
     "named": true