Tree-sitter grammar: add support for module declaration

backend/tests/Tests/NewParser.Tests.fs

@@ -768,6 +768,46 @@ let functionDeclarations =
       "let sum (a: Int64) (b: Int64): Int64 =\n  PACKAGE.Darklang.Stdlib.Int64.add a b" ]
   |> testList "function declarations"
 
+let moduleDeclarations =
+  [ t
+      "simple module"
+      "module MyModule =\n  type ID = Int64"
+      "module MyModule =\n  type ID =\n    Int64"
+
+    t
+      "module with types, fns, and consts"
+      """module MyModule =
+  type ID = Int64
+  type MyString = String
+  let myFn (i: Int64): Int64 = 1L
+  const x = 100L"""
+      "module MyModule =\n  type ID =\n    Int64\n\n  type MyString =\n    String\n\n  let myFn (i: Int64): Int64 =\n    1L\n\n  const x = 100L"
+
+    t
+      "module with types, fns, conts, and newlines"
+      """module MyModule =
+  type ID = Int64
+
+  type MyString = String
+
+  let myFn (i: Int64): Int64 = 1L
+
+  const x = 100L"""
+      "module MyModule =\n  type ID =\n    Int64\n\n  type MyString =\n    String\n\n  let myFn (i: Int64): Int64 =\n    1L\n\n  const x = 100L"
+
+    t
+      "nested module declaration"
+      """module MyModule1 =
+  type ID = Int64
+  module MyModule2 =
+    type ID = Int64
+    module MyModule3 =
+      type ID = Int64
+      const x = 100L
+      1L"""
+      "module MyModule1 =\n  type ID =\n    Int64\n\n  module MyModule2 =\n    type ID =\n      Int64\n\n    module MyModule3 =\n      type ID =\n        Int64\n\n      const x = 100L\n\n      1L" ]
+  |> testList "module declarations"
+
 
 let cliScripts =
   [
@@ -806,4 +846,5 @@ let tests =
       constantDeclarations
       exprs
       functionDeclarations
+      moduleDeclarations
       cliScripts ]

packages/darklang/languageTools/parser/cliScript.dark

@@ -51,6 +51,15 @@ module Darklang =
               { result with
                   exprsToEval = Stdlib.List.pushBack result.exprsToEval parsedExpr }
 
+          | "module_decl" ->
+            match ModuleDeclaration.parse decl with
+            | Error e -> updateUnparseableStuff result e
+            | Ok parsedModuleDecl ->
+              let newModule =
+                WrittenTypes.CliScriptDeclaration.Module parsedModuleDecl
+
+              { result with
+                  declarations = Stdlib.List.pushBack result.declarations newModule }
           | _ ->
             updateUnparseableStuff
               result

packages/darklang/languageTools/parser/moduleDeclaration.dark

@@ -0,0 +1,60 @@
+module Darklang =
+  module LanguageTools =
+    module Parser =
+      module ModuleDeclaration =
+        let parseDecls
+          (node: ParsedNode)
+          : Stdlib.Result.Result<WrittenTypes.ModuleDeclaration.Declaration, WrittenTypes.Unparseable> =
+          match node.typ with
+          | "type_decl" ->
+            (TypeDeclaration.parse node)
+            |> Stdlib.Result.map (fun x ->
+              WrittenTypes.ModuleDeclaration.Declaration.Type x)
+          | "fn_decl" ->
+            (FunctionDeclaration.parse node)
+            |> Stdlib.Result.map (fun x ->
+              WrittenTypes.ModuleDeclaration.Declaration.Function x)
+          | "const_decl" ->
+            (ConstantDeclaration.parse node)
+            |> Stdlib.Result.map (fun x ->
+              WrittenTypes.ModuleDeclaration.Declaration.Constant x)
+          | "module_decl" ->
+            (ModuleDeclaration.parse node)
+            |> Stdlib.Result.map (fun x ->
+              WrittenTypes.ModuleDeclaration.Declaration.SubModule x)
+          | "expression" ->
+            (Expr.parse node)
+            |> Stdlib.Result.map (fun x ->
+              WrittenTypes.ModuleDeclaration.Declaration.Expr x)
+          | _ -> createUnparseableError node
+
+        let parse
+          (node: ParsedNode)
+          : Stdlib.Result.Result<WrittenTypes.ModuleDeclaration.ModuleDeclaration, WrittenTypes.Unparseable> =
+          if node.typ == "module_decl" then
+            let keywordModule = findField node "keyword_module"
+            let name = (findField node "name") |> Builtin.unwrap
+
+            let declarations =
+              (Stdlib.List.filter node.children (fun node ->
+                node.typ == "module_content"))
+              |> Stdlib.List.map (fun node ->
+                node.children
+                |> Stdlib.List.head
+                |> Builtin.unwrap
+                |> ModuleDeclaration.parseDecls)
+              |> Stdlib.Result.collect
+
+            match keywordModule, declarations with
+            | Ok keywordModule, Ok declarations ->
+              (WrittenTypes.ModuleDeclaration.ModuleDeclaration
+                { range = node.range
+                  name = (name.range, name.text)
+                  declarations = declarations
+                  keywordModule = keywordModule.range })
+              |> Stdlib.Result.Result.Ok
+
+            | _ -> createUnparseableError node
+
+          else
+            createUnparseableError node

packages/darklang/languageTools/programTypes.dark

@@ -403,6 +403,12 @@ module Darklang =
           constants: List<PackageConstant.PackageConstant>
           fns: List<PackageFn.PackageFn> }
 
+      type Definitions =
+        { types: List<PackageType.PackageType>
+          constants: List<PackageConstant.PackageConstant>
+          fns: List<PackageFn.PackageFn>
+          // TODO db and handlers
+          exprs: List<Expr * List<String>> }
 
       // User/Canvas space
       type DB =
@@ -451,6 +457,7 @@ module Darklang =
           | Type of PackageType.PackageType
           | Constant of PackageConstant.PackageConstant
           | Function of PackageFn.PackageFn
+          | Module of Definitions
 
         type CliScript =
           { declarations: List<Declaration>

packages/darklang/languageTools/semanticTokens.dark

@@ -212,6 +212,30 @@ module Darklang =
           | QualifiedName q -> QualifiedTypeIdentifier.tokenize q
 
 
+      module ModuleDeclaration =
+        let tokenizeDeclaration
+          (d: WrittenTypes.ModuleDeclaration.Declaration)
+          : List<SemanticToken> =
+          match d with
+          | Type t -> TypeDeclaration.tokenize t
+          | Function f -> FnDeclaration.tokenize f
+          | Constant c -> ConstantDeclaration.tokenize c
+          | SubModule sm -> ModuleDeclaration.tokenize sm
+          | Expr e -> Expr.tokenize e
+
+        let tokenize
+          (m: WrittenTypes.ModuleDeclaration.ModuleDeclaration)
+          : List<SemanticToken> =
+          let (nameRange, name) = m.name
+
+          [ [ makeToken m.keywordModule TokenType.Keyword ]
+            [ makeToken nameRange TokenType.ModuleName ]
+            (m.declarations
+             |> Stdlib.List.map (fun d -> ModuleDeclaration.tokenizeDeclaration d)
+             |> Stdlib.List.flatten) ]
+          |> Stdlib.List.flatten
+
+
       module TypeDeclaration =
         let tokenizeDefinition
           (d: WrittenTypes.TypeDeclaration.Definition)
@@ -1028,6 +1052,7 @@ module Darklang =
                |> Stdlib.List.flatten) ]
             |> Stdlib.List.flatten
 
+
       module Const =
         let tokenize (c: WrittenTypes.Const) : List<SemanticToken> =
           match c with
@@ -1214,7 +1239,8 @@ module Darklang =
                 match typeOrFn with
                 | Type t -> TypeDeclaration.tokenize t
                 | Function fn -> FnDeclaration.tokenize fn
-                | Constant c -> ConstantDeclaration.tokenize c)
+                | Constant c -> ConstantDeclaration.tokenize c
+                | Module m -> ModuleDeclaration.tokenize m)
               |> Stdlib.List.flatten
 
             let exprsPart =

packages/darklang/languageTools/writtenTypes.dark

@@ -437,6 +437,24 @@ module Darklang =
           fields: List<Const> *
           symbolDot: Range
 
+
+      // Module declarations
+      module ModuleDeclaration =
+        type Declaration =
+          | Type of TypeDeclaration.TypeDeclaration
+          | Function of FnDeclaration.FnDeclaration
+          | Constant of ConstantDeclaration.ConstantDeclaration
+          | SubModule of ModuleDeclaration.ModuleDeclaration
+          // TODO: handle http handler and db
+          | Expr of Expr
+
+        type ModuleDeclaration =
+          { range: Range
+            name: Range * String
+            declarations: List<Declaration>
+            keywordModule: Range }
+
+
       // Fn declarations
       module FnDeclaration =
         type UnitParameter = { range: Range }
@@ -475,15 +493,17 @@ module Darklang =
             keywordConst: Range
             symbolEquals: Range }
 
-      type Unparseable =
-        { source: Parser.ParsedNode
-          note: Stdlib.Option.Option<String> }
-
       // Cli scripts (CLEANUP throw both of these types in a CliScript module)
       type CliScriptDeclaration =
         | Type of TypeDeclaration.TypeDeclaration
         | Function of FnDeclaration.FnDeclaration
         | Constant of ConstantDeclaration.ConstantDeclaration
+        | Module of ModuleDeclaration.ModuleDeclaration
+
+      type Unparseable =
+        { source: Parser.ParsedNode
+          note: Stdlib.Option.Option<String> }
+
 
       type CliScript =
         { range: Range