From 0a3771fb5b6722a934cb3422e3ee01099a1ba6a2 Mon Sep 17 00:00:00 2001 From: Kana <109441902+ai-kana@users.noreply.github.com> Date: Tue, 5 Nov 2024 01:53:07 -0800 Subject: [PATCH 1/2] Add negative and decimal literal support --- GDWeave/Script/ScriptTokenizer.cs | 152 +++++++++++++++++++++--------- 1 file changed, 108 insertions(+), 44 deletions(-) diff --git a/GDWeave/Script/ScriptTokenizer.cs b/GDWeave/Script/ScriptTokenizer.cs index 571349a..65d607e 100644 --- a/GDWeave/Script/ScriptTokenizer.cs +++ b/GDWeave/Script/ScriptTokenizer.cs @@ -30,9 +30,7 @@ public static class ScriptTokenizer { {"setget", TokenType.PrSetget}, {"static", TokenType.PrStatic}, - {"void", TokenType.PrVoid}, - {"enum", TokenType.PrEnum}, - + {"void", TokenType.PrVoid}, {"enum", TokenType.PrEnum}, {"preload", TokenType.PrPreload}, {"assert", TokenType.PrAssert}, @@ -182,83 +180,149 @@ public static class ScriptTokenizer { private static readonly List BuiltinFunctions = Enum.GetNames().ToList(); + private static void InsertNewLine(IEnumerator enumerator, uint baseIndent, List toFlush) { + if (!enumerator.MoveNext()) { + return; + } + + var tabCount = uint.Parse(enumerator.Current); + toFlush.Add(new Token(TokenType.Newline, tabCount + baseIndent)); + } + + private static void BuildIdentifierName(IEnumerator enumerator, List toFlush, out string? found) { + found = string.Empty; + if (!enumerator.MoveNext()) { + return; + } + + if (enumerator.Current == ":") { + toFlush.Add(new Token(TokenType.Wildcard)); + toFlush.Add(new Token(TokenType.Semicolon)); + return; + } + + found = "_" + enumerator.Current; + } + + private static void BuildNumber(IEnumerator enumerator, List toFlush) { + int sign = 1; + + if (enumerator.Current == "-") { + sign = -1; + if (!enumerator.MoveNext()) return; + } + + if (!long.TryParse(enumerator.Current, out long upper)) { + toFlush.Add(new Token(TokenType.OpSub)); + return; + } + + if (!enumerator.MoveNext()) return; + + if (enumerator.Current != ".") { + toFlush.Add(new ConstantToken(new IntVariant(upper * sign))); + return; + } + + if (!enumerator.MoveNext()) return; + + if (!long.TryParse(enumerator.Current, out long lower)) { + // I dont think there is really a proper return for here. + // You'd have a number that looks like this "1000." + // No following decimal + // Comment if you had ideas + return; + } + + var result = upper + (lower / Math.Pow(10, lower.ToString().Length)); + toFlush.Add(new ConstantToken(new RealVariant(result * sign))); + } + public static IEnumerable Tokenize(string gdScript, uint baseIndent = 0) { + var finalTokens = new List(); var tokens = SanitizeInput(TokenizeString(gdScript + " ")); var previous = string.Empty; var idName = string.Empty; - List toFlush = new(2); - yield return new Token(TokenType.Newline, baseIndent); - foreach (var current in tokens) { + var toFlush = new List(2); + finalTokens.Add(new Token(TokenType.Newline, baseIndent)); + var enumerator = tokens.GetEnumerator(); + while (enumerator.MoveNext()) { + var current = enumerator.Current; if (current == "\n") { - goto endAndFlushId; - } - - if (previous == "\n") { - var tabCount = uint.Parse(current); - toFlush.Add(new Token(TokenType.Newline, tabCount + baseIndent)); - goto end; + InsertNewLine(enumerator, baseIndent, toFlush); + endAndFlushId(); + continue; } if (current == "_") { - goto end; + BuildIdentifierName(enumerator, toFlush, out string? found); + if (found == string.Empty) { + endAndFlushId(); + continue; + } + + idName += found; + + end(); + continue; } - if (previous == "_" && current == ":") { - toFlush.Add(new Token(TokenType.Wildcard)); - toFlush.Add(new Token(TokenType.Semicolon)); - goto endAndFlushId; - } else if (previous == "_") { - idName += "_" + current; - goto end; + if (current == "-" || char.IsDigit(current[0])) { + BuildNumber(enumerator, toFlush); + endAndFlushId(); + continue; } if (BuiltinFunctions.Contains(current)) { toFlush.Add(new Token(TokenType.BuiltInFunc, (uint?) BuiltinFunctions.IndexOf(current))); - goto endAndFlushId; + endAndFlushId(); + continue; } if (Tokens.TryGetValue(current, out var type)) { toFlush.Add(new Token(type)); - goto endAndFlushId; + endAndFlushId(); + continue; } - if (current[0] == '"') { + if (current.StartsWith('"')) { toFlush.Add(new ConstantToken(new StringVariant(current.Substring(1, current.Length - 2)))); - goto endAndFlushId; + endAndFlushId(); + continue; } if (bool.TryParse(current, out var resultB)) { toFlush.Add(new ConstantToken(new BoolVariant(resultB))); - goto endAndFlushId; + endAndFlushId(); + continue; } - if (long.TryParse(current, out var resultL)) { - toFlush.Add(new ConstantToken(new IntVariant(resultL))); - goto endAndFlushId; - } + idName += current; - if (double.TryParse(current, out var result)) { - toFlush.Add(new ConstantToken(new RealVariant(result))); - goto endAndFlushId; + end(); + + void end() { + previous = enumerator.Current; + finalTokens.AddRange(toFlush); + toFlush.Clear(); } - idName += current; + void endAndFlushId() { + if (idName != string.Empty) { + finalTokens.Add(new IdentifierToken(idName)); + idName = string.Empty; + } - goto end; - endAndFlushId: - if (idName != string.Empty) { - yield return new IdentifierToken(idName); - idName = string.Empty; + end(); } - end: - previous = current; - foreach (var token in toFlush) yield return token; - toFlush.Clear(); } - yield return new(TokenType.Newline, baseIndent); + finalTokens.Add(new(TokenType.Newline, baseIndent)); + + foreach (var t in finalTokens) yield return t; + } private static IEnumerable SanitizeInput(IEnumerable tokens) { From e93f86dd1076c9f1dd265ebaae235f58656b73b7 Mon Sep 17 00:00:00 2001 From: Kana <109441902+ai-kana@users.noreply.github.com> Date: Tue, 5 Nov 2024 01:54:20 -0800 Subject: [PATCH 2/2] Fix J --- GDWeave/Script/ScriptTokenizer.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/GDWeave/Script/ScriptTokenizer.cs b/GDWeave/Script/ScriptTokenizer.cs index 65d607e..b87ecec 100644 --- a/GDWeave/Script/ScriptTokenizer.cs +++ b/GDWeave/Script/ScriptTokenizer.cs @@ -30,7 +30,8 @@ public static class ScriptTokenizer { {"setget", TokenType.PrSetget}, {"static", TokenType.PrStatic}, - {"void", TokenType.PrVoid}, {"enum", TokenType.PrEnum}, + {"void", TokenType.PrVoid}, + {"enum", TokenType.PrEnum}, {"preload", TokenType.PrPreload}, {"assert", TokenType.PrAssert},