check point

VisionBuilder.UI.Recipes.Scripted/Tokenizer.cs

@@ -0,0 +1,143 @@
+using System.Text.RegularExpressions;
+
+namespace VisionBuilder.UI.Recipes.Scripted;
+
+public enum ETokenType
+{
+    EOF,
+    UnexpectedToken,
+    Identifier,
+    String,
+
+
+    OpenParenthesis,
+    CloseParenthesis
+}
+public class Token
+{
+    public Token(ETokenType type, string value, int line, int column, int start, int end)
+    {
+        Type = type;
+        Value = value;
+        Line = line;
+        Column = column;
+        Start = start;
+        End = end;
+    }
+
+
+
+    public ETokenType Type { get; set; }
+    public string Value { get; set; }
+    public int Line { get; set; }
+    public int Column { get; set; }
+    public int Start { get; set; }
+    public int End { get; set; }
+
+    public override string ToString()
+    {
+        return string.Format("{0} ({1})", Value, Type);
+    }
+
+
+}
+public class Tokenizer
+{
+
+    public int Position { get; private set; }
+    public string Code { get; private set; }
+
+    public Tokenizer(string code)
+    {
+        Code = code;
+    }
+
+    private Dictionary<string, ETokenType?> _specs = new()
+    {
+        // comment
+        {@"^\/\/[^\n]*",null},
+        // multi-line comment
+        {@"^\/\*.*\*\/",null},
+        // whitespace
+        {@"^\s+",null},
+
+        {"^\"(\\\"|[^\"])*'",ETokenType.String},
+
+        {@"^\(",ETokenType.OpenParenthesis},
+        {@"^\)",ETokenType.CloseParenthesis},
+        {@"^[a-zA-Z_][a-zA-Z0-9_]*",ETokenType.Identifier},
+    };
+
+
+    private int GetCurrentLine()
+    {
+        var line = 1;
+        for (var i = 0; i < Position; i++)
+        {
+            if (Code[i] == '\n')
+                line++;
+        }
+
+        return line;
+
+    }
+
+    private int GetCurrentColumn()
+    {
+        var column = 1;
+        for (var i = 0; i < Position; i++)
+        {
+            if (Code[i] == '\n')
+                column = 1;
+            else
+                column++;
+        }
+
+        return column;
+    }
+
+    public bool HasNextToken()
+    {
+        return Position < Code.Length;
+    }
+    public Token NextToken()
+    {
+        if (!HasNextToken())
+            return new Token(ETokenType.EOF, string.Empty, GetCurrentLine(), GetCurrentColumn(), Position, Position);
+        var line = GetCurrentLine();
+        var column = GetCurrentColumn();
+        if (column == 1)
+        {
+
+            if (Regex.IsMatch(Code.Substring(Position), @"^\s*\*"))
+            {
+                var match = Regex.Match(Code.Substring(Position), @"^\s*\*[^\n]*");
+                Position += match.Length;
+                return NextToken();
+            }
+        }
+        foreach (var spec in _specs)
+        {
+            var match = Regex.Match(Code.Substring(Position), spec.Key);
+            if (match.Success)
+            {
+
+                var start = Position;
+                Position += match.Length;
+                if (spec.Value.HasValue)
+                {
+                    if (spec.Value.Value == ETokenType.String)
+                    {
+                        return new Token(spec.Value.Value, Regex.Unescape(match.Value.Substring(1, match.Value.Length - 2)), line, column, start, Position);
+                    }
+                    return new Token(spec.Value.Value, match.Value, line, column, start, Position);
+                }
+
+                else
+                    return NextToken();
+            }
+        }
+
+        return new Token(ETokenType.UnexpectedToken, Code.Substring(Position, 1), line, column, Position, Position + 1);
+    }
+}