//============= Copyright (c) Ludic GmbH, All rights reserved. ============== // // Purpose: Part of the My Behaviour Tree Code // //============================================================================= using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using UnityEngine; namespace MyBT { public enum SyntaxLabel { Undefined, // math operators Plus, Minus, SingleLineCommentStart, MultiLineCommentStart, MultiLineCommentEnd, Equal, EOL, QuoteToken, Colon, Semicolon, Comma, Questionmark, Whitespace, // generic functions and variable tokens Variable, Function, // behaviour tree tokens TreeNameKeyword, RunTreeNameKeyword, CompositeNameKeyword, ActionNameKeyword, DecoratorNameKeyword, // Literal Values TrueKeyword, FalseKeyword, NumberSymbol, CharSymbol, StringSymbol, OpenNormalBracket, CloseNormalBracket, OpenCurlyBracket, CloseCurlyBracket, OpenSquareBracket, CloseSquareBracket, } public enum PatternLabel { CharPattern, StringPattern, NumberCharPattern, NumberPattern, WhitespacePattern, NewlinePattern } public class Tokenizer { // TaskController taskController; private static readonly Dictionary keywords = new Dictionary() { {"True", SyntaxLabel.TrueKeyword }, {"False", SyntaxLabel.FalseKeyword }, {"Tree", SyntaxLabel.TreeNameKeyword }, {"Composite", SyntaxLabel.CompositeNameKeyword }, {"Action", SyntaxLabel.ActionNameKeyword }, {"Decorator", SyntaxLabel.DecoratorNameKeyword }, {"RunTree", SyntaxLabel.RunTreeNameKeyword } }; private static readonly Dictionary patterns = new Dictionary() { {PatternLabel.CharPattern, new Regex("[\\$a-zA-Z\\._]") }, {PatternLabel.StringPattern, new Regex("[^\"]") }, {PatternLabel.NumberPattern, new Regex("[-0-9\\.]") }, {PatternLabel.NumberCharPattern, new Regex("[-[0-9a-zA-Z\\._]") }, {PatternLabel.WhitespacePattern, new Regex("[ \t]") }, {PatternLabel.NewlinePattern, new Regex("\r\n|\n") } }; public static readonly Dictionary Definitions = new Dictionary() { { SyntaxLabel.Colon, new Regex(":") }, { SyntaxLabel.Semicolon, new Regex(";") }, { SyntaxLabel.Minus, new Regex("\\-") }, { SyntaxLabel.Plus, new Regex("\\+") }, { SyntaxLabel.Comma, new Regex(",") }, { SyntaxLabel.EOL, new Regex("\\r\\n|\\n")}, { SyntaxLabel.Whitespace, new Regex("\\s") }, { SyntaxLabel.Questionmark, new Regex("\\?") }, { SyntaxLabel.SingleLineCommentStart, new Regex("\\/\\/") }, { SyntaxLabel.MultiLineCommentStart, new Regex("\\/\\*") }, { SyntaxLabel.MultiLineCommentEnd, new Regex("\\*\\/") }, { SyntaxLabel.Equal, new Regex("==") }, { SyntaxLabel.CharSymbol, new Regex("\'") }, { SyntaxLabel.StringSymbol, new Regex("\"") }, { SyntaxLabel.NumberSymbol, new Regex("[0-9-]") }, { SyntaxLabel.OpenCurlyBracket, new Regex("\\{") }, { SyntaxLabel.CloseCurlyBracket, new Regex("\\}") }, { SyntaxLabel.OpenNormalBracket, new Regex("\\(") }, { SyntaxLabel.CloseNormalBracket, new Regex("\\)") }, { SyntaxLabel.OpenSquareBracket, new Regex("\\[") }, { SyntaxLabel.CloseSquareBracket, new Regex("\\]") } }; static readonly SyntaxLabel[] LiteralTokens = { SyntaxLabel.EOL, SyntaxLabel.Whitespace, SyntaxLabel.Questionmark, SyntaxLabel.Colon, SyntaxLabel.Semicolon, SyntaxLabel.Comma, SyntaxLabel.OpenNormalBracket, SyntaxLabel.CloseNormalBracket, SyntaxLabel.OpenSquareBracket, SyntaxLabel.CloseSquareBracket, SyntaxLabel.OpenCurlyBracket, SyntaxLabel.CloseCurlyBracket, SyntaxLabel.Plus, SyntaxLabel.Minus }; private static bool IsAllGenerated (List tokens) { bool generated = true; if ((tokens != null) && (tokens.Count > 0)) { // Debug.Log("NodeGenerator.IsAllGenerated "+tokens.Count); foreach (TokenList tl in tokens) { // Debug.Log("- TokenList: "+tl.Count+": '"+string.Join(", ",tl.Select(t=>t.type))+"'"); generated &= (tl != null) && (tl.Count > 0); // foreach (Token t in tl) { // Debug.Log("- Token: "+t.type); // } } } else { generated &= false; } return generated; } // private static bool IsGenerated (TokenList tokens) { // return (tokens != null) && (tokens.Count > 0); // } public static void DestroyAll (ref List tokens) { if (tokens != null) { foreach (TokenList tok in tokens) { foreach (Token t in tok) { if (t != null) { t.Destroy(); UnityEngine.Object.DestroyImmediate(t); } } } tokens = null; } } // private static void Reset (ref TokenList tokens) { // if (tokens != null) { // foreach (Token t in tokens) { // if (t != null) { // t.Reset(); // UnityEngine.Object.DestroyImmediate(t); // } // } // tokens = null; // } // } private static bool PatternMatch(Regex expression, int cursor, string code, int size = 1) => code.Length >= cursor + size && expression.IsMatch(code.Substring(cursor, size)); private static bool SynaxMatch(SyntaxLabel kind, int cursor, string code, int size = 1) => PatternMatch(Definitions[kind], cursor, code, size); public static bool Tokenize (out List tokenList, List taskScripts) { bool allSucceeded = true; tokenList = new List(); for (int i=0; i SynaxMatch(n, cursor, code))) { MakeToken(LiteralTokens.First(n => SynaxMatch(n, cursor, code)), cursor, 1, ref tokens, ref line, ref col, source); cursor++; continue; } //else { // throw new Exception($"CountNotTokenizeCharException char: {taskController.code[cursor]} could not be tokenized {cursor}"); //} if (savecursor == cursor) { //throw new Exception($"TokenizeScriptError char: {source.text[cursor]} unknown error on {cursor} line:{line}/col:{col}"); Debug.LogError($"TokenizeScriptError char: {source.text[cursor]} unknown error on {cursor} line:{line}/col:{col}"); } cursor++; } return true; } // public static string AsCode(TokenList tokens) { // return string.Join("", tokens.Select(n => n.GetValue())); // } private static void MakeToken(SyntaxLabel label, int pos, int len, ref TokenList tokens, ref int line, ref int col, UnityEngine.TextAsset source) { col += len; if (label == SyntaxLabel.EOL) { line++; col = 0; } Token t = ScriptableObject.CreateInstance(); TokenLocation l = ScriptableObject.CreateInstance(); l.Init(source, pos, len, line, col); t.Init(label, l); tokens.Add(t); } } }