Syntax coloration first attempt!

Currently, when there are two many styles, the text is *slightly* offset from the normal one.

LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/Setting/ExpressionEditor.qml

@@ -21,6 +21,8 @@ import QtQuick 2.12
 import QtQuick.Dialogs 1.3 as D
 import eu.ad5001.LogarithmPlotter.Popup 1.0 as Popup
 import "../js/mathlib.js" as MathLib
+import "../js/utils.js" as Utils
+import "../js/parsing/parsing.js" as Parsing
 
 
 /*!
@@ -77,6 +79,7 @@ Item {
     /*!
        \qmlproperty string ExpressionEditor::openAndCloseMatches
        Characters that when pressed, should be immediately followed up by their closing character.
+       TODO: Make it configurable.
     */
     readonly property var openAndCloseMatches: {
         "(": ")",
@@ -85,6 +88,21 @@ Item {
         '"': '"'
     }
     
+    /*!
+       \qmlproperty string ExpressionEditor::colorScheme
+       Color scheme of the editor, currently based on Breeze Light.
+       TODO: Make it configurable.
+    */
+    readonly property var colorScheme: {
+        'NORMAL': "#1F1C1B",
+        'VARIABLE': "#0057AE",
+        'CONSTANT': "#5E2F00",
+        'FUNCTION': "#644A9B",
+        'OPERATOR': "#A44EA4",
+        'STRING': "#9C0E0E",
+        'NUMBER': "#805C00"
+    }
+    
     Icon {
         id: iconLabel
         anchors.top: parent.top
@@ -117,6 +135,7 @@ Item {
         }
     }
     
+    
     TextField {
         id: editor
         anchors.top: parent.top
@@ -126,8 +145,9 @@ Item {
         height: parent.height
         verticalAlignment: TextInput.AlignVCenter
         horizontalAlignment: control.label == "" ? TextInput.AlignLeft : TextInput.AlignHCenter
+        font.pixelSize: 14
         text: control.defValue
-        color: sysPalette.windowText
+        color: "transparent"//sysPalette.windowText
         focus: true
         selectByMouse: true
         
@@ -154,6 +174,18 @@ Item {
                 event.accepted = true
             }
         }
+        
+        Text {
+            id: colorizedEditor
+            anchors.fill: editor
+            verticalAlignment: TextInput.AlignVCenter
+            horizontalAlignment: control.label == "" ? TextInput.AlignLeft : TextInput.AlignHCenter
+            textFormat: Text.StyledText
+            text: colorize(editor.text)
+            color: sysPalette.windowText
+            font.pixelSize: parent.font.pixelSize
+            //opacity: editor.activeFocus ? 0 : 1
+        }
     }
     
     Button {
@@ -210,5 +242,46 @@ Item {
         }
         return expr
     }
+    
+    /*!
+        \qmlmethod var ExpressionEditor::colorize(string expressionText)
+        Creates an HTML colorized string of the incomplete \c expressionText.
+        Returns the colorized and escaped expression if possible, null otherwise..
+    */
+    function colorize(text) {
+        let tokenizer = new Parsing.Tokenizer(new Parsing.Input(text), true, false)
+        let parsedText = ""
+        let token
+        console.log("Parsing text:", parsedText)
+        while((token = tokenizer.next()) != null) {
+            switch(token.type) {
+                case Parsing.TokenType.VARIABLE:
+                    parsedText += `<font color="${colorScheme.VARIABLE}">${token.value}</font>`
+                    break;
+                case Parsing.TokenType.CONSTANT:
+                    parsedText += `<font color="${colorScheme.CONSTANT}">${token.value}</font>`
+                    break;
+                case Parsing.TokenType.FUNCTION:
+                    parsedText += `<font color="${Utils.escapeHTML(colorScheme.FUNCTION)}">${token.value}</font>`
+                    break;
+                case Parsing.TokenType.OPERATOR:
+                    parsedText += `<font color="${colorScheme.OPERATOR}">${Utils.escapeHTML(token.value)}</font>`
+                    break;
+                case Parsing.TokenType.NUMBER:
+                    parsedText += `<font color="${colorScheme.NUMBER}">${Utils.escapeHTML(token.value)}</font>`
+                    break;
+                case Parsing.TokenType.STRING:
+                    parsedText += `<font color="${colorScheme.STRING}">${token.limitator}${Utils.escapeHTML(token.value)}${token.limitator}</font>`
+                    break;
+                case Parsing.TokenType.WHITESPACE:
+                case Parsing.TokenType.PUNCT:
+                default:
+                    parsedText += Utils.escapeHTML(token.value).replace(/ /g, '&nbsp;')
+                    break;
+            }
+        }
+        console.log("Parsed text:", parsedText)
+        return parsedText
+    }
 }
 

LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/builder.js

@@ -18,8 +18,8 @@
 
 .pragma library
 
-import "ast.js" as AST
+.import "ast.js" as AST
-import "tokenizer.js" as TK
+.import "tokenizer.js" as TK
 
 
 class ExpressionBuilder {

LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/common.js

@@ -33,7 +33,7 @@ class InputExpression {
     }
     
     skip(char) {
-        if(!atEnd() && peek() == char) {
+        if(!this.atEnd() && this.peek() == char) {
             this.position++;
         } else {
             this.raise("Unexpected character " + peek() + ". Expected character " + char);

LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/tokenizer.js

@@ -22,20 +22,22 @@
 
 const WHITESPACES = " \t\n\r"
 const STRING_LIMITORS = '"\'`';
-const OPERATORS = "+-*/^%";
+const OPERATORS = "+-*/^%?:=!><";
-const PUNCTUTATION = "()[]{},";
+const PUNCTUTATION = "()[]{},.";
 const NUMBER_CHARS = "0123456789."
 const IDENTIFIER_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789_₀₁₂₃₄₅₆₇₈₉αβγδεζηθκλμξρςστφχψωₐₑₒₓₔₕₖₗₘₙₚₛₜ"
 
-enum TokenType {
+var TokenType = {
     // Expression type
-    VARIABLE,
+    "WHITESPACE": "WHITESPACE",
-    CONSTANT,
+    "VARIABLE": "VARIABLE",
-    FUNCTION,
+    "CONSTANT": "CONSTANT",
-    OPERATOR,
+    "FUNCTION": "FUNCTION",
-    PUNCT,
+    "OPERATOR": "OPERATOR",
-    NUMBER,
+    "PUNCT": "PUNCT",
-    STRING
+    "NUMBER": "NUMBER",
+    "STRING": "STRING",
+    "UNKNOWN": "UNKNOWN"
 }
 
 class Token {
@@ -46,9 +48,11 @@ class Token {
 }
 
 class ExpressionTokenizer {
-    constructor(input) {
+    constructor(input, tokenizeWhitespaces = false, errorOnUnknown = true) {
         this.input = input;
         this.currentToken = null;
+        this.tokenizeWhitespaces = tokenizeWhitespaces
+        this.errorOnUnknown = errorOnUnknown
     }
     
     skipWhitespaces() {
@@ -56,6 +60,14 @@ class ExpressionTokenizer {
             this.input.next();
     }
     
+    readWhitespaces() {
+        let included = "";
+        while(!this.input.atEnd() && WHITESPACES.includes(this.input.peek())) {
+            included += this.input.next();
+        }
+        return new Token(TokenType.WHITESPACE, included)
+    }
+    
     readString() {
         let delimitation = this.input.peek();
         if(STRING_LIMITORS.includes(delimitation)) {
@@ -68,7 +80,9 @@ class ExpressionTokenizer {
                     included += this.input.next();
             }
             this.input.skip(delimitation)
-            return new Token(TokenType.STRING, included);
+            let token = new Token(TokenType.STRING, included)
+            token.limitator = delimitation
+            return token
         } else {
             this.input.raise("Unexpected " + delimitation + ". Expected  string delimitator")
         }
@@ -84,12 +98,20 @@ class ExpressionTokenizer {
             }
             included += this.input.next();
         }
+        return new Token(TokenType.NUMBER, included)
+    }
+    
+    readOperator() {
+        let included = "";
+        while(!this.input.atEnd() && OPERATORS.includes(this.input.peek())) {
+            included += this.input.next();
+        }
+        return new Token(TokenType.OPERATOR, included)
     }
     
     readIdentifier() {
         let identifier = "";
-        let hasDot = false;
+        while(!this.input.atEnd() && IDENTIFIER_CHARS.includes(this.input.peek().toLowerCase())) {
-        while(!this.input.atEnd() && IDENTIFIER_CHARS.includes(this.input.peek())) {
             identifier += this.input.next();
         }
         if(Reference.CONSTANTS_LIST.includes(identifier.toLowerCase())) {
@@ -102,16 +124,21 @@ class ExpressionTokenizer {
     }
     
     readNextToken() {
-        this.skipWhitespaces()
+        if(!this.tokenizeWhitespaces)
-        if(input.atEnd()) return null;
+            this.skipWhitespaces()
-        let c = input.peek();
+        if(this.input.atEnd()) return null;
+        let c = this.input.peek();
+        if(this.tokenizeWhitespaces && WHITESPACES.includes(c)) return this.readWhitespaces();
         if(STRING_LIMITORS.includes(c)) return this.readString();
         if(NUMBER_CHARS.includes(c)) return this.readNumber();
-        if(IDENTIFIER_CHARS.includes(c)) return this.readIdentifier();
+        if(IDENTIFIER_CHARS.includes(c.toLowerCase())) return this.readIdentifier();
+        if(OPERATORS.includes(c)) return this.readOperator();
         if(Reference.CONSTANTS_LIST.includes(c)) return new Token(TokenType.CONSTANT, c);
-        if(OPERATORS.includes(c)) return new Token(TokenType.OPERATOR, c);
+        if(PUNCTUTATION.includes(c)) return new Token(TokenType.PUNCT, this.input.next());
-        if(PUNCTUTATION.includes(c)) return new Token(TokenType.PUNCT, c);
+        if(this.errorOnUnknown)
-        this.input.throw("Unknown token character " + c)
+            this.input.throw("Unknown token character " + c)
+        else
+            return new Token(TokenType.UNKNOWN, this.input.next());
     }
 
     peek() {
@@ -134,8 +161,8 @@ class ExpressionTokenizer {
     }
     
     skip(type) {
-        Token next = Next();
+        let next = this.next();
         if(next.type != type)
-            input.raise("Unexpected token " + next.type.oLowerCase() + ' "' + next.value + '". Expected ' + type.toLowerCase());
+            input.raise("Unexpected token " + next.type.toLowerCase() + ' "' + next.value + '". Expected ' + type.toLowerCase());
     }
 }

LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/utils.js

@@ -347,3 +347,7 @@ function getRandomColor() {
     }
     return color;
 }
+
+function escapeHTML(str) {
+    return str.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;') ;
+}