From d7704110dddc0772e1d2a999172d82bebcee7e9b Mon Sep 17 00:00:00 2001 From: Ad5001 Date: Sun, 23 Oct 2022 15:43:47 +0200 Subject: [PATCH] Lots of changes to the AST Builder. Adding IDENTIFIER Token type with optional differentiation between functions, constants and variables for syntax highlighting. Adding substitute for many AST elements. Properly starting builder: parsing identifiers, functions, variables, array values, properties, numbers, strings and subexpressions. --- .../Setting/ExpressionEditor.qml | 2 +- .../ad5001/LogarithmPlotter/js/parsing/ast.js | 59 +++-- .../LogarithmPlotter/js/parsing/builder.js | 223 ++++++++++++++++-- .../LogarithmPlotter/js/parsing/reference.js | 15 ++ .../LogarithmPlotter/js/parsing/tokenizer.js | 45 +++- 5 files changed, 297 insertions(+), 47 deletions(-) diff --git a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/Setting/ExpressionEditor.qml b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/Setting/ExpressionEditor.qml index 34cc4b3..c587fcd 100644 --- a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/Setting/ExpressionEditor.qml +++ b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/Setting/ExpressionEditor.qml @@ -504,7 +504,7 @@ Item { Generates a list of tokens from the given. */ function tokens(text) { - let tokenizer = new Parsing.Tokenizer(new Parsing.Input(text), true, false) + let tokenizer = new Parsing.Tokenizer(new Parsing.Input(text), true, true, false) let tokenList = [] let token while((token = tokenizer.next()) != null) diff --git a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/ast.js b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/ast.js index b00e297..160ebfb 100644 --- a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/ast.js +++ b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/ast.js @@ -24,10 +24,7 @@ const DERIVATION_PRECISION = 0.01 const ZERO_EPISLON = 5e-11 // Number under which a variable is considered 0 when dealing with floating point rounding errors. -const BINARY_OPERATION_PRIORITY = { - "+": 10, "-": 10, - "*": 20, "/": 20 -} +const BINARY_OPERATION_PRIORITY = Reference.BINARY_OPERATION_PRIORITY enum ASEType { UNKNOWN, @@ -417,16 +414,17 @@ class FunctionElement extends AbstractSyntaxElement { } substitute(variable, substitution) { + return new FunctionElement(this.functionName, this.args.map(arg => arg.substitute(variable, substitution))) } derivation(variable) { //TODO: Use DERIVATIVES elements in reference. - return new DerivationElement([this, variable]) + return new DerivationElement([this, new Variable(variable)]) } integral(variable) { //TODO: Use INTEGRALS elements in reference. - return new IntegralElement([this, variable]) + return new IntegralElement([this, new Variable(variable)]) } toEditableString() { @@ -487,12 +485,20 @@ class DerivationElement extends FunctionElement { } simplify() { - return new DerivationElement([this.args[0].simplify(variables), this.args[1]]) + return this.args[0].simplify().derivation(this.args[1].variableName).simplify() + } + + substitute(variable, substitution) { + if(variable == this.args[1].variableName) { + // Simplifu, + return this.simplify().substitute(variable, substitution) + } else + return new DerivationElement([this.args[0].substitute(variable, substitution), this.args[1]]) } integral(variable) { // Check if we're integrating and derivating by the same variable - return variable.variableName == this.args[1].variableName ? this.args[1] : super(variable) + return variable == this.args[1].variableName ? this.args[1] : super(variable) } toLatex() { @@ -557,26 +563,44 @@ class IntegralElement extends FunctionElement { simplify() { // TODO: When full derivation and integrals are implemented, use dedicated functions for simplification. - let func = this.args[this.args.length-2].simplify(variables) + let func = this.f.simplify(variables) let newElem if(func.isConstant() && this.args.length == 4) // Simplify integral. newElem = new BinaryOperation( - new BinaryOperation(this.args[1], '-', this.args[0]).simplify(), + new BinaryOperation(this.b, '-', this.a).simplify(), '*', func - ) - else - newElem = new IntegralElement(this.args.length == 4 ? - [this.a.simplify(), this.b.simplify(), func, this.d] : - [func, this.d] - ) + ).simplify() + else { + let integrated = this.func.integral(this.d.variableName) + newElem = new BinaryOperation( + integrated.substitute(this.d.variableName, this.b), + '-', + integrated.substitute(this.d.variableName, this.a) + ).simplify() + //newElem = new IntegralElement(this.args.length == 4 ? + // [this.a.simplify(), this.b.simplify(), func, this.d] : + // [func, this.d] + //) + } return newElem } + substitute(variable, substitution) { + if(variable == this.args[1].variableName) { + // Simplify + return this.simplify().substitute(variable, substitution) + } else + return new IntegralElement(this.args.length == 4 ? + [this.a.substitute(variable, substitution), this.b.simplify(variable, substitution), + this.f.substitute(variable, substitution), this.d] : + [this.f.substitute(variable, substitution), this.d]) + } + derivation(variable) { // Check if we're integrating and derivating by the same variable - return variable.variableName == this.args[1].variableName ? this.args[1] : super(variable) + return variable == this.args[1].variableName ? this.args[1] : super(variable) } toLatex() { @@ -689,6 +713,7 @@ class BinaryOperation extends AbstractSyntaxElement { throw new EvalError("Unknown operator " + ope + ".") } } + // TODO: Check for all nearby operations simplifications return result } diff --git a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/builder.js b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/builder.js index 4c9a0ce..397804a 100644 --- a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/builder.js +++ b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/builder.js @@ -23,26 +23,217 @@ class ExpressionBuilder { - constructor(tokenizer) { + constructor(tokenizer, rememberTokens = false) { this.tokenizer = tokenizer; + if(tokenizer.tokenizeWhitespaces) { + console.warn('WARNING: The ExpressionTokenizer for ExpressionBuilder was initialized with whitespace support. Disabled.') + tokenizer.tokenizeWhitespaces = false + } + if(tokenizer.differentiateIdentifiers) { + console.warn('WARNING: The ExpressionTokenizer for ExpressionBuilder was initialized with identifier differentiation support. Disabled.') + tokenizer.differentiateIdentifiers = false + } + this.tokens = [] + this.rememberTokens = rememberTokens this.stack = [] } - parseExpression(delimitors = '') { - // Parse a sequence of operations, and orders them based on OPERATION_PRIORITY. - let elements = [] - let operators = [] - let firstToken = this.tokenizer.peek(); - if(firstToken.type == TK.TokenType.OPERATOR) // First operations. - if(firstToken.value == "-") { - // TODO: Set initial argument. - this.tokenizer.skip(TK.TokenType.OPERATOR) - } else - tokenizer.input.raise(`Invalid operator ${firstToken.value} at begining of statement.`) - else { - - } + /** + * Parses an expression until the end is reached. + * + * @throws {Error} When an unexpected token is encountered. + * @returns {AbstractSyntaxElement} + */ + parseFullExpression() { + this.parseExpression([]) + return this.stack.pop() } - parseOperation()` + /** + * Parses an expression until the end is reached. + * + * @param {string} punctuationDelimitators - List of deliminators that ends the expression + * @throws {Error} When an unexpected token is encountered. + */ + parseExpression(punctuationDelimitators = []) { + let token + while((token = this.tokenizer.peek()) != null) { + if(token.type == TK.TokenType.PUNCT && token.value != '(') // Still allow expression creation. + if(punctuationDelimitators.includes(token.value)) + break + else if(punctuationDelimitators.length == 1) + this.tokenizer.raise(`Unexpected ${token.value}. Expected '${punctuationDelimitators[0]}'.`) + else + this.tokenizer.raise(`Unexpected ${token.value}. Expected one of: '${punctuationDelimitators[0].join("', '")}'.`) + else + this.handleSingle() + } + + if(token == null && punctuationDelimitators.length > 0) + if(punctuationDelimitators.length == 1) + this.tokenizer.raise(`Unexpected end of expression. Expected '${punctuationDelimitators[0]}'.`) + else + this.tokenizer.raise(`Unexpected end of expression. Expected one of: '${punctuationDelimitators[0].join("', '")}'.`) + + + if(this.stack.length == 0) + if(token == null) + this.tokenizer.raise(`Unexpected end of expression. Expected at least one element.`) + else + this.tokenizer.raise(`Unexpected ${token.value}. Expected at least one element.`) + + if(this.stack.length > 1) + this.tokenizer.raise('Invalid expression.') + } + + /** + * Handles a single (assumed non-null) token based on its type. + * + * @param {AbstractSyntaxElement} token + * @throws {Error} When an unexpected token is encountered. + */ + handleSingle(token) { + switch(token.type) { + case TK.TokenType.IDENTIFIER: + this.parseIdentifier() + break + case TK.TokenType.OPERATOR: + if(this.stack.length == 0 && Reference.UNARY_OPERATORS.includes(token.value)) + this.parseSingleOperation() + else if(this.stack.length > 0 && Reference.BINARY_OPERATORS.includes(token.value)) + this.parseBinaryOperations() + else if(this.stack.length > 0 && Reference.TERTIARY_OPERATORS.includes(token.value)) + this.parseTertiaryOperation() + break + case TK.TokenType.NUMBER: + this.stack.push(new AST.NumberElement(this.tokenizer.next().value)) + break + case TK.TokenType.STRING: + this.stack.push(new AST.StringElement(this.tokenizer.next().value)) + break + case TK.TokenType.PUNCT: + if(token.value == '(') { + this.tokenizer.skip(TK.TokenType.PUNCT, '(') // Skip the opening parentheses. + this.parseExpression([')']) + } else + this.tokenizer.raise(`Unexpected ${token.value}. Expected a value.`) + break + default: + this.tokenizer.raise(`Unknown token provided: ${token.value}.`) + break + } + if(this.rememberTokens) + this.tokens.push(token) + } + + /** + * Parses a single token element. + * + * @throws {Error} When an unexpected token is encountered. + */ + parseSingle() { + let token = this.tokenizer.peek() + if(token != null) + this.handleSingle(token) + } + + parseIdentifier() { + // Assuming the right type. + let token = this.tokenizer.read(TK.TokenType.IDENTIFIER) + if(Reference.CONSTANTS_LIST.includes(token.value)) + this.stack.push(new AST.Constant(token.value)) + else + this.stack.push(new AST.Variable(token.value)) + this.checkIdentifierFollowupTokens() + } + + /** + * Parses a function based on a previously called identifier. + * NOTE: Expects to have at least one stack element for function name. + */ + parseFunction() { + // TODO: Implement dynamic functions values instead of being based on names. + let functionValue = this.stack.pop() + if(!(functionValue instanceof AST.Variable)) + this.tokenizer.raise("Executing functions from dynamic variables is not implemented".) + let functionName = functionValue.variableName + let args = [] + let token + while((token = this.tokenizer.peek()) != null && token.value != ')') { + this.tokenizer.skip(TK.TokenType.PUNCT) // Skip the opening parenthesis and the commas. + parseExpression([',',')']) + args.push(this.stack.pop()) + } + + if(token == null) + this.tokenizer.raise(`Unexpected end of expression. Expected ')'.`) + + if(this.functionName == 'derivation') + this.stack.push(new AST.DerivationElement(args)) + else if(this.functionName == 'integral') + this.stack.push(new AST.IntegralElement(args)) + else + this.stack.push(new AST.FunctionElement(functionName, args)) + } + + + /** + * Parses an object property based on a previously called identifier. + * NOTE: Expects to have at least one stack element for property object name. + */ + parseProperty() { + this.tokenizer.skip(TK.TokenType.PUNCT, '.') // Skipping the dot. + let token = this.tokenizer.read(TK.TokenType.IDENTIFIER) + this.stack.push(new AST.PropertyElement(this.stack.pop(), token.value)) + this.checkIdentifierFollowupTokens() + } + + /** + * Parses the value of the element of an array at a given index based on a previously called identifier. + * NOTE: Expects to have at least one stack element for property object name. + */ + parseArrayValue() { + this.tokenizer.skip(TK.TokenType.PUNCT, '[') // Skipping the array opener. + let obj = this.stack.pop() + parseExpression([']']) + this.stack.push(new AST.ArrayElement(obj, this.stack.pop())) + this.checkIdentifierFollowupTokens() + } + + /** + * Checks for followup tokens following a value getting. + * E.g: getting the property of an object, an array member, or calling a function. + * NOTE: Expects to have at least one stack element for previous calling object. + */ + checkIdentifierFollowupTokens() { + let peeked = this.tokenizer.peek() + if(peeked != null && peeked.type == TK.TokenType.PUNCT) + switch(peeked.value) { + case '(': + // Function call + this.parseFunction() + break + case '.': + // Member property + this.parseProperty() + break + case '[': + // Array value + this.parseArrayValue() + break + } + } + + parseBinaryOperations() { + if((this.tokenizer.peek().value in AST.BINARY_OPERATION_PRIORITY)) + throw new Error("Current token is not a binary operator.") + if(this.stack.length == 0) + throw new Error(`The operator ${this.tokenizer.peek().value} can only be used after a value.`) + // Parse a sequence of operations, and orders them based on OPERATION_PRIORITY. + let elements = [this.stack.pop()] + let operators = [this.tokenizer.next()] + let token + while((token = this.tokenizer.peek()) != null) { + } + } } diff --git a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/reference.js b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/reference.js index e92bc08..ab3fc8a 100644 --- a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/reference.js +++ b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/reference.js @@ -28,6 +28,21 @@ const CONSTANTS = { }; const CONSTANTS_LIST = Object.keys(CONSTANTS); +const UNARY_OPERATORS = [] + +const BINARY_OPERATION_PRIORITY = { + 'and': 0, 'or': 0, 'in': 0, + '==': 10, '!=' : 10, + '>': 10, '<': 10, '>=': 10, '<=': 10, + '%': 20, '||': 20, + '+': 30, '-': 30, + '*': 40, '/': 40, + '^': 50 +} +const BINARY_OPERATORS = Object.keys(BINARY_OPERATION_PRIORITY) + +const TERTIARY_OPERATORS = ['?'] + const FUNCTIONS = { "abs": Math.abs, "acos": Math.acos, diff --git a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/tokenizer.js b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/tokenizer.js index 63358c0..e49a171 100644 --- a/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/tokenizer.js +++ b/LogarithmPlotter/qml/eu/ad5001/LogarithmPlotter/js/parsing/tokenizer.js @@ -23,13 +23,14 @@ const WHITESPACES = " \t\n\r" const STRING_LIMITORS = '"\'`'; const OPERATORS = "+-*/^%?:=!><"; -const PUNCTUTATION = "()[]{},."; +const PUNCTUTATION = "()[],."; const NUMBER_CHARS = "0123456789" const IDENTIFIER_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789_₀₁₂₃₄₅₆₇₈₉αβγδεζηθκλμξρςστφχψωₐₑₒₓₔₕₖₗₘₙₚₛₜ" var TokenType = { // Expression type "WHITESPACE": "WHITESPACE", + "IDENTIFIER": "IDENTIFIER", "VARIABLE": "VARIABLE", "CONSTANT": "CONSTANT", "FUNCTION": "FUNCTION", @@ -49,10 +50,11 @@ class Token { } class ExpressionTokenizer { - constructor(input, tokenizeWhitespaces = false, errorOnUnknown = true) { - this.input = input; - this.currentToken = null; + constructor(input, tokenizeWhitespaces = false, differentiateIdentifiers = false, errorOnUnknown = true) { + this.input = input + this.currentToken = null this.tokenizeWhitespaces = tokenizeWhitespaces + this.differentiateIdentifiers = differentiateIdentifiers this.errorOnUnknown = errorOnUnknown } @@ -115,10 +117,16 @@ class ExpressionTokenizer { while(!this.input.atEnd() && IDENTIFIER_CHARS.includes(this.input.peek().toLowerCase())) { identifier += this.input.next(); } - if(Reference.CONSTANTS_LIST.includes(identifier.toLowerCase())) { - return new Token(TokenType.CONSTANT, identifier.toLowerCase(), this.input.position-identifier.length) - } else if(Reference.FUNCTIONS_LIST.includes(identifier.toLowerCase())) { - return new Token(TokenType.FUNCTION, identifier.toLowerCase(), this.input.position-identifier.length) + let identifierLC = identifier.toLowerCase() + if(Reference.CONSTANTS_LIST.includes(identifierLC)) { + return new Token(TokenType.CONSTANT, identifierLC, this.input.position-identifier.length) + } else if(Reference.FUNCTIONS_LIST.includes(identifierLC)) { + return new Token(TokenType.FUNCTION, identifierLC, this.input.position-identifier.length) + } else if(Reference.UNARY_OPERATORS.includes(identifierLC) || + Reference.BINARY_OPERATORS.includes(identifierLC) || + Reference.TERTIARY_OPERATORS.includes(identifierLC) + ) { + return new Token(TokenType.OPERATOR, identifierLC, this.input.position-identifier.length) } else { return new Token(TokenType.VARIABLE, identifier, this.input.position-identifier.length) } @@ -137,7 +145,7 @@ class ExpressionTokenizer { if(Reference.CONSTANTS_LIST.includes(c)) return new Token(TokenType.CONSTANT, this.input.next(), this.input.position-1); if(PUNCTUTATION.includes(c)) return new Token(TokenType.PUNCT, this.input.next(), this.input.position-1); if(this.errorOnUnknown) - this.input.throw("Unknown token character " + c) + this.raise("Unknown token character " + c) else return new Token(TokenType.UNKNOWN, this.input.next(), this.input.position-1); } @@ -156,14 +164,25 @@ class ExpressionTokenizer { this.currentToken = null; return tmp; } + + read(type, value) { + let next = this.next() + if(type != null && next.type != type) + this.raise(`Unexpected ${next.type.toLowerCase()} ${next.value}. Expected type was ${type.toLowerCase()}.`); + if(value != null && next.value == value) + this.raise(`Unexpected ${next.type.toLowerCase()} ${next.value}. Expected value was ${value}.`); + return next + } atEnd() { return this.peek() == null; } - skip(type) { - let next = this.next(); - if(next.type != type) - input.raise("Unexpected token " + next.type.toLowerCase() + ' "' + next.value + '". Expected ' + type.toLowerCase()); + skip(type, value) { + this.read(type, value) + } + + raise(message) { + this.input.raise(message) } }