Lots of changes to the AST Builder.
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
Adding IDENTIFIER Token type with optional differentiation between functions, constants and variables for syntax highlighting. Adding substitute for many AST elements. Properly starting builder: parsing identifiers, functions, variables, array values, properties, numbers, strings and subexpressions.
This commit is contained in:
parent
666d611e95
commit
d7704110dd
5 changed files with 297 additions and 47 deletions
|
@ -504,7 +504,7 @@ Item {
|
|||
Generates a list of tokens from the given.
|
||||
*/
|
||||
function tokens(text) {
|
||||
let tokenizer = new Parsing.Tokenizer(new Parsing.Input(text), true, false)
|
||||
let tokenizer = new Parsing.Tokenizer(new Parsing.Input(text), true, true, false)
|
||||
let tokenList = []
|
||||
let token
|
||||
while((token = tokenizer.next()) != null)
|
||||
|
|
|
@ -24,10 +24,7 @@
|
|||
const DERIVATION_PRECISION = 0.01
|
||||
const ZERO_EPISLON = 5e-11 // Number under which a variable is considered 0 when dealing with floating point rounding errors.
|
||||
|
||||
const BINARY_OPERATION_PRIORITY = {
|
||||
"+": 10, "-": 10,
|
||||
"*": 20, "/": 20
|
||||
}
|
||||
const BINARY_OPERATION_PRIORITY = Reference.BINARY_OPERATION_PRIORITY
|
||||
|
||||
enum ASEType {
|
||||
UNKNOWN,
|
||||
|
@ -417,16 +414,17 @@ class FunctionElement extends AbstractSyntaxElement {
|
|||
}
|
||||
|
||||
substitute(variable, substitution) {
|
||||
return new FunctionElement(this.functionName, this.args.map(arg => arg.substitute(variable, substitution)))
|
||||
}
|
||||
|
||||
derivation(variable) {
|
||||
//TODO: Use DERIVATIVES elements in reference.
|
||||
return new DerivationElement([this, variable])
|
||||
return new DerivationElement([this, new Variable(variable)])
|
||||
}
|
||||
|
||||
integral(variable) {
|
||||
//TODO: Use INTEGRALS elements in reference.
|
||||
return new IntegralElement([this, variable])
|
||||
return new IntegralElement([this, new Variable(variable)])
|
||||
}
|
||||
|
||||
toEditableString() {
|
||||
|
@ -487,12 +485,20 @@ class DerivationElement extends FunctionElement {
|
|||
}
|
||||
|
||||
simplify() {
|
||||
return new DerivationElement([this.args[0].simplify(variables), this.args[1]])
|
||||
return this.args[0].simplify().derivation(this.args[1].variableName).simplify()
|
||||
}
|
||||
|
||||
substitute(variable, substitution) {
|
||||
if(variable == this.args[1].variableName) {
|
||||
// Simplifu,
|
||||
return this.simplify().substitute(variable, substitution)
|
||||
} else
|
||||
return new DerivationElement([this.args[0].substitute(variable, substitution), this.args[1]])
|
||||
}
|
||||
|
||||
integral(variable) {
|
||||
// Check if we're integrating and derivating by the same variable
|
||||
return variable.variableName == this.args[1].variableName ? this.args[1] : super(variable)
|
||||
return variable == this.args[1].variableName ? this.args[1] : super(variable)
|
||||
}
|
||||
|
||||
toLatex() {
|
||||
|
@ -557,26 +563,44 @@ class IntegralElement extends FunctionElement {
|
|||
|
||||
simplify() {
|
||||
// TODO: When full derivation and integrals are implemented, use dedicated functions for simplification.
|
||||
let func = this.args[this.args.length-2].simplify(variables)
|
||||
let func = this.f.simplify(variables)
|
||||
let newElem
|
||||
if(func.isConstant() && this.args.length == 4)
|
||||
// Simplify integral.
|
||||
newElem = new BinaryOperation(
|
||||
new BinaryOperation(this.args[1], '-', this.args[0]).simplify(),
|
||||
new BinaryOperation(this.b, '-', this.a).simplify(),
|
||||
'*',
|
||||
func
|
||||
)
|
||||
else
|
||||
newElem = new IntegralElement(this.args.length == 4 ?
|
||||
[this.a.simplify(), this.b.simplify(), func, this.d] :
|
||||
[func, this.d]
|
||||
)
|
||||
).simplify()
|
||||
else {
|
||||
let integrated = this.func.integral(this.d.variableName)
|
||||
newElem = new BinaryOperation(
|
||||
integrated.substitute(this.d.variableName, this.b),
|
||||
'-',
|
||||
integrated.substitute(this.d.variableName, this.a)
|
||||
).simplify()
|
||||
//newElem = new IntegralElement(this.args.length == 4 ?
|
||||
// [this.a.simplify(), this.b.simplify(), func, this.d] :
|
||||
// [func, this.d]
|
||||
//)
|
||||
}
|
||||
return newElem
|
||||
}
|
||||
|
||||
substitute(variable, substitution) {
|
||||
if(variable == this.args[1].variableName) {
|
||||
// Simplify
|
||||
return this.simplify().substitute(variable, substitution)
|
||||
} else
|
||||
return new IntegralElement(this.args.length == 4 ?
|
||||
[this.a.substitute(variable, substitution), this.b.simplify(variable, substitution),
|
||||
this.f.substitute(variable, substitution), this.d] :
|
||||
[this.f.substitute(variable, substitution), this.d])
|
||||
}
|
||||
|
||||
derivation(variable) {
|
||||
// Check if we're integrating and derivating by the same variable
|
||||
return variable.variableName == this.args[1].variableName ? this.args[1] : super(variable)
|
||||
return variable == this.args[1].variableName ? this.args[1] : super(variable)
|
||||
}
|
||||
|
||||
toLatex() {
|
||||
|
@ -689,6 +713,7 @@ class BinaryOperation extends AbstractSyntaxElement {
|
|||
throw new EvalError("Unknown operator " + ope + ".")
|
||||
}
|
||||
}
|
||||
// TODO: Check for all nearby operations simplifications
|
||||
return result
|
||||
}
|
||||
|
||||
|
|
|
@ -23,26 +23,217 @@
|
|||
|
||||
|
||||
class ExpressionBuilder {
|
||||
constructor(tokenizer) {
|
||||
constructor(tokenizer, rememberTokens = false) {
|
||||
this.tokenizer = tokenizer;
|
||||
if(tokenizer.tokenizeWhitespaces) {
|
||||
console.warn('WARNING: The ExpressionTokenizer for ExpressionBuilder was initialized with whitespace support. Disabled.')
|
||||
tokenizer.tokenizeWhitespaces = false
|
||||
}
|
||||
if(tokenizer.differentiateIdentifiers) {
|
||||
console.warn('WARNING: The ExpressionTokenizer for ExpressionBuilder was initialized with identifier differentiation support. Disabled.')
|
||||
tokenizer.differentiateIdentifiers = false
|
||||
}
|
||||
this.tokens = []
|
||||
this.rememberTokens = rememberTokens
|
||||
this.stack = []
|
||||
}
|
||||
|
||||
parseExpression(delimitors = '') {
|
||||
// Parse a sequence of operations, and orders them based on OPERATION_PRIORITY.
|
||||
let elements = []
|
||||
let operators = []
|
||||
let firstToken = this.tokenizer.peek();
|
||||
if(firstToken.type == TK.TokenType.OPERATOR) // First operations.
|
||||
if(firstToken.value == "-") {
|
||||
// TODO: Set initial argument.
|
||||
this.tokenizer.skip(TK.TokenType.OPERATOR)
|
||||
} else
|
||||
tokenizer.input.raise(`Invalid operator ${firstToken.value} at begining of statement.`)
|
||||
else {
|
||||
|
||||
}
|
||||
/**
|
||||
* Parses an expression until the end is reached.
|
||||
*
|
||||
* @throws {Error} When an unexpected token is encountered.
|
||||
* @returns {AbstractSyntaxElement}
|
||||
*/
|
||||
parseFullExpression() {
|
||||
this.parseExpression([])
|
||||
return this.stack.pop()
|
||||
}
|
||||
|
||||
parseOperation()`
|
||||
/**
|
||||
* Parses an expression until the end is reached.
|
||||
*
|
||||
* @param {string} punctuationDelimitators - List of deliminators that ends the expression
|
||||
* @throws {Error} When an unexpected token is encountered.
|
||||
*/
|
||||
parseExpression(punctuationDelimitators = []) {
|
||||
let token
|
||||
while((token = this.tokenizer.peek()) != null) {
|
||||
if(token.type == TK.TokenType.PUNCT && token.value != '(') // Still allow expression creation.
|
||||
if(punctuationDelimitators.includes(token.value))
|
||||
break
|
||||
else if(punctuationDelimitators.length == 1)
|
||||
this.tokenizer.raise(`Unexpected ${token.value}. Expected '${punctuationDelimitators[0]}'.`)
|
||||
else
|
||||
this.tokenizer.raise(`Unexpected ${token.value}. Expected one of: '${punctuationDelimitators[0].join("', '")}'.`)
|
||||
else
|
||||
this.handleSingle()
|
||||
}
|
||||
|
||||
if(token == null && punctuationDelimitators.length > 0)
|
||||
if(punctuationDelimitators.length == 1)
|
||||
this.tokenizer.raise(`Unexpected end of expression. Expected '${punctuationDelimitators[0]}'.`)
|
||||
else
|
||||
this.tokenizer.raise(`Unexpected end of expression. Expected one of: '${punctuationDelimitators[0].join("', '")}'.`)
|
||||
|
||||
|
||||
if(this.stack.length == 0)
|
||||
if(token == null)
|
||||
this.tokenizer.raise(`Unexpected end of expression. Expected at least one element.`)
|
||||
else
|
||||
this.tokenizer.raise(`Unexpected ${token.value}. Expected at least one element.`)
|
||||
|
||||
if(this.stack.length > 1)
|
||||
this.tokenizer.raise('Invalid expression.')
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles a single (assumed non-null) token based on its type.
|
||||
*
|
||||
* @param {AbstractSyntaxElement} token
|
||||
* @throws {Error} When an unexpected token is encountered.
|
||||
*/
|
||||
handleSingle(token) {
|
||||
switch(token.type) {
|
||||
case TK.TokenType.IDENTIFIER:
|
||||
this.parseIdentifier()
|
||||
break
|
||||
case TK.TokenType.OPERATOR:
|
||||
if(this.stack.length == 0 && Reference.UNARY_OPERATORS.includes(token.value))
|
||||
this.parseSingleOperation()
|
||||
else if(this.stack.length > 0 && Reference.BINARY_OPERATORS.includes(token.value))
|
||||
this.parseBinaryOperations()
|
||||
else if(this.stack.length > 0 && Reference.TERTIARY_OPERATORS.includes(token.value))
|
||||
this.parseTertiaryOperation()
|
||||
break
|
||||
case TK.TokenType.NUMBER:
|
||||
this.stack.push(new AST.NumberElement(this.tokenizer.next().value))
|
||||
break
|
||||
case TK.TokenType.STRING:
|
||||
this.stack.push(new AST.StringElement(this.tokenizer.next().value))
|
||||
break
|
||||
case TK.TokenType.PUNCT:
|
||||
if(token.value == '(') {
|
||||
this.tokenizer.skip(TK.TokenType.PUNCT, '(') // Skip the opening parentheses.
|
||||
this.parseExpression([')'])
|
||||
} else
|
||||
this.tokenizer.raise(`Unexpected ${token.value}. Expected a value.`)
|
||||
break
|
||||
default:
|
||||
this.tokenizer.raise(`Unknown token provided: ${token.value}.`)
|
||||
break
|
||||
}
|
||||
if(this.rememberTokens)
|
||||
this.tokens.push(token)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a single token element.
|
||||
*
|
||||
* @throws {Error} When an unexpected token is encountered.
|
||||
*/
|
||||
parseSingle() {
|
||||
let token = this.tokenizer.peek()
|
||||
if(token != null)
|
||||
this.handleSingle(token)
|
||||
}
|
||||
|
||||
parseIdentifier() {
|
||||
// Assuming the right type.
|
||||
let token = this.tokenizer.read(TK.TokenType.IDENTIFIER)
|
||||
if(Reference.CONSTANTS_LIST.includes(token.value))
|
||||
this.stack.push(new AST.Constant(token.value))
|
||||
else
|
||||
this.stack.push(new AST.Variable(token.value))
|
||||
this.checkIdentifierFollowupTokens()
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a function based on a previously called identifier.
|
||||
* NOTE: Expects to have at least one stack element for function name.
|
||||
*/
|
||||
parseFunction() {
|
||||
// TODO: Implement dynamic functions values instead of being based on names.
|
||||
let functionValue = this.stack.pop()
|
||||
if(!(functionValue instanceof AST.Variable))
|
||||
this.tokenizer.raise("Executing functions from dynamic variables is not implemented".)
|
||||
let functionName = functionValue.variableName
|
||||
let args = []
|
||||
let token
|
||||
while((token = this.tokenizer.peek()) != null && token.value != ')') {
|
||||
this.tokenizer.skip(TK.TokenType.PUNCT) // Skip the opening parenthesis and the commas.
|
||||
parseExpression([',',')'])
|
||||
args.push(this.stack.pop())
|
||||
}
|
||||
|
||||
if(token == null)
|
||||
this.tokenizer.raise(`Unexpected end of expression. Expected ')'.`)
|
||||
|
||||
if(this.functionName == 'derivation')
|
||||
this.stack.push(new AST.DerivationElement(args))
|
||||
else if(this.functionName == 'integral')
|
||||
this.stack.push(new AST.IntegralElement(args))
|
||||
else
|
||||
this.stack.push(new AST.FunctionElement(functionName, args))
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parses an object property based on a previously called identifier.
|
||||
* NOTE: Expects to have at least one stack element for property object name.
|
||||
*/
|
||||
parseProperty() {
|
||||
this.tokenizer.skip(TK.TokenType.PUNCT, '.') // Skipping the dot.
|
||||
let token = this.tokenizer.read(TK.TokenType.IDENTIFIER)
|
||||
this.stack.push(new AST.PropertyElement(this.stack.pop(), token.value))
|
||||
this.checkIdentifierFollowupTokens()
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the value of the element of an array at a given index based on a previously called identifier.
|
||||
* NOTE: Expects to have at least one stack element for property object name.
|
||||
*/
|
||||
parseArrayValue() {
|
||||
this.tokenizer.skip(TK.TokenType.PUNCT, '[') // Skipping the array opener.
|
||||
let obj = this.stack.pop()
|
||||
parseExpression([']'])
|
||||
this.stack.push(new AST.ArrayElement(obj, this.stack.pop()))
|
||||
this.checkIdentifierFollowupTokens()
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks for followup tokens following a value getting.
|
||||
* E.g: getting the property of an object, an array member, or calling a function.
|
||||
* NOTE: Expects to have at least one stack element for previous calling object.
|
||||
*/
|
||||
checkIdentifierFollowupTokens() {
|
||||
let peeked = this.tokenizer.peek()
|
||||
if(peeked != null && peeked.type == TK.TokenType.PUNCT)
|
||||
switch(peeked.value) {
|
||||
case '(':
|
||||
// Function call
|
||||
this.parseFunction()
|
||||
break
|
||||
case '.':
|
||||
// Member property
|
||||
this.parseProperty()
|
||||
break
|
||||
case '[':
|
||||
// Array value
|
||||
this.parseArrayValue()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
parseBinaryOperations() {
|
||||
if((this.tokenizer.peek().value in AST.BINARY_OPERATION_PRIORITY))
|
||||
throw new Error("Current token is not a binary operator.")
|
||||
if(this.stack.length == 0)
|
||||
throw new Error(`The operator ${this.tokenizer.peek().value} can only be used after a value.`)
|
||||
// Parse a sequence of operations, and orders them based on OPERATION_PRIORITY.
|
||||
let elements = [this.stack.pop()]
|
||||
let operators = [this.tokenizer.next()]
|
||||
let token
|
||||
while((token = this.tokenizer.peek()) != null) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -28,6 +28,21 @@ const CONSTANTS = {
|
|||
};
|
||||
const CONSTANTS_LIST = Object.keys(CONSTANTS);
|
||||
|
||||
const UNARY_OPERATORS = []
|
||||
|
||||
const BINARY_OPERATION_PRIORITY = {
|
||||
'and': 0, 'or': 0, 'in': 0,
|
||||
'==': 10, '!=' : 10,
|
||||
'>': 10, '<': 10, '>=': 10, '<=': 10,
|
||||
'%': 20, '||': 20,
|
||||
'+': 30, '-': 30,
|
||||
'*': 40, '/': 40,
|
||||
'^': 50
|
||||
}
|
||||
const BINARY_OPERATORS = Object.keys(BINARY_OPERATION_PRIORITY)
|
||||
|
||||
const TERTIARY_OPERATORS = ['?']
|
||||
|
||||
const FUNCTIONS = {
|
||||
"abs": Math.abs,
|
||||
"acos": Math.acos,
|
||||
|
|
|
@ -23,13 +23,14 @@
|
|||
const WHITESPACES = " \t\n\r"
|
||||
const STRING_LIMITORS = '"\'`';
|
||||
const OPERATORS = "+-*/^%?:=!><";
|
||||
const PUNCTUTATION = "()[]{},.";
|
||||
const PUNCTUTATION = "()[],.";
|
||||
const NUMBER_CHARS = "0123456789"
|
||||
const IDENTIFIER_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789_₀₁₂₃₄₅₆₇₈₉αβγδεζηθκλμξρςστφχψωₐₑₒₓₔₕₖₗₘₙₚₛₜ"
|
||||
|
||||
var TokenType = {
|
||||
// Expression type
|
||||
"WHITESPACE": "WHITESPACE",
|
||||
"IDENTIFIER": "IDENTIFIER",
|
||||
"VARIABLE": "VARIABLE",
|
||||
"CONSTANT": "CONSTANT",
|
||||
"FUNCTION": "FUNCTION",
|
||||
|
@ -49,10 +50,11 @@ class Token {
|
|||
}
|
||||
|
||||
class ExpressionTokenizer {
|
||||
constructor(input, tokenizeWhitespaces = false, errorOnUnknown = true) {
|
||||
this.input = input;
|
||||
this.currentToken = null;
|
||||
constructor(input, tokenizeWhitespaces = false, differentiateIdentifiers = false, errorOnUnknown = true) {
|
||||
this.input = input
|
||||
this.currentToken = null
|
||||
this.tokenizeWhitespaces = tokenizeWhitespaces
|
||||
this.differentiateIdentifiers = differentiateIdentifiers
|
||||
this.errorOnUnknown = errorOnUnknown
|
||||
}
|
||||
|
||||
|
@ -115,10 +117,16 @@ class ExpressionTokenizer {
|
|||
while(!this.input.atEnd() && IDENTIFIER_CHARS.includes(this.input.peek().toLowerCase())) {
|
||||
identifier += this.input.next();
|
||||
}
|
||||
if(Reference.CONSTANTS_LIST.includes(identifier.toLowerCase())) {
|
||||
return new Token(TokenType.CONSTANT, identifier.toLowerCase(), this.input.position-identifier.length)
|
||||
} else if(Reference.FUNCTIONS_LIST.includes(identifier.toLowerCase())) {
|
||||
return new Token(TokenType.FUNCTION, identifier.toLowerCase(), this.input.position-identifier.length)
|
||||
let identifierLC = identifier.toLowerCase()
|
||||
if(Reference.CONSTANTS_LIST.includes(identifierLC)) {
|
||||
return new Token(TokenType.CONSTANT, identifierLC, this.input.position-identifier.length)
|
||||
} else if(Reference.FUNCTIONS_LIST.includes(identifierLC)) {
|
||||
return new Token(TokenType.FUNCTION, identifierLC, this.input.position-identifier.length)
|
||||
} else if(Reference.UNARY_OPERATORS.includes(identifierLC) ||
|
||||
Reference.BINARY_OPERATORS.includes(identifierLC) ||
|
||||
Reference.TERTIARY_OPERATORS.includes(identifierLC)
|
||||
) {
|
||||
return new Token(TokenType.OPERATOR, identifierLC, this.input.position-identifier.length)
|
||||
} else {
|
||||
return new Token(TokenType.VARIABLE, identifier, this.input.position-identifier.length)
|
||||
}
|
||||
|
@ -137,7 +145,7 @@ class ExpressionTokenizer {
|
|||
if(Reference.CONSTANTS_LIST.includes(c)) return new Token(TokenType.CONSTANT, this.input.next(), this.input.position-1);
|
||||
if(PUNCTUTATION.includes(c)) return new Token(TokenType.PUNCT, this.input.next(), this.input.position-1);
|
||||
if(this.errorOnUnknown)
|
||||
this.input.throw("Unknown token character " + c)
|
||||
this.raise("Unknown token character " + c)
|
||||
else
|
||||
return new Token(TokenType.UNKNOWN, this.input.next(), this.input.position-1);
|
||||
}
|
||||
|
@ -156,14 +164,25 @@ class ExpressionTokenizer {
|
|||
this.currentToken = null;
|
||||
return tmp;
|
||||
}
|
||||
|
||||
read(type, value) {
|
||||
let next = this.next()
|
||||
if(type != null && next.type != type)
|
||||
this.raise(`Unexpected ${next.type.toLowerCase()} ${next.value}. Expected type was ${type.toLowerCase()}.`);
|
||||
if(value != null && next.value == value)
|
||||
this.raise(`Unexpected ${next.type.toLowerCase()} ${next.value}. Expected value was ${value}.`);
|
||||
return next
|
||||
}
|
||||
|
||||
atEnd() {
|
||||
return this.peek() == null;
|
||||
}
|
||||
|
||||
skip(type) {
|
||||
let next = this.next();
|
||||
if(next.type != type)
|
||||
input.raise("Unexpected token " + next.type.toLowerCase() + ' "' + next.value + '". Expected ' + type.toLowerCase());
|
||||
skip(type, value) {
|
||||
this.read(type, value)
|
||||
}
|
||||
|
||||
raise(message) {
|
||||
this.input.raise(message)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue