From 5d0afd498458eeae2f29d7e61cd01b1787b20658 Mon Sep 17 00:00:00 2001 From: Adam Pippin Date: Tue, 16 Feb 2021 23:51:50 -0800 Subject: [PATCH] Reimplement expression parser Moving towards something a little more formal -- eventual goal here is that we can evaluate expressions as far as possible given everything available to us at the time and translate the rest into a cloudformation condition --- app/Cfnpp/Expression/Expression.php | 335 +++++++++--------- .../Expression/ICloudformationNative.php | 10 + app/Cfnpp/Expression/Token.php | 21 +- app/Cfnpp/Expression/Token/NumericLiteral.php | 62 ++++ app/Cfnpp/Expression/Token/OperatorBinary.php | 146 ++++++++ app/Cfnpp/Expression/Token/OperatorUnary.php | 80 +++++ app/Cfnpp/Expression/Token/Parameter.php | 60 ++++ app/Cfnpp/Expression/Token/Variable.php | 54 +++ app/Cfnpp/Expression/TokenBinary.php | 12 + app/Cfnpp/Expression/TokenLiteral.php | 12 + app/Cfnpp/Expression/TokenUnary.php | 12 + app/Util/GraphNode.php | 114 ++++++ app/Util/Stack.php | 39 ++ 13 files changed, 762 insertions(+), 195 deletions(-) create mode 100644 app/Cfnpp/Expression/ICloudformationNative.php create mode 100644 app/Cfnpp/Expression/Token/NumericLiteral.php create mode 100644 app/Cfnpp/Expression/Token/OperatorBinary.php create mode 100644 app/Cfnpp/Expression/Token/OperatorUnary.php create mode 100644 app/Cfnpp/Expression/Token/Parameter.php create mode 100644 app/Cfnpp/Expression/Token/Variable.php create mode 100644 app/Cfnpp/Expression/TokenBinary.php create mode 100644 app/Cfnpp/Expression/TokenLiteral.php create mode 100644 app/Cfnpp/Expression/TokenUnary.php create mode 100644 app/Util/GraphNode.php create mode 100644 app/Util/Stack.php diff --git a/app/Cfnpp/Expression/Expression.php b/app/Cfnpp/Expression/Expression.php index a04c460..19e7f87 100644 --- a/app/Cfnpp/Expression/Expression.php +++ b/app/Cfnpp/Expression/Expression.php @@ -4,231 +4,212 @@ declare(strict_types=1); namespace App\Cfnpp\Expression; -/** - * Expression that can be evaluated. - * - * @author Adam Pippin - */ +use App\Util\GraphNode; +use App\Util\Stack; + class Expression { - /** - * List of Tokens in the expression. - * @var Token[] - */ - protected $tokens; - - /** - * Evaluation stack. - * @var mixed[] - */ - protected $stack; - - /** - * Variables that can be referenced in the expression. - * @var array - */ - protected $variables; - - /** - * Create a new expression. - * - * @param Token[] $tokens - */ - public function __construct(array $tokens) + public const TOKEN_TYPES = [ + Token\NumericLiteral::class, + Token\OperatorUnary::class, + Token\OperatorBinary::class, + Token\Variable::class + ]; + + protected $nodes; + + protected $solved = false; + + public function __construct(string $expression, \App\Engine\IOptions $options) { - $this->tokens = $tokens; + $tokens = static::tokenize($expression); + $nodes = static::parse($tokens); + $this->nodes = static::solve($nodes, $options->getVariables(), array_keys($options->getParameters())); + $this->options = $options; } - /** - * Find all referenced variables so we can do proper ordering of variable - * block evaluate. - * - * @return string[] - */ - public function getReferencedVariables(): array + public function isComplete() { - $variables = []; - foreach ($this->tokens as $token) + // I think this works? If we resolved down to a flat set of values and + // all are scalar, then we're done? + + $complete = true; + + foreach ($this->nodes as $node) { - if ($token instanceof TokenVariable) + if ($node->getValue() instanceof Token || + $node->hasChildren()) { - $variables[] = $token->getName(); + $complete = false; + break; } } - return array_values(array_unique($variables)); + + return $complete; } - /** - * Evaluate the tokens contained in this expression. - * - * @param array $variables variables that can be referenced - * @return mixed if expression evaluated down to a single value a scalar, otherwise an array - */ - public function evaluate(array $variables = []) + public function toArray(): array { - $this->variables = $variables; - $this->stack = []; - $tokens = $this->tokens; + return static::unwrap($this->nodes); + } - while (sizeof($tokens)) - { - $token = array_shift($tokens); - assert(isset($token)); + public function toCloudformation(): array + { + return static::cloudformation($this->nodes); + } - $token_class = get_class($token); - $token_name = substr($token_class, strrpos($token_class, '\\') + 1); - $func = 'evaluate'.$token_name; - if (method_exists($this, $func)) - { - $this->{$func}($token); - } - else + protected static function tokenize(string $expression): array + { + $tokens = []; + while (strlen($expression) > 0) + { + foreach (static::TOKEN_TYPES as $token_class) { - throw new \Exception('Unhandled expression token type: '.basename(get_class($token))); + if ($token_class::isToken($expression)) + { + $tokens[] = $token_class::getToken($expression); + + if (strlen($expression) > 0 && substr($expression, 0, 1) != ' ') + { + throw new \Exception('incompletely consumed token'); + } + $expression = substr($expression, 1); + continue 2; + } } - } - if (sizeof($this->stack) == 1) - { - return end($this->stack); + throw new \Exception('unparseable value'); } - return $this->stack; + return $tokens; } - /** - * Evaluate and mutate the stack given a numeric literal. - * - * @param TokenNumericLiteral $token - * @return void - */ - protected function evaluateTokenNumericLiteral(TokenNumericLiteral $token): void + protected static function parse(array $tokens): array { - $this->push($token->getValue()); + $stack = new Stack(); + + foreach ($tokens as $token) + { + if ($token instanceof TokenLiteral) + { + $stack->push(new GraphNode($token)); + } + elseif ($token instanceof TokenUnary) + { + $node = new GraphNode($token); + $node->appendChild($stack->pop()); + $stack->push($node); + } + elseif ($token instanceof TokenBinary) + { + $node = new GraphNode($token); + $node->appendChild($stack->pop()); + $node->appendChild($stack->pop()); + $stack->push($node); + } + } + return $stack->get(); } - /** - * Evaluate and mutate the stack given a string literal. - * - * @param TokenStringLiteral $token - * @return void - */ - protected function evaluateTokenStringLiteral(TokenStringLiteral $token): void + protected static function solve(array $nodes, array $variables = [], array $parameters = []) { - $this->push($token->getValue()); + static::fillVariables($nodes, $variables, $parameters); + static::collapse($nodes); + return $nodes; } - /** - * Evaluate and mutate the stack given a comparison operator. - * - * $this->pop() == $this->pop() is valid. - * @suppress PhanPluginDuplicateExpressionBinaryOp - * @param TokenOperator $token - * @return void - */ - protected function evaluateTokenOperator(TokenOperator $token): void + protected static function execute(array $nodes, array $variables = [], array $parameters = []) { - switch ($token->getOperator()) - { - case 'eq': - $this->push($this->pop() == $this->pop()); - break; - case 'neq': - $this->push($this->pop() != $this->pop()); - break; - case 'gt': - $this->push($this->pop(1) > $this->pop()); - break; - case 'gte': - $this->push($this->pop(1) >= $this->pop()); - break; - case 'lt': - $this->push($this->pop(1) < $this->pop()); - break; - case 'lte': - $this->push($this->pop(1) <= $this->pop()); - break; - case 'and': - $var1 = $this->pop(); - $var2 = $this->pop(); - $this->push($var1 && $var2); - break; - case 'or': - $var1 = $this->pop(); - $var2 = $this->pop(); - $this->push($var1 || $var2); - break; - case 'not': - $this->push(!$this->pop()); - break; - default: - throw new \Exception('Unhandled comparison operator: '.$token->getOperator()); - } + return static::unwrap(static::solve($nodes, $variables, $parameters)); } - /** - * Evaluate and mutate the stack given a function token. - * - * @param TokenFunction $token - * @return void - */ - protected function evaluateTokenFunction(TokenFunction $token): void + protected static function cloudformation(array $nodes): array { - switch ($token->getFunction()) + foreach ($nodes as $node) { - case 'concat': - $this->push($this->pop(1).$this->pop()); - break; - case 'concat*': - while (sizeof($this->stack) > 1) + $node->walk(static function($node) { + if (!($node->getValue() instanceof ICloudformationNative)) { - $this->push($this->pop(1).$this->pop()); + throw new \Exception('Token '.basename(get_class($node->getValue())).' is not natively supported by CloudFormation.'); } - break; - default: - throw new \Exception('Unhandled function: '.$token->getFunction()); + + $node->setValue($node->getValue()->toCloudformation($node->getChildren())); + $node->clearChildren(); + }); + } + if (sizeof($nodes) > 1) + { + throw new \Exception('Expression cannot be converted to CloudFormation -- contains multiple nodes'); } + return $nodes[0]->getValue(); } - /** - * Evaluate and mutate the stack given a variable reference. - * - * @param TokenVariable $token - * @return void - */ - protected function evaluateTokenVariable(TokenVariable $token) + protected static function fillVariables(array $nodes, array $variables, array $parameters) { - $name = $token->getName(); - if (!isset($this->variables[$name])) + foreach ($nodes as $node) { - throw new \Exception('Undefined variable: '.$name); + $node->walk(static function($node) use ($variables, $parameters) { + if ($node->getValue() instanceof Token\Variable) + { + $var_name = $node->getValue()->getName(); + if (in_array($var_name, $parameters)) + { + $node->getParent()->replaceChild($node, new GraphNode(new Token\Parameter($var_name))); + } + elseif (!isset($variables[$var_name])) + { + throw new \Exception('Undefined variable: '.$var_name); + } + else + { + $node->setValue($variables[$var_name]); + } + } + }); } - $this->push($this->variables[$name]); } - /** - * Pop an item off the stack. - * - * @param int $offset offset from the end of the stack to pop from - * @return mixed - */ - protected function pop(int $offset = 0) + protected static function collapse(array $nodes) { - if (sizeof($this->stack) < $offset + 1) + foreach ($nodes as $node) { - throw new \Exception('Expression stack underflow!'); + $node->walk(static function($node) { + if ($node->getValue() instanceof Token\Parameter) + { + return; + } + + if ($node->getValue() instanceof Token) + { + $result = $node->getValue()->execute($node->getChildren()); + if (is_scalar($result)) + { + $node->setValue($result); + $node->clearChildren(); + } + elseif ($result instanceof Token) + { + $node->setValue($result); + $node->clearChildren(); + } + elseif ($result instanceof GraphNode) + { + $node->getParent()->replaceChild($node, $result); + } + } + }); } - return array_splice($this->stack, -1 * ($offset + 1), 1)[0]; } - /** - * Push an item onto the end of the stack. - * - * @param mixed $value - * @return void - */ - protected function push($value): void + protected static function unwrap(array $nodes) { - array_push($this->stack, $value); + return array_map(static function($node) { + if ($node->hasChildren()) + { + throw new \Exception('Cannot unwrap node: still has children'); + } + + return $node->getValue(); + }, $nodes); } } diff --git a/app/Cfnpp/Expression/ICloudformationNative.php b/app/Cfnpp/Expression/ICloudformationNative.php new file mode 100644 index 0000000..6b3dae1 --- /dev/null +++ b/app/Cfnpp/Expression/ICloudformationNative.php @@ -0,0 +1,10 @@ + + * Token parent class. */ abstract class Token { - /** - * Check whether the stream passed in contains a valid token of this - * type. - * - * @param string $stream - * @return bool - */ abstract public static function isToken(string $stream): bool; - /** - * Consume a token from the stream. - * - * Only valid if isToken is true. - * - * @param string $stream - * @return Token - */ abstract public static function getToken(string &$stream): Token; + + abstract public function execute(?array $arguments = null); } diff --git a/app/Cfnpp/Expression/Token/NumericLiteral.php b/app/Cfnpp/Expression/Token/NumericLiteral.php new file mode 100644 index 0000000..d1bf00c --- /dev/null +++ b/app/Cfnpp/Expression/Token/NumericLiteral.php @@ -0,0 +1,62 @@ +value = $value; + } + + public function getValue() + { + return $this->value; + } + + public static function isToken(string $stream): bool + { + return is_numeric($stream[0]); + } + + public static function getToken(string &$stream): Token + { + $buffer = ''; + for ($i = 0; $i < strlen($stream); $i++) + { + if (preg_match('/^[0-9]$/', $stream[$i])) + { + $buffer .= $stream[$i]; + } + else + { + break; + } + } + + $stream = substr($stream, strlen($buffer)); + + if (stristr($buffer, '.')) + { + $buffer = (float)$buffer; + } + else + { + $buffer = (int)$buffer; + } + + return new NumericLiteral($buffer); + } + + public function execute(?array $arguments = null) + { + return $this->getValue(); + } +} diff --git a/app/Cfnpp/Expression/Token/OperatorBinary.php b/app/Cfnpp/Expression/Token/OperatorBinary.php new file mode 100644 index 0000000..3715f2f --- /dev/null +++ b/app/Cfnpp/Expression/Token/OperatorBinary.php @@ -0,0 +1,146 @@ +operator = $operator; + } + + public function getOperator() + { + return $this->operator; + } + + public static function isToken(string $stream): bool + { + foreach (static::OPERATORS as $operator) + { + if (strlen($stream) >= strlen($operator) && + substr($stream, 0, strlen($operator)) == $operator) + { + return true; + } + } + return false; + } + + public static function getToken(string &$stream): Token + { + foreach (static::OPERATORS as $operator) + { + if (strlen($stream) >= strlen($operator) && + substr($stream, 0, strlen($operator)) == $operator) + { + $operator = substr($stream, 0, strlen($operator)); + $stream = substr($stream, strlen($operator)); + return new OperatorBinary($operator); + } + } + throw new \Exception('Could not parse OperatorBinary'); + } + + public function execute(?array $arguments = null) + { + $value1 = $arguments[0]->getValue(); + $value2 = $arguments[1]->getValue(); + + switch ($this->getOperator()) + { + case 'and': + if (is_scalar($value1) && is_scalar($value2)) + { + return $value1 && $value2; + } + elseif (is_scalar($value1)) + { + return $value1 ? $value2 : false; + } + elseif (is_scalar($value2)) + { + return $value2 ? $value1 : false; + } + elseif ($value1 instanceof Parameter && $value2 instanceof Parameter && $value1->getName() == $value2->getName()) + { + return $value1; + } + else + { + return null; + } + // no break + case 'or': + if (is_scalar($value1) && is_scalar($value2)) + { + return $value1 || $value2; + } + elseif (is_scalar($value1)) + { + return $value1 ? true : $value1; + } + elseif (is_scalar($value2)) + { + return $value2 ? true : $value1; + } + elseif ($value1 instanceof Parameter && $value2 instanceof Parameter && $value1->getName() == $value2->getName()) + { + return $value1; + } + else + { + return null; + } + // no break + case 'eq': + if (is_scalar($value1) && is_scalar($value2)) + { + return $arguments[0] == $arguments[1]; + } + elseif ($value1 instanceof Parameter && $value2 instanceof Parameter && $value1->getName() == $value2->getName()) + { + return true; + } + else + { + return null; + } + // no break + default: + throw new \Exception('Missing implementation for unary operator: '.$this->getOperator()); + } + } + + public function toCloudformation(?array $arguments = null): array + { + $value1 = $arguments[0]->getValue(); + $value2 = $arguments[1]->getValue(); + + switch ($this->getOperator()) + { + case 'and': + return ['Fn::And' => [$value1, $value2]]; + case 'or': + return ['Fn::Or' => [$value1, $value2]]; + case 'eq': + return ['Fn::Equals' => [$value1, $value2]]; + default: + throw new \Exception('Missing implementation for unary operator: '.$this->getOperator()); + } + } +} diff --git a/app/Cfnpp/Expression/Token/OperatorUnary.php b/app/Cfnpp/Expression/Token/OperatorUnary.php new file mode 100644 index 0000000..6fbc4a0 --- /dev/null +++ b/app/Cfnpp/Expression/Token/OperatorUnary.php @@ -0,0 +1,80 @@ +operator = $operator; + } + + public function getOperator() + { + return $this->operator; + } + + public static function isToken(string $stream): bool + { + foreach (static::OPERATORS as $operator) + { + if (strlen($stream) >= strlen($operator) && + substr($stream, 0, strlen($operator)) == $operator) + { + return true; + } + } + return false; + } + + public static function getToken(string &$stream): Token + { + foreach (static::OPERATORS as $operator) + { + if (strlen($stream) >= strlen($operator) && + substr($stream, 0, strlen($operator)) == $operator) + { + $operator = substr($stream, 0, strlen($operator)); + $stream = substr($stream, strlen($operator)); + return new OperatorUnary($operator); + } + } + throw new \Exception('Could not parse OperatorUnary'); + } + + public function execute(?array $arguments = null) + { + $value = $arguments[0]->getValue(); + switch ($this->getOperator()) + { + case 'not': + return is_scalar($value) ? !$value : null; + default: + throw new \Exception('Missing implementation for unary operator: '.$this->getOperator()); + } + } + + public function toCloudformation(?array $arguments = null): array + { + $value = $arguments[0]->getValue(); + switch ($this->getOperator()) + { + case 'not': + return ['Fn::Not' => [$value]]; + default: + throw new \Exception('Missing implementation for unary operator: '.$this->getOperator()); + } + } +} diff --git a/app/Cfnpp/Expression/Token/Parameter.php b/app/Cfnpp/Expression/Token/Parameter.php new file mode 100644 index 0000000..cd1b4be --- /dev/null +++ b/app/Cfnpp/Expression/Token/Parameter.php @@ -0,0 +1,60 @@ +name = $name; + } + + public function getName() + { + return $this->name; + } + + public static function isToken(string $stream): bool + { + return (bool)preg_match('/^[A-Za-z]$/', $stream[0]); + } + + public static function getToken(string &$stream): Token + { + $buffer = ''; + $buffer = $stream[0]; + for ($i = 1; $i < strlen($stream); $i++) + { + if (preg_match('/^[A-Za-z0-9]$/', $stream[$i])) + { + $buffer .= $stream[$i]; + } + else + { + break; + } + } + + $stream = substr($stream, strlen($buffer)); + + return new Parameter($buffer); + } + + public function execute(?array $arguments = null) + { + throw new \Exception('Unreplaced parameter'); + } + + public function toCloudformation(?array $arguments = null): array + { + return ['Ref' => $this->getName()]; + } +} diff --git a/app/Cfnpp/Expression/Token/Variable.php b/app/Cfnpp/Expression/Token/Variable.php new file mode 100644 index 0000000..edb2b06 --- /dev/null +++ b/app/Cfnpp/Expression/Token/Variable.php @@ -0,0 +1,54 @@ +name = $name; + } + + public function getName() + { + return $this->name; + } + + public static function isToken(string $stream): bool + { + return (bool)preg_match('/^[A-Za-z]$/', $stream[0]); + } + + public static function getToken(string &$stream): Token + { + $buffer = ''; + $buffer = $stream[0]; + for ($i = 1; $i < strlen($stream); $i++) + { + if (preg_match('/^[A-Za-z0-9]$/', $stream[$i])) + { + $buffer .= $stream[$i]; + } + else + { + break; + } + } + + $stream = substr($stream, strlen($buffer)); + + return new Variable($buffer); + } + + public function execute(?array $arguments = null) + { + throw new \Exception('Unreplaced variable'); + } +} diff --git a/app/Cfnpp/Expression/TokenBinary.php b/app/Cfnpp/Expression/TokenBinary.php new file mode 100644 index 0000000..1aa96ac --- /dev/null +++ b/app/Cfnpp/Expression/TokenBinary.php @@ -0,0 +1,12 @@ +value = $value; + $this->children = []; + } + + public function __invoke() + { + return $this->value; + } + + public function setValue($value): void + { + $this->value = $value; + } + + public function getValue() + { + return $this->value; + } + + public function setParent(GraphNode $node): void + { + $this->parent = $node; + } + + public function hasParent(): bool + { + return isset($this->parent); + } + + public function getParent(): GraphNode + { + return $this->parent; + } + + public function countChildren(): int + { + return sizeof($this->children); + } + + public function hasChildren(): bool + { + return sizeof($this->children) > 0; + } + + public function getChildren(): array + { + return $this->children; + } + + public function appendChild(GraphNode $child): void + { + $this->children[] = $child; + $child->setParent($this); + } + + public function replaceChild(GraphNode $original, GraphNode $new): void + { + for ($i = 0; $i < sizeof($this->children); $i++) + { + if ($this->children[$i] === $original) + { + $this->children[$i] = $new; + $new->setParent($this); + return; + } + } + } + + public function clearChildren(): void + { + $this->children = []; + } + + public function add($value): GraphNode + { + $this->children[] = new GraphNode($value); + end($this->children)->setParent($this); + return end($this->children); + } + + public function walk(callable $callback) + { + static::walkNodes([$this], $callback); + } + + protected static function walkNodes(array $nodes, callable $callback) + { + foreach ($nodes as $node) + { + if ($node->hasChildren()) + { + static::walkNodes($node->getChildren(), $callback); + } + + $callback($node); + } + } +} diff --git a/app/Util/Stack.php b/app/Util/Stack.php new file mode 100644 index 0000000..f052a53 --- /dev/null +++ b/app/Util/Stack.php @@ -0,0 +1,39 @@ +stack = []; + } + + public function count(): int + { + return sizeof($this->stack); + } + + public function get(): array + { + return $this->stack; + } + + public function pop(int $offset = 0) + { + if (sizeof($this->stack) < $offset + 1) + { + throw new \Exception('Stack underflow!'); + } + return array_splice($this->stack, -1 * ($offset + 1), 1)[0]; + } + + public function push($value): void + { + array_push($this->stack, $value); + } +}