414 lines
19 KiB
PHP
414 lines
19 KiB
PHP
<?php
|
|
/*---------------------------------------------------------------------------------------------
|
|
* Copyright (c) Microsoft Corporation. All rights reserved.
|
|
* Licensed under the MIT License. See License.txt in the project root for license information.
|
|
*--------------------------------------------------------------------------------------------*/
|
|
|
|
namespace Microsoft\PhpParser;
|
|
|
|
// If this predates PHP 7.4, T_COALESCE_EQUAL and T_FN are unavailable.
|
|
// The replacement value is arbitrary - it just has to be different from other values of token constants.
|
|
define(__NAMESPACE__ . '\T_COALESCE_EQUAL', defined('T_COALESCE_EQUAL') ? constant('T_COALESCE_EQUAL') : 'T_COALESCE_EQUAL');
|
|
define(__NAMESPACE__ . '\T_FN', defined('T_FN') ? constant('T_FN') : 'T_FN');
|
|
// If this predates PHP 8.0, T_MATCH is unavailable. The replacement value is arbitrary - it just has to be different from other values of token constants.
|
|
define(__NAMESPACE__ . '\T_MATCH', defined('T_MATCH') ? constant('T_MATCH') : 'T_MATCH');
|
|
define(__NAMESPACE__ . '\T_NULLSAFE_OBJECT_OPERATOR', defined('T_NULLSAFE_OBJECT_OPERATOR') ? constant('T_NULLSAFE_OBJECT_OPERATOR') : 'T_NULLSAFE_OBJECT_OPERATOR');
|
|
define(__NAMESPACE__ . '\T_ATTRIBUTE', defined('T_ATTRIBUTE') ? constant('T_ATTRIBUTE') : 'T_ATTRIBUTE');
|
|
// If this predates PHP 8.1, T_ENUM is unavailable. The replacement value is arbitrary - it just has to be different from other values of token constants.
|
|
define(__NAMESPACE__ . '\T_ENUM', defined('T_ENUM') ? constant('T_ENUM') : 'T_ENUM');
|
|
define(__NAMESPACE__ . '\T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG', defined('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG') ? constant('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG') : 'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG');
|
|
define(__NAMESPACE__ . '\T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG', defined('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG') ? constant('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG') : 'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG');
|
|
define(__NAMESPACE__ . '\T_READONLY', defined('T_READONLY') ? constant('T_READONLY') : 'T_READONLY');
|
|
|
|
/**
|
|
* Tokenizes content using PHP's built-in `token_get_all`, and converts to "lightweight" Token representation.
|
|
*
|
|
* Initially we tried hand-spinning the lexer (see `experiments/Lexer.php`), but we had difficulties optimizing
|
|
* performance (especially when working with Unicode characters.)
|
|
*
|
|
* Class PhpTokenizer
|
|
* @package Microsoft\PhpParser
|
|
*/
|
|
class PhpTokenizer implements TokenStreamProviderInterface {
|
|
public $pos;
|
|
public $endOfFilePos;
|
|
|
|
private $tokensArray;
|
|
|
|
public function __construct($content) {
|
|
$this->tokensArray = $this->getTokensArrayFromContent($content);
|
|
$this->endOfFilePos = \count($this->tokensArray) - 1;
|
|
$this->pos = 0;
|
|
}
|
|
|
|
public function scanNextToken() : Token {
|
|
return $this->pos >= $this->endOfFilePos
|
|
? $this->tokensArray[$this->endOfFilePos]
|
|
: $this->tokensArray[$this->pos++];
|
|
}
|
|
|
|
public function getCurrentPosition() : int {
|
|
return $this->pos;
|
|
}
|
|
|
|
public function setCurrentPosition(int $pos) {
|
|
$this->pos = $pos;
|
|
}
|
|
|
|
public function getEndOfFilePosition() : int {
|
|
return $this->endOfFilePos;
|
|
}
|
|
|
|
public function getTokensArray() : array {
|
|
return $this->tokensArray;
|
|
}
|
|
|
|
/**
|
|
* Return an array of Token object instances created from $content.
|
|
*
|
|
* This method is optimized heavily - this processes every single token being created.
|
|
*
|
|
* @param string $content the raw php code
|
|
* @param ?int $parseContext can be SourceElements when extracting doc comments
|
|
* @param int $initialPos
|
|
* @param bool $treatCommentsAsTrivia
|
|
* @return Token[]
|
|
*/
|
|
public static function getTokensArrayFromContent(
|
|
$content, $parseContext = null, $initialPos = 0, $treatCommentsAsTrivia = true
|
|
) : array {
|
|
if ($parseContext !== null) {
|
|
// If needed, add a prefix so that token_get_all will tokenize the remaining $contents
|
|
$prefix = self::PARSE_CONTEXT_TO_PREFIX[$parseContext];
|
|
$content = $prefix . $content;
|
|
}
|
|
|
|
$tokens = static::tokenGetAll($content, $parseContext);
|
|
|
|
$arr = [];
|
|
$fullStart = $start = $pos = $initialPos;
|
|
if ($parseContext !== null) {
|
|
// If needed, skip over the prefix we added for token_get_all and remove those tokens.
|
|
// This was moved out of the below main loop as an optimization.
|
|
// (the common case of parsing an entire file uses a null parseContext)
|
|
foreach ($tokens as $i => $token) {
|
|
unset($tokens[$i]);
|
|
if (\is_array($token)) {
|
|
$pos += \strlen($token[1]);
|
|
} else {
|
|
$pos += \strlen($token);
|
|
}
|
|
if (\strlen($prefix) < $pos) {
|
|
$fullStart = $start = $pos = $initialPos;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Convert tokens from token_get_all to Token instances,
|
|
// skipping whitespace and (usually, when parseContext is null) comments.
|
|
foreach ($tokens as $token) {
|
|
if (\is_array($token)) {
|
|
$tokenKind = $token[0];
|
|
$strlen = \strlen($token[1]);
|
|
} else {
|
|
$pos += \strlen($token);
|
|
$newTokenKind = self::TOKEN_MAP[$token] ?? TokenKind::Unknown;
|
|
$arr[] = new Token($newTokenKind, $fullStart, $start, $pos - $fullStart);
|
|
$start = $fullStart = $pos;
|
|
continue;
|
|
}
|
|
|
|
$pos += $strlen;
|
|
|
|
// Optimization note: In PHP < 7.2, the switch statement would check case by case,
|
|
// so putting the most common cases first is slightly faster
|
|
switch ($tokenKind) {
|
|
case \T_WHITESPACE:
|
|
$start += $strlen;
|
|
break;
|
|
case \T_STRING:
|
|
$name = \strtolower($token[1]);
|
|
$newTokenKind = TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name;
|
|
$arr[] = new Token($newTokenKind, $fullStart, $start, $pos - $fullStart);
|
|
$start = $fullStart = $pos;
|
|
break;
|
|
case \T_OPEN_TAG:
|
|
$arr[] = new Token(TokenKind::ScriptSectionStartTag, $fullStart, $start, $pos-$fullStart);
|
|
$start = $fullStart = $pos;
|
|
break;
|
|
case \PHP_VERSION_ID >= 80000 ? \T_NAME_QUALIFIED : -1000:
|
|
case \PHP_VERSION_ID >= 80000 ? \T_NAME_FULLY_QUALIFIED : -1001:
|
|
// NOTE: This switch is called on every token of every file being parsed, so this traded performance for readability.
|
|
//
|
|
// PHP's Opcache is able to optimize switches that are exclusively known longs,
|
|
// but not switches that mix strings and longs or have unknown longs.
|
|
// Longs are only known if they're declared within the same *class* or an internal constant (tokenizer).
|
|
//
|
|
// For some reason, the SWITCH_LONG opcode was not generated when the expression was part of a class constant.
|
|
// (seen with php -d opcache.opt_debug_level=0x20000)
|
|
//
|
|
// Use negative values because that's not expected to overlap with token kinds that token_get_all() will return.
|
|
//
|
|
// T_NAME_* was added in php 8.0 to forbid whitespace between parts of names.
|
|
// Here, emulate the tokenization of php 7 by splitting it up into 1 or more tokens.
|
|
foreach (\explode('\\', $token[1]) as $i => $name) {
|
|
if ($i) {
|
|
$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1 + $start - $fullStart);
|
|
$start++;
|
|
$fullStart = $start;
|
|
}
|
|
if ($name === '') {
|
|
continue;
|
|
}
|
|
// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
|
|
$len = \strlen($name);
|
|
$arr[] = new Token(TokenKind::Name, $fullStart, $start, $len + $start - $fullStart);
|
|
$start += $len;
|
|
$fullStart = $start;
|
|
}
|
|
break;
|
|
case \PHP_VERSION_ID >= 80000 ? \T_NAME_RELATIVE : -1002:
|
|
// This is a namespace-relative name: namespace\...
|
|
foreach (\explode('\\', $token[1]) as $i => $name) {
|
|
$len = \strlen($name);
|
|
if (!$i) {
|
|
$arr[] = new Token(TokenKind::NamespaceKeyword, $fullStart, $start, $len + $start - $fullStart);
|
|
$start += $len;
|
|
$fullStart = $start;
|
|
continue;
|
|
}
|
|
$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1);
|
|
$start++;
|
|
|
|
// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
|
|
$arr[] = new Token(TokenKind::Name, $start, $start, $len);
|
|
|
|
$start += $len;
|
|
$fullStart = $start;
|
|
}
|
|
break;
|
|
case \T_COMMENT:
|
|
case \T_DOC_COMMENT:
|
|
if ($treatCommentsAsTrivia) {
|
|
$start += $strlen;
|
|
break;
|
|
}
|
|
// fall through
|
|
default:
|
|
$newTokenKind = self::TOKEN_MAP[$tokenKind] ?? TokenKind::Unknown;
|
|
$arr[] = new Token($newTokenKind, $fullStart, $start, $pos - $fullStart);
|
|
$start = $fullStart = $pos;
|
|
break;
|
|
}
|
|
}
|
|
|
|
$arr[] = new Token(TokenKind::EndOfFileToken, $fullStart, $start, $pos - $fullStart);
|
|
return $arr;
|
|
}
|
|
|
|
/**
|
|
* @param string $content the raw php code
|
|
* @param ?int $parseContext can be SourceElements when extracting doc comments.
|
|
* Having this available may be useful for subclasses to decide whether or not to post-process results, cache results, etc.
|
|
* @return array[]|string[] an array of tokens. When concatenated, these tokens must equal $content.
|
|
*
|
|
* This exists so that it can be overridden in subclasses, e.g. to cache the result of tokenizing entire files.
|
|
* Applications using tolerant-php-parser may often end up needing to use the token stream for other reasons that are hard to do in the resulting AST,
|
|
* such as iterating over T_COMMENTS, checking for inline html,
|
|
* looking up all tokens (including skipped tokens) on a given line, etc.
|
|
*/
|
|
protected static function tokenGetAll(string $content, $parseContext): array
|
|
{
|
|
return @\token_get_all($content);
|
|
}
|
|
|
|
const TOKEN_MAP = [
|
|
T_CLASS_C => TokenKind::Name,
|
|
T_DIR => TokenKind::Name,
|
|
T_FILE => TokenKind::Name,
|
|
T_FUNC_C => TokenKind::Name,
|
|
T_HALT_COMPILER => TokenKind::Name,
|
|
T_METHOD_C => TokenKind::Name,
|
|
T_NS_C => TokenKind::Name,
|
|
T_TRAIT_C => TokenKind::Name,
|
|
T_LINE => TokenKind::Name,
|
|
|
|
T_STRING => TokenKind::Name,
|
|
T_VARIABLE => TokenKind::VariableName,
|
|
|
|
T_ABSTRACT => TokenKind::AbstractKeyword,
|
|
T_LOGICAL_AND => TokenKind::AndKeyword,
|
|
T_ARRAY => TokenKind::ArrayKeyword,
|
|
T_AS => TokenKind::AsKeyword,
|
|
T_BREAK => TokenKind::BreakKeyword,
|
|
T_CALLABLE => TokenKind::CallableKeyword,
|
|
T_CASE => TokenKind::CaseKeyword,
|
|
T_CATCH => TokenKind::CatchKeyword,
|
|
T_CLASS => TokenKind::ClassKeyword,
|
|
T_CLONE => TokenKind::CloneKeyword,
|
|
T_CONST => TokenKind::ConstKeyword,
|
|
T_CONTINUE => TokenKind::ContinueKeyword,
|
|
T_DECLARE => TokenKind::DeclareKeyword,
|
|
T_DEFAULT => TokenKind::DefaultKeyword,
|
|
T_DO => TokenKind::DoKeyword,
|
|
T_ECHO => TokenKind::EchoKeyword,
|
|
T_ELSE => TokenKind::ElseKeyword,
|
|
T_ELSEIF => TokenKind::ElseIfKeyword,
|
|
T_EMPTY => TokenKind::EmptyKeyword,
|
|
T_ENDDECLARE => TokenKind::EndDeclareKeyword,
|
|
T_ENDFOR => TokenKind::EndForKeyword,
|
|
T_ENDFOREACH => TokenKind::EndForEachKeyword,
|
|
T_ENDIF => TokenKind::EndIfKeyword,
|
|
T_ENDSWITCH => TokenKind::EndSwitchKeyword,
|
|
T_ENDWHILE => TokenKind::EndWhileKeyword,
|
|
T_ENUM => TokenKind::EnumKeyword,
|
|
T_EVAL => TokenKind::EvalKeyword,
|
|
T_EXIT => TokenKind::ExitKeyword,
|
|
T_EXTENDS => TokenKind::ExtendsKeyword,
|
|
T_FINAL => TokenKind::FinalKeyword,
|
|
T_FINALLY => TokenKind::FinallyKeyword,
|
|
T_FOR => TokenKind::ForKeyword,
|
|
T_FOREACH => TokenKind::ForeachKeyword,
|
|
T_FN => TokenKind::FnKeyword,
|
|
T_FUNCTION => TokenKind::FunctionKeyword,
|
|
T_GLOBAL => TokenKind::GlobalKeyword,
|
|
T_GOTO => TokenKind::GotoKeyword,
|
|
T_IF => TokenKind::IfKeyword,
|
|
T_IMPLEMENTS => TokenKind::ImplementsKeyword,
|
|
T_INCLUDE => TokenKind::IncludeKeyword,
|
|
T_INCLUDE_ONCE => TokenKind::IncludeOnceKeyword,
|
|
T_INSTANCEOF => TokenKind::InstanceOfKeyword,
|
|
T_INSTEADOF => TokenKind::InsteadOfKeyword,
|
|
T_INTERFACE => TokenKind::InterfaceKeyword,
|
|
T_ISSET => TokenKind::IsSetKeyword,
|
|
T_LIST => TokenKind::ListKeyword,
|
|
T_MATCH => TokenKind::MatchKeyword,
|
|
T_NAMESPACE => TokenKind::NamespaceKeyword,
|
|
T_NEW => TokenKind::NewKeyword,
|
|
T_LOGICAL_OR => TokenKind::OrKeyword,
|
|
T_PRINT => TokenKind::PrintKeyword,
|
|
T_PRIVATE => TokenKind::PrivateKeyword,
|
|
T_PROTECTED => TokenKind::ProtectedKeyword,
|
|
T_PUBLIC => TokenKind::PublicKeyword,
|
|
T_READONLY => TokenKind::ReadonlyKeyword,
|
|
T_REQUIRE => TokenKind::RequireKeyword,
|
|
T_REQUIRE_ONCE => TokenKind::RequireOnceKeyword,
|
|
T_RETURN => TokenKind::ReturnKeyword,
|
|
T_STATIC => TokenKind::StaticKeyword,
|
|
T_SWITCH => TokenKind::SwitchKeyword,
|
|
T_THROW => TokenKind::ThrowKeyword,
|
|
T_TRAIT => TokenKind::TraitKeyword,
|
|
T_TRY => TokenKind::TryKeyword,
|
|
T_UNSET => TokenKind::UnsetKeyword,
|
|
T_USE => TokenKind::UseKeyword,
|
|
T_VAR => TokenKind::VarKeyword,
|
|
T_WHILE => TokenKind::WhileKeyword,
|
|
T_LOGICAL_XOR => TokenKind::XorKeyword,
|
|
T_YIELD => TokenKind::YieldKeyword,
|
|
T_YIELD_FROM => TokenKind::YieldFromKeyword,
|
|
|
|
"[" => TokenKind::OpenBracketToken,
|
|
"]" => TokenKind::CloseBracketToken,
|
|
"(" => TokenKind::OpenParenToken,
|
|
")" => TokenKind::CloseParenToken,
|
|
"{" => TokenKind::OpenBraceToken,
|
|
"}" => TokenKind::CloseBraceToken,
|
|
"." => TokenKind::DotToken,
|
|
T_OBJECT_OPERATOR => TokenKind::ArrowToken,
|
|
T_NULLSAFE_OBJECT_OPERATOR => TokenKind::QuestionArrowToken,
|
|
T_ATTRIBUTE => TokenKind::AttributeToken,
|
|
T_INC => TokenKind::PlusPlusToken,
|
|
T_DEC => TokenKind::MinusMinusToken,
|
|
T_POW => TokenKind::AsteriskAsteriskToken,
|
|
"*" => TokenKind::AsteriskToken,
|
|
"+" => TokenKind::PlusToken,
|
|
"-" => TokenKind::MinusToken,
|
|
"~" => TokenKind::TildeToken,
|
|
"!" => TokenKind::ExclamationToken,
|
|
"$" => TokenKind::DollarToken,
|
|
"/" => TokenKind::SlashToken,
|
|
"%" => TokenKind::PercentToken,
|
|
T_SL => TokenKind::LessThanLessThanToken,
|
|
T_SR => TokenKind::GreaterThanGreaterThanToken,
|
|
"<" => TokenKind::LessThanToken,
|
|
">" => TokenKind::GreaterThanToken,
|
|
T_IS_SMALLER_OR_EQUAL => TokenKind::LessThanEqualsToken,
|
|
T_IS_GREATER_OR_EQUAL => TokenKind::GreaterThanEqualsToken,
|
|
T_IS_EQUAL => TokenKind::EqualsEqualsToken,
|
|
T_IS_IDENTICAL => TokenKind::EqualsEqualsEqualsToken,
|
|
T_IS_NOT_EQUAL => TokenKind::ExclamationEqualsToken,
|
|
T_IS_NOT_IDENTICAL => TokenKind::ExclamationEqualsEqualsToken,
|
|
"^" => TokenKind::CaretToken,
|
|
"|" => TokenKind::BarToken,
|
|
"&" => TokenKind::AmpersandToken,
|
|
T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG => TokenKind::AmpersandToken,
|
|
T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG => TokenKind::AmpersandToken,
|
|
T_BOOLEAN_AND => TokenKind::AmpersandAmpersandToken,
|
|
T_BOOLEAN_OR => TokenKind::BarBarToken,
|
|
":" => TokenKind::ColonToken,
|
|
";" => TokenKind::SemicolonToken,
|
|
"=" => TokenKind::EqualsToken,
|
|
T_POW_EQUAL => TokenKind::AsteriskAsteriskEqualsToken,
|
|
T_MUL_EQUAL => TokenKind::AsteriskEqualsToken,
|
|
T_DIV_EQUAL => TokenKind::SlashEqualsToken,
|
|
T_MOD_EQUAL => TokenKind::PercentEqualsToken,
|
|
T_PLUS_EQUAL => TokenKind::PlusEqualsToken,
|
|
T_MINUS_EQUAL => TokenKind::MinusEqualsToken,
|
|
T_CONCAT_EQUAL => TokenKind::DotEqualsToken,
|
|
T_SL_EQUAL => TokenKind::LessThanLessThanEqualsToken,
|
|
T_SR_EQUAL => TokenKind::GreaterThanGreaterThanEqualsToken,
|
|
T_AND_EQUAL => TokenKind::AmpersandEqualsToken,
|
|
T_XOR_EQUAL => TokenKind::CaretEqualsToken,
|
|
T_OR_EQUAL => TokenKind::BarEqualsToken,
|
|
"," => TokenKind::CommaToken,
|
|
namespace\T_COALESCE_EQUAL => TokenKind::QuestionQuestionEqualsToken,
|
|
T_COALESCE => TokenKind::QuestionQuestionToken,
|
|
T_SPACESHIP => TokenKind::LessThanEqualsGreaterThanToken,
|
|
T_ELLIPSIS => TokenKind::DotDotDotToken,
|
|
T_NS_SEPARATOR => TokenKind::BackslashToken,
|
|
T_PAAMAYIM_NEKUDOTAYIM => TokenKind::ColonColonToken,
|
|
T_DOUBLE_ARROW => TokenKind::DoubleArrowToken, // TODO missing from spec
|
|
|
|
"@" => TokenKind::AtSymbolToken,
|
|
"`" => TokenKind::BacktickToken,
|
|
"?" => TokenKind::QuestionToken,
|
|
|
|
T_LNUMBER => TokenKind::IntegerLiteralToken,
|
|
|
|
T_DNUMBER => TokenKind::FloatingLiteralToken,
|
|
|
|
T_OPEN_TAG => TokenKind::ScriptSectionStartTag,
|
|
T_OPEN_TAG_WITH_ECHO => TokenKind::ScriptSectionStartWithEchoTag,
|
|
T_CLOSE_TAG => TokenKind::ScriptSectionEndTag,
|
|
|
|
T_INLINE_HTML => TokenKind::InlineHtml,
|
|
|
|
"\"" => TokenKind::DoubleQuoteToken,
|
|
"'" => TokenKind::SingleQuoteToken,
|
|
T_ENCAPSED_AND_WHITESPACE => TokenKind::EncapsedAndWhitespace,
|
|
T_DOLLAR_OPEN_CURLY_BRACES => TokenKind::DollarOpenBraceToken,
|
|
T_CURLY_OPEN => TokenKind::OpenBraceDollarToken,
|
|
T_CONSTANT_ENCAPSED_STRING => TokenKind::StringLiteralToken,
|
|
|
|
T_ARRAY_CAST => TokenKind::ArrayCastToken,
|
|
T_BOOL_CAST => TokenKind::BoolCastToken,
|
|
T_DOUBLE_CAST => TokenKind::DoubleCastToken,
|
|
T_INT_CAST => TokenKind::IntCastToken,
|
|
T_OBJECT_CAST => TokenKind::ObjectCastToken,
|
|
T_STRING_CAST => TokenKind::StringCastToken,
|
|
T_UNSET_CAST => TokenKind::UnsetCastToken,
|
|
|
|
T_START_HEREDOC => TokenKind::HeredocStart,
|
|
T_END_HEREDOC => TokenKind::HeredocEnd,
|
|
T_STRING_VARNAME => TokenKind::StringVarname,
|
|
T_COMMENT => TokenKind::CommentToken,
|
|
T_DOC_COMMENT => TokenKind::DocCommentToken,
|
|
T_NUM_STRING => TokenKind::IntegerLiteralToken
|
|
];
|
|
|
|
const PARSE_CONTEXT_TO_PREFIX = [
|
|
ParseContext::SourceElements => "<?php "
|
|
];
|
|
}
|