FM / Tokenizers

/var/www/vhosts/ihelp.ro/httpdocs/vendor/squizlabs/php_codesniffer/src/Tokenizers/

/var/www/vhosts/ihelp.ro/httpdocs/vendor/squizlabs/php_codesniffer/src/Tokenizers

Name	Size	Mode	Actions
Comment.php	8795	0644	edit dl rm
CSS.php	22203	0644	edit dl rm
JS.php	50179	0644	edit dl rm
PHP.php	145396	0644	edit dl rm
Tokenizer.php	79805	0644	edit dl rm

Edit: /var/www/vhosts/ihelp.ro/httpdocs/vendor/squizlabs/php_codesniffer/src/Tokenizers/JS.php (50179B)

<?php
/**
 * Tokenizes JS code.
 *
 * @author    Greg Sherwood <gsherwood@squiz.net>
 * @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
 * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
 */

namespace PHP_CodeSniffer\Tokenizers;

use PHP_CodeSniffer\Config;
use PHP_CodeSniffer\Exceptions\TokenizerException;
use PHP_CodeSniffer\Util;

class JS extends Tokenizer
{

/**
     * A list of tokens that are allowed to open a scope.
     *
     * This array also contains information about what kind of token the scope
     * opener uses to open and close the scope, if the token strictly requires
     * an opener, if the token can share a scope closer, and who it can be shared
     * with. An example of a token that shares a scope closer is a CASE scope.
     *
     * @var array
     */
    public $scopeOpeners = [
        T_IF       => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => false,
            'shared' => false,
            'with'   => [],
        ],
        T_TRY      => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => true,
            'shared' => false,
            'with'   => [],
        ],
        T_CATCH    => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => true,
            'shared' => false,
            'with'   => [],
        ],
        T_ELSE     => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => false,
            'shared' => false,
            'with'   => [],
        ],
        T_FOR      => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => false,
            'shared' => false,
            'with'   => [],
        ],
        T_CLASS    => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => true,
            'shared' => false,
            'with'   => [],
        ],
        T_FUNCTION => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => false,
            'shared' => false,
            'with'   => [],
        ],
        T_WHILE    => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => false,
            'shared' => false,
            'with'   => [],
        ],
        T_DO       => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => true,
            'shared' => false,
            'with'   => [],
        ],
        T_SWITCH   => [
            'start'  => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
            'end'    => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
            'strict' => true,
            'shared' => false,
            'with'   => [],
        ],
        T_CASE     => [
            'start'  => [T_COLON => T_COLON],
            'end'    => [
                T_BREAK    => T_BREAK,
                T_RETURN   => T_RETURN,
                T_CONTINUE => T_CONTINUE,
                T_THROW    => T_THROW,
            ],
            'strict' => true,
            'shared' => true,
            'with'   => [
                T_DEFAULT => T_DEFAULT,
                T_CASE    => T_CASE,
                T_SWITCH  => T_SWITCH,
            ],
        ],
        T_DEFAULT  => [
            'start'  => [T_COLON => T_COLON],
            'end'    => [
                T_BREAK    => T_BREAK,
                T_RETURN   => T_RETURN,
                T_CONTINUE => T_CONTINUE,
                T_THROW    => T_THROW,
            ],
            'strict' => true,
            'shared' => true,
            'with'   => [
                T_CASE   => T_CASE,
                T_SWITCH => T_SWITCH,
            ],
        ],
    ];

/**
     * A list of tokens that end the scope.
     *
     * This array is just a unique collection of the end tokens
     * from the _scopeOpeners array. The data is duplicated here to
     * save time during parsing of the file.
     *
     * @var array
     */
    public $endScopeTokens = [
        T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
        T_BREAK               => T_BREAK,
    ];

/**
     * A list of special JS tokens and their types.
     *
     * @var array
     */
    protected $tokenValues = [
        'class'     => 'T_CLASS',
        'function'  => 'T_FUNCTION',
        'prototype' => 'T_PROTOTYPE',
        'try'       => 'T_TRY',
        'catch'     => 'T_CATCH',
        'return'    => 'T_RETURN',
        'throw'     => 'T_THROW',
        'break'     => 'T_BREAK',
        'switch'    => 'T_SWITCH',
        'continue'  => 'T_CONTINUE',
        'if'        => 'T_IF',
        'else'      => 'T_ELSE',
        'do'        => 'T_DO',
        'while'     => 'T_WHILE',
        'for'       => 'T_FOR',
        'var'       => 'T_VAR',
        'case'      => 'T_CASE',
        'default'   => 'T_DEFAULT',
        'true'      => 'T_TRUE',
        'false'     => 'T_FALSE',
        'null'      => 'T_NULL',
        'this'      => 'T_THIS',
        'typeof'    => 'T_TYPEOF',
        '('         => 'T_OPEN_PARENTHESIS',
        ')'         => 'T_CLOSE_PARENTHESIS',
        '{'         => 'T_OPEN_CURLY_BRACKET',
        '}'         => 'T_CLOSE_CURLY_BRACKET',
        '['         => 'T_OPEN_SQUARE_BRACKET',
        ']'         => 'T_CLOSE_SQUARE_BRACKET',
        '?'         => 'T_INLINE_THEN',
        '.'         => 'T_OBJECT_OPERATOR',
        '+'         => 'T_PLUS',
        '-'         => 'T_MINUS',
        '*'         => 'T_MULTIPLY',
        '%'         => 'T_MODULUS',
        '/'         => 'T_DIVIDE',
        '^'         => 'T_LOGICAL_XOR',
        ','         => 'T_COMMA',
        ';'         => 'T_SEMICOLON',
        ':'         => 'T_COLON',
        '<'         => 'T_LESS_THAN',
        '>'         => 'T_GREATER_THAN',
        '<<'        => 'T_SL',
        '>>'        => 'T_SR',
        '>>>'       => 'T_ZSR',
        '<<='       => 'T_SL_EQUAL',
        '>>='       => 'T_SR_EQUAL',
        '>>>='      => 'T_ZSR_EQUAL',
        '<='        => 'T_IS_SMALLER_OR_EQUAL',
        '>='        => 'T_IS_GREATER_OR_EQUAL',
        '=>'        => 'T_DOUBLE_ARROW',
        '!'         => 'T_BOOLEAN_NOT',
        '||'        => 'T_BOOLEAN_OR',
        '&&'        => 'T_BOOLEAN_AND',
        '|'         => 'T_BITWISE_OR',
        '&'         => 'T_BITWISE_AND',
        '!='        => 'T_IS_NOT_EQUAL',
        '!=='       => 'T_IS_NOT_IDENTICAL',
        '='         => 'T_EQUAL',
        '=='        => 'T_IS_EQUAL',
        '==='       => 'T_IS_IDENTICAL',
        '-='        => 'T_MINUS_EQUAL',
        '+='        => 'T_PLUS_EQUAL',
        '*='        => 'T_MUL_EQUAL',
        '/='        => 'T_DIV_EQUAL',
        '%='        => 'T_MOD_EQUAL',
        '++'        => 'T_INC',
        '--'        => 'T_DEC',
        '//'        => 'T_COMMENT',
        '/*'        => 'T_COMMENT',
        '/**'       => 'T_DOC_COMMENT',
        '*/'        => 'T_COMMENT',
    ];

/**
     * A list string delimiters.
     *
     * @var array
     */
    protected $stringTokens = [
        '\'' => '\'',
        '"'  => '"',
    ];

/**
     * A list tokens that start and end comments.
     *
     * @var array
     */
    protected $commentTokens = [
        '//'  => null,
        '/*'  => '*/',
        '/**' => '*/',
    ];

/**
     * Initialise the tokenizer.
     *
     * Pre-checks the content to see if it looks minified.
     *
     * @param string                  $content The content to tokenize,
     * @param \PHP_CodeSniffer\Config $config  The config data for the run.
     * @param string                  $eolChar The EOL char used in the content.
     *
     * @return void
     * @throws \PHP_CodeSniffer\Exceptions\TokenizerException If the file appears to be minified.
     */
    public function __construct($content, Config $config, $eolChar='\n')
    {
        if ($this->isMinifiedContent($content, $eolChar) === true) {
            throw new TokenizerException('File appears to be minified and cannot be processed');
        }

parent::__construct($content, $config, $eolChar);

}//end __construct()

/**
     * Creates an array of tokens when given some JS code.
     *
     * @param string $string The string to tokenize.
     *
     * @return array
     */
    public function tokenize($string)
    {
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** START JS TOKENIZING ***".PHP_EOL;
        }

$maxTokenLength = 0;
        foreach ($this->tokenValues as $token => $values) {
            if (strlen($token) > $maxTokenLength) {
                $maxTokenLength = strlen($token);
            }
        }

$tokens          = [];
        $inString        = '';
        $stringChar      = null;
        $inComment       = '';
        $buffer          = '';
        $preStringBuffer = '';
        $cleanBuffer     = false;

$commentTokenizer = new Comment();

$tokens[] = [
            'code'    => T_OPEN_TAG,
            'type'    => 'T_OPEN_TAG',
            'content' => '',
        ];

// Convert newlines to single characters for ease of
        // processing. We will change them back later.
        $string = str_replace($this->eolChar, "\n", $string);

$chars    = str_split($string);
        $numChars = count($chars);
        for ($i = 0; $i < $numChars; $i++) {
            $char = $chars[$i];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                $content       = Util\Common::prepareForOutput($char);
                $bufferContent = Util\Common::prepareForOutput($buffer);

if ($inString !== '') {
                    echo "\t";
                }

if ($inComment !== '') {
                    echo "\t";
                }

echo "\tProcess char $i => $content (buffer: $bufferContent)".PHP_EOL;
            }//end if

if ($inString === '' && $inComment === '' && $buffer !== '') {
                // If the buffer only has whitespace and we are about to
                // add a character, store the whitespace first.
                if (trim($char) !== '' && trim($buffer) === '') {
                    $tokens[] = [
                        'code'    => T_WHITESPACE,
                        'type'    => 'T_WHITESPACE',
                        'content' => str_replace("\n", $this->eolChar, $buffer),
                    ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput($buffer);
                        echo "\t=> Added token T_WHITESPACE ($content)".PHP_EOL;
                    }

$buffer = '';
                }

// If the buffer is not whitespace and we are about to
                // add a whitespace character, store the content first.
                if ($inString === ''
                    && $inComment === ''
                    && trim($char) === ''
                    && trim($buffer) !== ''
                ) {
                    $tokens[] = [
                        'code'    => T_STRING,
                        'type'    => 'T_STRING',
                        'content' => str_replace("\n", $this->eolChar, $buffer),
                    ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput($buffer);
                        echo "\t=> Added token T_STRING ($content)".PHP_EOL;
                    }

$buffer = '';
                }
            }//end if

// Process strings.
            if ($inComment === '' && isset($this->stringTokens[$char]) === true) {
                if ($inString === $char) {
                    // This could be the end of the string, but make sure it
                    // is not escaped first.
                    $escapes = 0;
                    for ($x = ($i - 1); $x >= 0; $x--) {
                        if ($chars[$x] !== '\\') {
                            break;
                        }

$escapes++;
                    }

if ($escapes === 0 || ($escapes % 2) === 0) {
                        // There is an even number escape chars,
                        // so this is not escaped, it is the end of the string.
                        $tokens[] = [
                            'code'    => T_CONSTANT_ENCAPSED_STRING,
                            'type'    => 'T_CONSTANT_ENCAPSED_STRING',
                            'content' => str_replace("\n", $this->eolChar, $buffer).$char,
                        ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            echo "\t\t* found end of string *".PHP_EOL;
                            $content = Util\Common::prepareForOutput($buffer.$char);
                            echo "\t=> Added token T_CONSTANT_ENCAPSED_STRING ($content)".PHP_EOL;
                        }

$buffer          = '';
                        $preStringBuffer = '';
                        $inString        = '';
                        $stringChar      = null;
                        continue;
                    }//end if
                } else if ($inString === '') {
                    $inString        = $char;
                    $stringChar      = $i;
                    $preStringBuffer = $buffer;

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t\t* looking for string closer *".PHP_EOL;
                    }
                }//end if
            }//end if

if ($inString !== '' && $char === "\n") {
                // Unless this newline character is escaped, the string did not
                // end before the end of the line, which means it probably
                // wasn't a string at all (maybe a regex).
                if ($chars[($i - 1)] !== '\\') {
                    $i      = $stringChar;
                    $buffer = $preStringBuffer;
                    $preStringBuffer = '';
                    $inString        = '';
                    $stringChar      = null;
                    $char            = $chars[$i];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t\t* found newline before end of string, bailing *".PHP_EOL;
                    }
                }
            }

$buffer .= $char;

// We don't look for special tokens inside strings,
            // so if we are in a string, we can continue here now
            // that the current char is in the buffer.
            if ($inString !== '') {
                continue;
            }

// Special case for T_DIVIDE which can actually be
            // the start of a regular expression.
            if ($buffer === $char && $char === '/' && $chars[($i + 1)] !== '*') {
                $regex = $this->getRegexToken($i, $string, $chars, $tokens);
                if ($regex !== null) {
                    $tokens[] = [
                        'code'    => T_REGULAR_EXPRESSION,
                        'type'    => 'T_REGULAR_EXPRESSION',
                        'content' => $regex['content'],
                    ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput($regex['content']);
                        echo "\t=> Added token T_REGULAR_EXPRESSION ($content)".PHP_EOL;
                    }

$i           = $regex['end'];
                    $buffer      = '';
                    $cleanBuffer = false;
                    continue;
                }//end if
            }//end if

// Check for known tokens, but ignore tokens found that are not at
            // the end of a string, like FOR and this.FORmat.
            if (isset($this->tokenValues[strtolower($buffer)]) === true
                && (preg_match('|[a-zA-z0-9_]|', $char) === 0
                || isset($chars[($i + 1)]) === false
                || preg_match('|[a-zA-z0-9_]|', $chars[($i + 1)]) === 0)
            ) {
                $matchedToken    = false;
                $lookAheadLength = ($maxTokenLength - strlen($buffer));

if ($lookAheadLength > 0) {
                    // The buffer contains a token type, but we need
                    // to look ahead at the next chars to see if this is
                    // actually part of a larger token. For example,
                    // FOR and FOREACH.
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t\t* buffer possibly contains token, looking ahead $lookAheadLength chars *".PHP_EOL;
                    }

$charBuffer = $buffer;
                    for ($x = 1; $x <= $lookAheadLength; $x++) {
                        if (isset($chars[($i + $x)]) === false) {
                            break;
                        }

$charBuffer .= $chars[($i + $x)];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $content = Util\Common::prepareForOutput($charBuffer);
                            echo "\t\t=> Looking ahead $x chars => $content".PHP_EOL;
                        }

if (isset($this->tokenValues[strtolower($charBuffer)]) === true) {
                            // We've found something larger that matches
                            // so we can ignore this char. Except for 1 very specific
                            // case where a comment like /**/ needs to tokenize as
                            // T_COMMENT and not T_DOC_COMMENT.
                            $oldType = $this->tokenValues[strtolower($buffer)];
                            $newType = $this->tokenValues[strtolower($charBuffer)];
                            if ($oldType === 'T_COMMENT'
                                && $newType === 'T_DOC_COMMENT'
                                && $chars[($i + $x + 1)] === '/'
                            ) {
                                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                    echo "\t\t* look ahead ignored T_DOC_COMMENT, continuing *".PHP_EOL;
                                }
                            } else {
                                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                    echo "\t\t* look ahead found more specific token ($newType), ignoring $i *".PHP_EOL;
                                }

$matchedToken = true;
                                break;
                            }
                        }//end if
                    }//end for
                }//end if

if ($matchedToken === false) {
                    if (PHP_CODESNIFFER_VERBOSITY > 1 && $lookAheadLength > 0) {
                        echo "\t\t* look ahead found nothing *".PHP_EOL;
                    }

$value = $this->tokenValues[strtolower($buffer)];

if ($value === 'T_FUNCTION' && $buffer !== 'function') {
                        // The function keyword needs to be all lowercase or else
                        // it is just a function called "Function".
                        $value = 'T_STRING';
                    }

$tokens[] = [
                        'code'    => constant($value),
                        'type'    => $value,
                        'content' => $buffer,
                    ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput($buffer);
                        echo "\t=> Added token $value ($content)".PHP_EOL;
                    }

$cleanBuffer = true;
                }//end if
            } else if (isset($this->tokenValues[strtolower($char)]) === true) {
                // No matter what token we end up using, we don't
                // need the content in the buffer any more because we have
                // found a valid token.
                $newContent = substr(str_replace("\n", $this->eolChar, $buffer), 0, -1);
                if ($newContent !== '') {
                    $tokens[] = [
                        'code'    => T_STRING,
                        'type'    => 'T_STRING',
                        'content' => $newContent,
                    ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput(substr($buffer, 0, -1));
                        echo "\t=> Added token T_STRING ($content)".PHP_EOL;
                    }
                }

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    echo "\t\t* char is token, looking ahead ".($maxTokenLength - 1).' chars *'.PHP_EOL;
                }

// The char is a token type, but we need to look ahead at the
                // next chars to see if this is actually part of a larger token.
                // For example, = and ===.
                $charBuffer   = $char;
                $matchedToken = false;
                for ($x = 1; $x <= $maxTokenLength; $x++) {
                    if (isset($chars[($i + $x)]) === false) {
                        break;
                    }

$charBuffer .= $chars[($i + $x)];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput($charBuffer);
                        echo "\t\t=> Looking ahead $x chars => $content".PHP_EOL;
                    }

if (isset($this->tokenValues[strtolower($charBuffer)]) === true) {
                        // We've found something larger that matches
                        // so we can ignore this char.
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $type = $this->tokenValues[strtolower($charBuffer)];
                            echo "\t\t* look ahead found more specific token ($type), ignoring $i *".PHP_EOL;
                        }

$matchedToken = true;
                        break;
                    }
                }//end for

if ($matchedToken === false) {
                    $value    = $this->tokenValues[strtolower($char)];
                    $tokens[] = [
                        'code'    => constant($value),
                        'type'    => $value,
                        'content' => $char,
                    ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t\t* look ahead found nothing *".PHP_EOL;
                        $content = Util\Common::prepareForOutput($char);
                        echo "\t=> Added token $value ($content)".PHP_EOL;
                    }

$cleanBuffer = true;
                } else {
                    $buffer = $char;
                }//end if
            }//end if

// Keep track of content inside comments.
            if ($inComment === ''
                && array_key_exists($buffer, $this->commentTokens) === true
            ) {
                // This is not really a comment if the content
                // looks like \// (i.e., it is escaped).
                if (isset($chars[($i - 2)]) === true && $chars[($i - 2)] === '\\') {
                    $lastToken   = array_pop($tokens);
                    $lastContent = $lastToken['content'];
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $value   = $this->tokenValues[strtolower($lastContent)];
                        $content = Util\Common::prepareForOutput($lastContent);
                        echo "\t=> Removed token $value ($content)".PHP_EOL;
                    }

$lastChars    = str_split($lastContent);
                    $lastNumChars = count($lastChars);
                    for ($x = 0; $x < $lastNumChars; $x++) {
                        $lastChar = $lastChars[$x];
                        $value    = $this->tokenValues[strtolower($lastChar)];
                        $tokens[] = [
                            'code'    => constant($value),
                            'type'    => $value,
                            'content' => $lastChar,
                        ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $content = Util\Common::prepareForOutput($lastChar);
                            echo "\t=> Added token $value ($content)".PHP_EOL;
                        }
                    }
                } else {
                    // We have started a comment.
                    $inComment = $buffer;

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t\t* looking for end of comment *".PHP_EOL;
                    }
                }//end if
            } else if ($inComment !== '') {
                if ($this->commentTokens[$inComment] === null) {
                    // Comment ends at the next newline.
                    if (strpos($buffer, "\n") !== false) {
                        $inComment = '';
                    }
                } else {
                    if ($this->commentTokens[$inComment] === $buffer) {
                        $inComment = '';
                    }
                }

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    if ($inComment === '') {
                        echo "\t\t* found end of comment *".PHP_EOL;
                    }
                }

if ($inComment === '' && $cleanBuffer === false) {
                    $tokens[] = [
                        'code'    => T_STRING,
                        'type'    => 'T_STRING',
                        'content' => str_replace("\n", $this->eolChar, $buffer),
                    ];

$buffer = '';
                }
            }//end if

if ($cleanBuffer === true) {
                $buffer      = '';
                $cleanBuffer = false;
            }
        }//end for

if (empty($buffer) === false) {
            if ($inString !== '') {
                // The string did not end before the end of the file,
                // which means there was probably a syntax error somewhere.
                $tokens[] = [
                    'code'    => T_STRING,
                    'type'    => 'T_STRING',
                    'content' => str_replace("\n", $this->eolChar, $buffer),
                ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    $content = Util\Common::prepareForOutput($buffer);
                    echo "\t=> Added token T_STRING ($content)".PHP_EOL;
                }
            } else {
                // Buffer contains whitespace from the end of the file.
                $tokens[] = [
                    'code'    => T_WHITESPACE,
                    'type'    => 'T_WHITESPACE',
                    'content' => str_replace("\n", $this->eolChar, $buffer),
                ];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    $content = Util\Common::prepareForOutput($buffer);
                    echo "\t=> Added token T_WHITESPACE ($content)".PHP_EOL;
                }
            }//end if
        }//end if

$tokens[] = [
            'code'    => T_CLOSE_TAG,
            'type'    => 'T_CLOSE_TAG',
            'content' => '',
        ];

/*
            Now that we have done some basic tokenizing, we need to
            modify the tokens to join some together and split some apart
            so they match what the PHP tokenizer does.
        */

$finalTokens = [];
        $newStackPtr = 0;
        $numTokens   = count($tokens);
        for ($stackPtr = 0; $stackPtr < $numTokens; $stackPtr++) {
            $token = $tokens[$stackPtr];

/*
                Look for comments and join the tokens together.
            */

if ($token['code'] === T_COMMENT || $token['code'] === T_DOC_COMMENT) {
                $newContent   = '';
                $tokenContent = $token['content'];

$endContent = null;
                if (isset($this->commentTokens[$tokenContent]) === true) {
                    $endContent = $this->commentTokens[$tokenContent];
                }

while ($tokenContent !== $endContent) {
                    if ($endContent === null
                        && strpos($tokenContent, $this->eolChar) !== false
                    ) {
                        // A null end token means the comment ends at the end of
                        // the line so we look for newlines and split the token.
                        $tokens[$stackPtr]['content'] = substr(
                            $tokenContent,
                            (strpos($tokenContent, $this->eolChar) + strlen($this->eolChar))
                        );

$tokenContent = substr(
                            $tokenContent,
                            0,
                            (strpos($tokenContent, $this->eolChar) + strlen($this->eolChar))
                        );

// If the substr failed, skip the token as the content
                        // will now be blank.
                        if ($tokens[$stackPtr]['content'] !== false
                            && $tokens[$stackPtr]['content'] !== ''
                        ) {
                            $stackPtr--;
                        }

break;
                    }//end if

$stackPtr++;
                    $newContent .= $tokenContent;
                    if (isset($tokens[$stackPtr]) === false) {
                        break;
                    }

$tokenContent = $tokens[$stackPtr]['content'];
                }//end while

if ($token['code'] === T_DOC_COMMENT) {
                    $commentTokens = $commentTokenizer->tokenizeString($newContent.$tokenContent, $this->eolChar, $newStackPtr);
                    foreach ($commentTokens as $commentToken) {
                        $finalTokens[$newStackPtr] = $commentToken;
                        $newStackPtr++;
                    }

continue;
                } else {
                    // Save the new content in the current token so
                    // the code below can chop it up on newlines.
                    $token['content'] = $newContent.$tokenContent;
                }
            }//end if

/*
                If this token has newlines in its content, split each line up
                and create a new token for each line. We do this so it's easier
                to ascertain where errors occur on a line.
                Note that $token[1] is the token's content.
            */

if (strpos($token['content'], $this->eolChar) !== false) {
                $tokenLines = explode($this->eolChar, $token['content']);
                $numLines   = count($tokenLines);

for ($i = 0; $i < $numLines; $i++) {
                    $newToken = ['content' => $tokenLines[$i]];
                    if ($i === ($numLines - 1)) {
                        if ($tokenLines[$i] === '') {
                            break;
                        }
                    } else {
                        $newToken['content'] .= $this->eolChar;
                    }

$newToken['type']          = $token['type'];
                    $newToken['code']          = $token['code'];
                    $finalTokens[$newStackPtr] = $newToken;
                    $newStackPtr++;
                }
            } else {
                $finalTokens[$newStackPtr] = $token;
                $newStackPtr++;
            }//end if

// Convert numbers, including decimals.
            if ($token['code'] === T_STRING
                || $token['code'] === T_OBJECT_OPERATOR
            ) {
                $newContent  = '';
                $oldStackPtr = $stackPtr;
                while (preg_match('|^[0-9\.]+$|', $tokens[$stackPtr]['content']) !== 0) {
                    $newContent .= $tokens[$stackPtr]['content'];
                    $stackPtr++;
                }

if ($newContent !== '' && $newContent !== '.') {
                    $finalTokens[($newStackPtr - 1)]['content'] = $newContent;
                    if (ctype_digit($newContent) === true) {
                        $finalTokens[($newStackPtr - 1)]['code'] = constant('T_LNUMBER');
                        $finalTokens[($newStackPtr - 1)]['type'] = 'T_LNUMBER';
                    } else {
                        $finalTokens[($newStackPtr - 1)]['code'] = constant('T_DNUMBER');
                        $finalTokens[($newStackPtr - 1)]['type'] = 'T_DNUMBER';
                    }

$stackPtr--;
                    continue;
                } else {
                    $stackPtr = $oldStackPtr;
                }
            }//end if

// Convert the token after an object operator into a string, in most cases.
            if ($token['code'] === T_OBJECT_OPERATOR) {
                for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
                    if (isset(Util\Tokens::$emptyTokens[$tokens[$i]['code']]) === true) {
                        continue;
                    }

if ($tokens[$i]['code'] !== T_PROTOTYPE
                        && $tokens[$i]['code'] !== T_LNUMBER
                        && $tokens[$i]['code'] !== T_DNUMBER
                    ) {
                        $tokens[$i]['code'] = T_STRING;
                        $tokens[$i]['type'] = 'T_STRING';
                    }

break;
                }
            }
        }//end for

if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** END TOKENIZING ***".PHP_EOL;
        }

return $finalTokens;

}//end tokenize()

/**
     * Tokenizes a regular expression if one is found.
     *
     * If a regular expression is not found, NULL is returned.
     *
     * @param string $char   The index of the possible regex start character.
     * @param string $string The complete content of the string being tokenized.
     * @param string $chars  An array of characters being tokenized.
     * @param string $tokens The current array of tokens found in the string.
     *
     * @return array<string, string>|null
     */
    public function getRegexToken($char, $string, $chars, $tokens)
    {
        $beforeTokens = [
            T_EQUAL               => true,
            T_IS_NOT_EQUAL        => true,
            T_IS_IDENTICAL        => true,
            T_IS_NOT_IDENTICAL    => true,
            T_OPEN_PARENTHESIS    => true,
            T_OPEN_SQUARE_BRACKET => true,
            T_RETURN              => true,
            T_BOOLEAN_OR          => true,
            T_BOOLEAN_AND         => true,
            T_BOOLEAN_NOT         => true,
            T_BITWISE_OR          => true,
            T_BITWISE_AND         => true,
            T_COMMA               => true,
            T_COLON               => true,
            T_TYPEOF              => true,
            T_INLINE_THEN         => true,
            T_INLINE_ELSE         => true,
        ];

$afterTokens = [
            ','            => true,
            ')'            => true,
            ']'            => true,
            ';'            => true,
            ' '            => true,
            '.'            => true,
            ':'            => true,
            $this->eolChar => true,
        ];

// Find the last non-whitespace token that was added
        // to the tokens array.
        $numTokens = count($tokens);
        for ($prev = ($numTokens - 1); $prev >= 0; $prev--) {
            if (isset(Util\Tokens::$emptyTokens[$tokens[$prev]['code']]) === false) {
                break;
            }
        }

if (isset($beforeTokens[$tokens[$prev]['code']]) === false) {
            return null;
        }

// This is probably a regular expression, so look for the end of it.
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t* token possibly starts a regular expression *".PHP_EOL;
        }

$numChars = count($chars);
        for ($next = ($char + 1); $next < $numChars; $next++) {
            if ($chars[$next] === '/') {
                // Just make sure this is not escaped first.
                if ($chars[($next - 1)] !== '\\') {
                    // In the simple form: /.../ so we found the end.
                    break;
                } else if ($chars[($next - 2)] === '\\') {
                    // In the form: /...\\/ so we found the end.
                    break;
                }
            } else {
                $possibleEolChar = substr($string, $next, strlen($this->eolChar));
                if ($possibleEolChar === $this->eolChar) {
                    // This is the last token on the line and regular
                    // expressions need to be defined on a single line,
                    // so this is not a regular expression.
                    break;
                }
            }
        }

if ($chars[$next] !== '/') {
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                echo "\t* could not find end of regular expression *".PHP_EOL;
            }

return null;
        }

while (preg_match('|[a-zA-Z]|', $chars[($next + 1)]) !== 0) {
            // The token directly after the end of the regex can
            // be modifiers like global and case insensitive
            // (.e.g, /pattern/gi).
            $next++;
        }

$regexEnd = $next;
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t* found end of regular expression at token $regexEnd *".PHP_EOL;
        }

for ($next += 1; $next < $numChars; $next++) {
            if ($chars[$next] !== ' ') {
                break;
            } else {
                $possibleEolChar = substr($string, $next, strlen($this->eolChar));
                if ($possibleEolChar === $this->eolChar) {
                    // This is the last token on the line.
                    break;
                }
            }
        }

if (isset($afterTokens[$chars[$next]]) === false) {
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                echo "\t* tokens after regular expression do not look correct *".PHP_EOL;
            }

return null;
        }

// This is a regular expression, so join all the tokens together.
        $content = '';
        for ($x = $char; $x <= $regexEnd; $x++) {
            $content .= $chars[$x];
        }

$token = [
            'start'   => $char,
            'end'     => $regexEnd,
            'content' => $content,
        ];

return $token;

}//end getRegexToken()

/**
     * Performs additional processing after main tokenizing.
     *
     * This additional processing looks for properties, closures, labels and objects.
     *
     * @return void
     */
    public function processAdditional()
    {
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** START ADDITIONAL JS PROCESSING ***".PHP_EOL;
        }

$numTokens  = count($this->tokens);
        $classStack = [];

for ($i = 0; $i < $numTokens; $i++) {
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                $type    = $this->tokens[$i]['type'];
                $content = Util\Common::prepareForOutput($this->tokens[$i]['content']);

echo str_repeat("\t", count($classStack));
                echo "\tProcess token $i: $type => $content".PHP_EOL;
            }

// Looking for functions that are actually closures.
            if ($this->tokens[$i]['code'] === T_FUNCTION && isset($this->tokens[$i]['scope_opener']) === true) {
                for ($x = ($i + 1); $x < $numTokens; $x++) {
                    if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
                        break;
                    }
                }

if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS) {
                    $this->tokens[$i]['code'] = T_CLOSURE;
                    $this->tokens[$i]['type'] = 'T_CLOSURE';
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $line = $this->tokens[$i]['line'];
                        echo str_repeat("\t", count($classStack));
                        echo "\t* token $i on line $line changed from T_FUNCTION to T_CLOSURE *".PHP_EOL;
                    }

for ($x = ($this->tokens[$i]['scope_opener'] + 1); $x < $this->tokens[$i]['scope_closer']; $x++) {
                        if (isset($this->tokens[$x]['conditions'][$i]) === false) {
                            continue;
                        }

$this->tokens[$x]['conditions'][$i] = T_CLOSURE;
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $type = $this->tokens[$x]['type'];
                            echo str_repeat("\t", count($classStack));
                            echo "\t\t* cleaned $x ($type) *".PHP_EOL;
                        }
                    }
                }//end if

continue;
            } else if ($this->tokens[$i]['code'] === T_OPEN_CURLY_BRACKET
                && isset($this->tokens[$i]['scope_condition']) === false
                && isset($this->tokens[$i]['bracket_closer']) === true
            ) {
                $condition = $this->tokens[$i]['conditions'];
                $condition = end($condition);
                if ($condition === T_CLASS) {
                    // Possibly an ES6 method. To be classified as one, the previous
                    // non-empty tokens need to be a set of parenthesis, and then a string
                    // (the method name).
                    for ($parenCloser = ($i - 1); $parenCloser > 0; $parenCloser--) {
                        if (isset(Util\Tokens::$emptyTokens[$this->tokens[$parenCloser]['code']]) === false) {
                            break;
                        }
                    }

if ($this->tokens[$parenCloser]['code'] === T_CLOSE_PARENTHESIS) {
                        $parenOpener = $this->tokens[$parenCloser]['parenthesis_opener'];
                        for ($name = ($parenOpener - 1); $name > 0; $name--) {
                            if (isset(Util\Tokens::$emptyTokens[$this->tokens[$name]['code']]) === false) {
                                break;
                            }
                        }

if ($this->tokens[$name]['code'] === T_STRING) {
                            // We found a method name.
                            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                $line = $this->tokens[$name]['line'];
                                echo str_repeat("\t", count($classStack));
                                echo "\t* token $name on line $line changed from T_STRING to T_FUNCTION *".PHP_EOL;
                            }

$closer = $this->tokens[$i]['bracket_closer'];

$this->tokens[$name]['code'] = T_FUNCTION;
                            $this->tokens[$name]['type'] = 'T_FUNCTION';

foreach ([$name, $i, $closer] as $token) {
                                $this->tokens[$token]['scope_condition']    = $name;
                                $this->tokens[$token]['scope_opener']       = $i;
                                $this->tokens[$token]['scope_closer']       = $closer;
                                $this->tokens[$token]['parenthesis_opener'] = $parenOpener;
                                $this->tokens[$token]['parenthesis_closer'] = $parenCloser;
                                $this->tokens[$token]['parenthesis_owner']  = $name;
                            }

$this->tokens[$parenOpener]['parenthesis_owner'] = $name;
                            $this->tokens[$parenCloser]['parenthesis_owner'] = $name;

for ($x = ($i + 1); $x < $closer; $x++) {
                                $this->tokens[$x]['conditions'][$name] = T_FUNCTION;
                                ksort($this->tokens[$x]['conditions'], SORT_NUMERIC);
                                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                    $type = $this->tokens[$x]['type'];
                                    echo str_repeat("\t", count($classStack));
                                    echo "\t\t* added T_FUNCTION condition to $x ($type) *".PHP_EOL;
                                }
                            }

continue;
                        }//end if
                    }//end if
                }//end if

$classStack[] = $i;

$closer = $this->tokens[$i]['bracket_closer'];
                $this->tokens[$i]['code']      = T_OBJECT;
                $this->tokens[$i]['type']      = 'T_OBJECT';
                $this->tokens[$closer]['code'] = T_CLOSE_OBJECT;
                $this->tokens[$closer]['type'] = 'T_CLOSE_OBJECT';

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    echo str_repeat("\t", count($classStack));
                    echo "\t* token $i converted from T_OPEN_CURLY_BRACKET to T_OBJECT *".PHP_EOL;
                    echo str_repeat("\t", count($classStack));
                    echo "\t* token $closer converted from T_CLOSE_CURLY_BRACKET to T_CLOSE_OBJECT *".PHP_EOL;
                }

for ($x = ($i + 1); $x < $closer; $x++) {
                    $this->tokens[$x]['conditions'][$i] = T_OBJECT;
                    ksort($this->tokens[$x]['conditions'], SORT_NUMERIC);
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $type = $this->tokens[$x]['type'];
                        echo str_repeat("\t", count($classStack));
                        echo "\t\t* added T_OBJECT condition to $x ($type) *".PHP_EOL;
                    }
                }
            } else if ($this->tokens[$i]['code'] === T_CLOSE_OBJECT) {
                $opener = array_pop($classStack);
            } else if ($this->tokens[$i]['code'] === T_COLON) {
                // If it is a scope opener, it belongs to a
                // DEFAULT or CASE statement.
                if (isset($this->tokens[$i]['scope_condition']) === true) {
                    continue;
                }

// Make sure this is not part of an inline IF statement.
                for ($x = ($i - 1); $x >= 0; $x--) {
                    if ($this->tokens[$x]['code'] === T_INLINE_THEN) {
                        $this->tokens[$i]['code'] = T_INLINE_ELSE;
                        $this->tokens[$i]['type'] = 'T_INLINE_ELSE';

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            echo str_repeat("\t", count($classStack));
                            echo "\t* token $i converted from T_COLON to T_INLINE_THEN *".PHP_EOL;
                        }

continue(2);
                    } else if ($this->tokens[$x]['line'] < $this->tokens[$i]['line']) {
                        break;
                    }
                }

// The string to the left of the colon is either a property or label.
                for ($label = ($i - 1); $label >= 0; $label--) {
                    if (isset(Util\Tokens::$emptyTokens[$this->tokens[$label]['code']]) === false) {
                        break;
                    }
                }

if ($this->tokens[$label]['code'] !== T_STRING
                    && $this->tokens[$label]['code'] !== T_CONSTANT_ENCAPSED_STRING
                ) {
                    continue;
                }

if (empty($classStack) === false) {
                    $this->tokens[$label]['code'] = T_PROPERTY;
                    $this->tokens[$label]['type'] = 'T_PROPERTY';

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo str_repeat("\t", count($classStack));
                        echo "\t* token $label converted from T_STRING to T_PROPERTY *".PHP_EOL;
                    }
                } else {
                    $this->tokens[$label]['code'] = T_LABEL;
                    $this->tokens[$label]['type'] = 'T_LABEL';

if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo str_repeat("\t", count($classStack));
                        echo "\t* token $label converted from T_STRING to T_LABEL *".PHP_EOL;
                    }
                }//end if
            }//end if
        }//end for

if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** END ADDITIONAL JS PROCESSING ***".PHP_EOL;
        }

}//end processAdditional()

}//end class

cmd: