src/JmesPath/Lexer.php
public function tokenize($input)
    {
        $tokens = [];

        if ($input === '') {
            goto eof;
        }

        $chars = str_split($input);

        while (false !== ($current = current($chars))) {

            // Every character must be in the transition character table.
            if (!isset(self::$transitionTable[$current])) {
                $tokens[] = [
                    'type'  => self::T_UNKNOWN,
                    'pos'   => key($chars),
                    'value' => $current
                ];
                next($chars);
                continue;
            }

            $state = self::$transitionTable[$current];

            if ($state === self::STATE_SINGLE_CHAR) {

                // Consume simple tokens like ".", ",", "@", etc.
                $tokens[] = [
                    'type'  => $this->simpleTokens[$current],
                    'pos'   => key($chars),
                    'value' => $current
                ];
                next($chars);

            } elseif ($state === self::STATE_IDENTIFIER) {

                // Consume identifiers
                $start = key($chars);
                $buffer = '';
                do {
                    $buffer .= $current;
                    $current = next($chars);
                } while ($current !== false && isset($this->validIdentifier[$current]));
                $tokens[] = [
                    'type'  => self::T_IDENTIFIER,
                    'value' => $buffer,
                    'pos'   => $start
                ];

            } elseif ($state === self::STATE_WHITESPACE) {

                // Skip whitespace
                next($chars);

            } elseif ($state === self::STATE_LBRACKET) {

                // Consume "[", "[?", and "[]"
                $position = key($chars);
                $actual = next($chars);
                if ($actual === ']') {
                    next($chars);
                    $tokens[] = [
                        'type'  => self::T_FLATTEN,
                        'pos'   => $position,
                        'value' => '[]'
                    ];
                } elseif ($actual === '?') {
                    next($chars);
                    $tokens[] = [
                        'type'  => self::T_FILTER,
                        'pos'   => $position,
                        'value' => '[?'
                    ];
                } else {
                    $tokens[] = [
                        'type'  => self::T_LBRACKET,
                        'pos'   => $position,
                        'value' => '['
                    ];
                }

            } elseif ($state === self::STATE_STRING_LITERAL) {

                // Consume raw string literals
                $t = $this->inside($chars, "'", self::T_LITERAL);
                $t['value'] = str_replace("\\'", "'", $t['value']);
                $tokens[] = $t;

            } elseif ($state === self::STATE_PIPE) {

                // Consume pipe and OR
                $tokens[] = $this->matchOr($chars, '|', '|', self::T_OR, self::T_PIPE);

            } elseif ($state == self::STATE_JSON_LITERAL) {

                // Consume JSON literals
                $token = $this->inside($chars, '`', self::T_LITERAL);
                if ($token['type'] === self::T_LITERAL) {
                    $token['value'] = str_replace('\\`', '`', $token['value']);
                    $token = $this->parseJson($token);
                }
                $tokens[] = $token;

            } elseif ($state == self::STATE_NUMBER) {

                // Consume numbers
                $start = key($chars);
                $buffer = '';
                do {
                    $buffer .= $current;
                    $current = next($chars);
                } while ($current !== false && isset($this->numbers[$current]));
                $tokens[] = [
                    'type'  => self::T_NUMBER,
                    'value' => (int)$buffer,
                    'pos'   => $start
                ];

            } elseif ($state === self::STATE_QUOTED_STRING) {

                // Consume quoted identifiers
                $token = $this->inside($chars, '"', self::T_QUOTED_IDENTIFIER);
                if ($token['type'] === self::T_QUOTED_IDENTIFIER) {
                    $token['value'] = '"' . $token['value'] . '"';
                    $token = $this->parseJson($token);
                }
                $tokens[] = $token;

            } elseif ($state === self::STATE_EQ) {

                // Consume equals
                $tokens[] = $this->matchOr($chars, '=', '=', self::T_COMPARATOR, self::T_UNKNOWN);

            } elseif ($state == self::STATE_AND) {

                $tokens[] = $this->matchOr($chars, '&', '&', self::T_AND, self::T_EXPREF);

            } elseif ($state === self::STATE_NOT) {

                // Consume not equal
                $tokens[] = $this->matchOr($chars, '!', '=', self::T_COMPARATOR, self::T_NOT);

            } else {

                // either '<' or '>'
                // Consume less than and greater than
                $tokens[] = $this->matchOr($chars, $current, '=', self::T_COMPARATOR, self::T_COMPARATOR);

            }
        }

        eof:
        $tokens[] = [
            'type'  => self::T_EOF,
            'pos'   => strlen($input),
            'value' => null
        ];

        return $tokens;
    }
public function tokenize($input)
    {
        $tokens = [];

        if ($input === '') {
            goto eof;
        }

        $chars = str_split($input);

        while (false !== ($current = current($chars))) {

            // Every character must be in the transition character table.
            if (!isset(self::$transitionTable[$current])) {
                $tokens[] = [
                    'type'  => self::T_UNKNOWN,
                    'pos'   => key($chars),
                    'value' => $current
                ];
                next($chars);
                continue;
            }

            $state = self::$transitionTable[$current];

            if ($state === self::STATE_SINGLE_CHAR) {

                // Consume simple tokens like ".", ",", "@", etc.
                $tokens[] = [
                    'type'  => $this->simpleTokens[$current],
                    'pos'   => key($chars),
                    'value' => $current
                ];
                next($chars);

            } elseif ($state === self::STATE_IDENTIFIER) {

                // Consume identifiers
                $start = key($chars);
                $buffer = '';
                do {
                    $buffer .= $current;
                    $current = next($chars);
                } while ($current !== false && isset($this->validIdentifier[$current]));
                $tokens[] = [
                    'type'  => self::T_IDENTIFIER,
                    'value' => $buffer,
                    'pos'   => $start
                ];

            } elseif ($state === self::STATE_WHITESPACE) {

                // Skip whitespace
                next($chars);

            } elseif ($state === self::STATE_LBRACKET) {

                // Consume "[", "[?", and "[]"
                $position = key($chars);
                $actual = next($chars);
                if ($actual === ']') {
                    next($chars);
                    $tokens[] = [
                        'type'  => self::T_FLATTEN,
                        'pos'   => $position,
                        'value' => '[]'
                    ];
                } elseif ($actual === '?') {
                    next($chars);
                    $tokens[] = [
                        'type'  => self::T_FILTER,
                        'pos'   => $position,
                        'value' => '[?'
                    ];
                } else {
                    $tokens[] = [
                        'type'  => self::T_LBRACKET,
                        'pos'   => $position,
                        'value' => '['
                    ];
                }

            } elseif ($state === self::STATE_STRING_LITERAL) {

                // Consume raw string literals
                $t = $this->inside($chars, "'", self::T_LITERAL);
                $t['value'] = str_replace("\\'", "'", $t['value']);
                $tokens[] = $t;

            } elseif ($state === self::STATE_PIPE) {

                // Consume pipe and OR
                $tokens[] = $this->matchOr($chars, '|', '|', self::T_OR, self::T_PIPE);

            } elseif ($state == self::STATE_JSON_LITERAL) {

                // Consume JSON literals
                $token = $this->inside($chars, '`', self::T_LITERAL);
                if ($token['type'] === self::T_LITERAL) {
                    $token['value'] = str_replace('\\`', '`', $token['value']);
                    $token = $this->parseJson($token);
                }
                $tokens[] = $token;

            } elseif ($state == self::STATE_NUMBER) {

                // Consume numbers
                $start = key($chars);
                $buffer = '';
                do {
                    $buffer .= $current;
                    $current = next($chars);
                } while ($current !== false && isset($this->numbers[$current]));
                $tokens[] = [
                    'type'  => self::T_NUMBER,
                    'value' => (int)$buffer,
                    'pos'   => $start
                ];

            } elseif ($state === self::STATE_QUOTED_STRING) {

                // Consume quoted identifiers
                $token = $this->inside($chars, '"', self::T_QUOTED_IDENTIFIER);
                if ($token['type'] === self::T_QUOTED_IDENTIFIER) {
                    $token['value'] = '"' . $token['value'] . '"';
                    $token = $this->parseJson($token);
                }
                $tokens[] = $token;

            } elseif ($state === self::STATE_EQ) {

                // Consume equals
                $tokens[] = $this->matchOr($chars, '=', '=', self::T_COMPARATOR, self::T_UNKNOWN);

            } elseif ($state == self::STATE_AND) {

                $tokens[] = $this->matchOr($chars, '&', '&', self::T_AND, self::T_EXPREF);

            } elseif ($state === self::STATE_NOT) {

                // Consume not equal
                $tokens[] = $this->matchOr($chars, '!', '=', self::T_COMPARATOR, self::T_NOT);

            } else {

                // either '<' or '>'
                // Consume less than and greater than
                $tokens[] = $this->matchOr($chars, $current, '=', self::T_COMPARATOR, self::T_COMPARATOR);

            }
        }

        eof:
        $tokens[] = [
            'type'  => self::T_EOF,
            'pos'   => strlen($input),
            'value' => null
        ];

        return $tokens;
    }
public function tokenize ( $ input ) { $ tokens = [ ] ; if ( $ input === '' ) { goto eof ; } $ chars = str_split ( $ input ) ; while ( false !== ( $ current = current ( $ chars ) ) ) { // Every character must be in the transition character table. if ( ! isset ( self :: $ transitionTable [ $ current ] ) ) { $ tokens [ ] = [ 'type' => self :: T_UNKNOWN , 'pos' => key ( $ chars ) , 'value' => $ current ] ; next ( $ chars ) ; continue ; } $ state = self :: $ transitionTable [ $ current ] ; if ( $ state === self :: STATE_SINGLE_CHAR ) { // Consume simple tokens like ".", ",", "@", etc. $ tokens [ ] = [ 'type' => $ this -> simpleTokens [ $ current ] , 'pos' => key ( $ chars ) , 'value' => $ current ] ; next ( $ chars ) ; } elseif ( $ state === self :: STATE_IDENTIFIER ) { // Consume identifiers $ start = key ( $ chars ) ; $ buffer = '' ; do { $ buffer .= $ current ; $ current = next ( $ chars ) ; } while ( $ current !== false && isset ( $ this -> validIdentifier [ $ current ] ) ) ; $ tokens [ ] = [ 'type' => self :: T_IDENTIFIER , 'value' => $ buffer , 'pos' => $ start ] ; } elseif ( $ state === self :: STATE_WHITESPACE ) { // Skip whitespace next ( $ chars ) ; } elseif ( $ state === self :: STATE_LBRACKET ) { // Consume "[", "[?", and "[]" $ position = key ( $ chars ) ; $ actual = next ( $ chars ) ; if ( $ actual === ']' ) { next ( $ chars ) ; $ tokens [ ] = [ 'type' => self :: T_FLATTEN , 'pos' => $ position , 'value' => '[]' ] ; } elseif ( $ actual === '?' ) { next ( $ chars ) ; $ tokens [ ] = [ 'type' => self :: T_FILTER , 'pos' => $ position , 'value' => '[?' ] ; } else { $ tokens [ ] = [ 'type' => self :: T_LBRACKET , 'pos' => $ position , 'value' => '[' ] ; } } elseif ( $ state === self :: STATE_STRING_LITERAL ) { // Consume raw string literals $ t = $ this -> inside ( $ chars , "'" , self :: T_LITERAL ) ; $ t [ 'value' ] = str_replace ( "\\'" , "'" , $ t [ 'value' ] ) ; $ tokens [ ] = $ t ; } elseif ( $ state === self :: STATE_PIPE ) { // Consume pipe and OR $ tokens [ ] = $ this -> matchOr ( $ chars , '|' , '|' , self :: T_OR , self :: T_PIPE ) ; } elseif ( $ state == self :: STATE_JSON_LITERAL ) { // Consume JSON literals $ token = $ this -> inside ( $ chars , '`' , self :: T_LITERAL ) ; if ( $ token [ 'type' ] === self :: T_LITERAL ) { $ token [ 'value' ] = str_replace ( '\\`' , '`' , $ token [ 'value' ] ) ; $ token = $ this -> parseJson ( $ token ) ; } $ tokens [ ] = $ token ; } elseif ( $ state == self :: STATE_NUMBER ) { // Consume numbers $ start = key ( $ chars ) ; $ buffer = '' ; do { $ buffer .= $ current ; $ current = next ( $ chars ) ; } while ( $ current !== false && isset ( $ this -> numbers [ $ current ] ) ) ; $ tokens [ ] = [ 'type' => self :: T_NUMBER , 'value' => ( int ) $ buffer , 'pos' => $ start ] ; } elseif ( $ state === self :: STATE_QUOTED_STRING ) { // Consume quoted identifiers $ token = $ this -> inside ( $ chars , '"' , self :: T_QUOTED_IDENTIFIER ) ; if ( $ token [ 'type' ] === self :: T_QUOTED_IDENTIFIER ) { $ token [ 'value' ] = '"' . $ token [ 'value' ] . '"' ; $ token = $ this -> parseJson ( $ token ) ; } $ tokens [ ] = $ token ; } elseif ( $ state === self :: STATE_EQ ) { // Consume equals $ tokens [ ] = $ this -> matchOr ( $ chars , '=' , '=' , self :: T_COMPARATOR , self :: T_UNKNOWN ) ; } elseif ( $ state == self :: STATE_AND ) { $ tokens [ ] = $ this -> matchOr ( $ chars , '&' , '&' , self :: T_AND , self :: T_EXPREF ) ; } elseif ( $ state === self :: STATE_NOT ) { // Consume not equal $ tokens [ ] = $ this -> matchOr ( $ chars , '!' , '=' , self :: T_COMPARATOR , self :: T_NOT ) ; } else { // either '<' or '>' // Consume less than and greater than $ tokens [ ] = $ this -> matchOr ( $ chars , $ current , '=' , self :: T_COMPARATOR , self :: T_COMPARATOR ) ; } } eof : $ tokens [ ] = [ 'type' => self :: T_EOF , 'pos' => strlen ( $ input ) , 'value' => null ] ; return $ tokens ; }
Tokenize the JMESPath expression into an array of tokens hashes that
contain a 'type', 'value', and 'key'.

@param string $input JMESPath input

@return array
@throws SyntaxErrorException
Tokenize the JMESPath expression into an array of tokens hashes that contain a type value and key .