src/Utility/Tokenizer.php
public static function getTokens(&$source)
    {
        $newline = 0;
        $tokens = token_get_all($source);
        $ntokens = [];
        foreach ($tokens as $token) {
            $tname = is_array($token) ? $token[0] : null;
            $tdata = is_array($token) ? $token[1] : $token;
            $split = preg_split("%(\r\n|\n)%", $tdata, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY);
            if ($tname == T_CONSTANT_ENCAPSED_STRING || substr($tdata, 0, 2) == '/*') {
                $ntokens[] = [$tname, $tdata];
                foreach ($split as $value) {
                    if ($value == "\r\n" || $value == "\n") {
                        $newline++;
                    }
                }
                continue;
            }
            foreach ($split as $data) {
                if ($data == "\r\n" || $data == "\n") {
                    $newline++;
                    $ntokens[] = [self::T_NEWLINE, $data, $newline];
                } else {
                    $ntokens[] = is_array($token) ? [$tname, $data] : $data;
                }
            }
        }
        return $ntokens;
    }
public static function getTokens(&$source)
    {
        $newline = 0;
        $tokens = token_get_all($source);
        $ntokens = [];
        foreach ($tokens as $token) {
            $tname = is_array($token) ? $token[0] : null;
            $tdata = is_array($token) ? $token[1] : $token;
            $split = preg_split("%(\r\n|\n)%", $tdata, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY);
            if ($tname == T_CONSTANT_ENCAPSED_STRING || substr($tdata, 0, 2) == '/*') {
                $ntokens[] = [$tname, $tdata];
                foreach ($split as $value) {
                    if ($value == "\r\n" || $value == "\n") {
                        $newline++;
                    }
                }
                continue;
            }
            foreach ($split as $data) {
                if ($data == "\r\n" || $data == "\n") {
                    $newline++;
                    $ntokens[] = [self::T_NEWLINE, $data, $newline];
                } else {
                    $ntokens[] = is_array($token) ? [$tname, $data] : $data;
                }
            }
        }
        return $ntokens;
    }
public static function getTokens ( & $ source ) { $ newline = 0 ; $ tokens = token_get_all ( $ source ) ; $ ntokens = [ ] ; foreach ( $ tokens as $ token ) { $ tname = is_array ( $ token ) ? $ token [ 0 ] : null ; $ tdata = is_array ( $ token ) ? $ token [ 1 ] : $ token ; $ split = preg_split ( "%(\r\n|\n)%" , $ tdata , - 1 , PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY ) ; if ( $ tname == T_CONSTANT_ENCAPSED_STRING || substr ( $ tdata , 0 , 2 ) == '/*' ) { $ ntokens [ ] = [ $ tname , $ tdata ] ; foreach ( $ split as $ value ) { if ( $ value == "\r\n" || $ value == "\n" ) { $ newline ++ ; } } continue ; } foreach ( $ split as $ data ) { if ( $ data == "\r\n" || $ data == "\n" ) { $ newline ++ ; $ ntokens [ ] = [ self :: T_NEWLINE , $ data , $ newline ] ; } else { $ ntokens [ ] = is_array ( $ token ) ? [ $ tname , $ data ] : $ data ; } } } return $ ntokens ; }
Get tokens use token_get_all() but post process to interpolate new line
markers so we can check the line number of each token.

@param string $source
@return array
Get tokens use token_get_all () but post process to interpolate new line markers so we can check the line number of each token .