', '===', '!==', '**=', '??=', '<<=', '>>=', '...', '&&', '||', '=>', '->', '==', '!=', '<>', '<=', '>=', '++', '--', '+=', '-=', '*=', '/=', '.=', '%=', '&=', '|=', '^=', '??', '::', '<<', '>>', '?>', '(', ')', '[', ']', ',', '!', ';', '=', '<', '>', '+', '-', '*', '/', '%', '.', '&', '|', '^', '~', '?', ':', '@', '`', '{', '}', '\\' ); /** * Tokenize. * * @param string $src Source. * @return array{ok:bool, tokens:array, error:string} */ public static function tokenize( $src ) { $src = (string) $src; $len = strlen( $src ); $i = 0; $tokens = array(); while ( $i < $len ) { $c = $src[ $i ]; if ( ctype_space( $c ) ) { ++$i; continue; } // Comments. if ( '#' === $c || ( '/' === $c && $i + 1 < $len && '/' === $src[ $i + 1 ] ) ) { $start = $i; while ( $i < $len && "\n" !== $src[ $i ] && "\r" !== $src[ $i ] ) { ++$i; } $tokens[] = self::tok( 'comment_line', substr( $src, $start, $i - $start ), $start, $i ); continue; } if ( '/' === $c && $i + 1 < $len && '*' === $src[ $i + 1 ] ) { $close = strpos( $src, '*/', $i + 2 ); if ( false === $close ) { return self::fail( 'unterminated_comment' ); } $tokens[] = self::tok( 'comment_block', substr( $src, $i, $close + 2 - $i ), $i, $close + 2 ); $i = $close + 2; continue; } // Strings. if ( "'" === $c ) { $j = $i + 1; $val = ''; $ok = false; while ( $j < $len ) { $d = $src[ $j ]; if ( '\\' === $d && $j + 1 < $len && ( "'" === $src[ $j + 1 ] || '\\' === $src[ $j + 1 ] ) ) { $val .= $src[ $j + 1 ]; $j += 2; continue; } if ( "'" === $d ) { $ok = true; break; } $val .= $d; ++$j; } if ( ! $ok ) { return self::fail( 'unterminated_string' ); } $tokens[] = self::tok( 'sq', $val, $i, $j + 1 ); $i = $j + 1; continue; } if ( '"' === $c ) { $j = $i + 1; $val = ''; $ok = false; $foreign = false; while ( $j < $len ) { $d = $src[ $j ]; if ( '\\' === $d ) { // Any backslash makes the operand foreign: PHP escape decoding is not emulated. $foreign = true; $val .= $d; if ( $j + 1 < $len ) { $val .= $src[ $j + 1 ]; } $j += 2; continue; } if ( '"' === $d ) { $ok = true; break; } if ( '$' === $d || '{' === $d ) { $foreign = true; } $val .= $d; ++$j; } if ( ! $ok ) { return self::fail( 'unterminated_string' ); } $t = self::tok( 'dq', $val, $i, $j + 1 ); $t['foreign'] = $foreign; $tokens[] = $t; $i = $j + 1; continue; } // Heredoc / nowdoc: not supported, whole string fails to lex. if ( '<' === $c && substr( $src, $i, 3 ) === '<<<' ) { return self::fail( 'heredoc' ); } // Numbers. if ( ctype_digit( $c ) || ( '.' === $c && $i + 1 < $len && ctype_digit( $src[ $i + 1 ] ) ) ) { $j = $i; while ( $j < $len && ( ctype_alnum( $src[ $j ] ) || '_' === $src[ $j ] || '.' === $src[ $j ] ) ) { ++$j; } $raw = substr( $src, $i, $j - $i ); if ( preg_match( '/^(0|[1-9][0-9]*)$/', $raw ) && strlen( $raw ) <= 18 ) { $tokens[] = self::tok( 'int', (int) $raw, $i, $j ); } else { // Hex, octal, binary, underscores, floats, leading zeros: foreign (never re-interpreted). $tokens[] = self::tok( 'num_foreign', $raw, $i, $j ); } $i = $j; continue; } // Variables. if ( '$' === $c ) { $j = $i + 1; while ( $j < $len && ( ctype_alnum( $src[ $j ] ) || '_' === $src[ $j ] || ord( $src[ $j ] ) >= 0x80 ) ) { ++$j; } $tokens[] = self::tok( 'var', substr( $src, $i, $j - $i ), $i, $j ); $i = max( $j, $i + 1 ); continue; } // Identifiers and keywords (case-insensitive, as in PHP). if ( ctype_alpha( $c ) || '_' === $c || ord( $c ) >= 0x80 ) { $j = $i; while ( $j < $len && ( ctype_alnum( $src[ $j ] ) || '_' === $src[ $j ] || ord( $src[ $j ] ) >= 0x80 ) ) { ++$j; } $raw = substr( $src, $i, $j - $i ); $lower = strtolower( $raw ); if ( in_array( $lower, self::KEYWORDS, true ) ) { $t = self::tok( 'kw', $lower, $i, $j ); $t['raw'] = $raw; } else { $t = self::tok( 'ident', $lower, $i, $j ); $t['raw'] = $raw; } $tokens[] = $t; $i = $j; continue; } // Operators and punctuation. $matched = false; foreach ( self::OPS as $op ) { $l = strlen( $op ); if ( substr( $src, $i, $l ) === $op ) { $type = in_array( $op, array( '(', ')', '[', ']', ',', '!', '&&', '||', '=>', ';' ), true ) ? 'punct' : 'op'; $tokens[] = self::tok( $type, $op, $i, $i + $l ); $i += $l; $matched = true; break; } } if ( ! $matched ) { return self::fail( 'unexpected_character' ); } } return array( 'ok' => true, 'tokens' => $tokens, 'error' => '', ); } /** * Make a token. * * @param string $type Type. * @param mixed $value Value. * @param int $start Start offset. * @param int $end End offset. * @return array */ private static function tok( $type, $value, $start, $end ) { return array( 'type' => $type, 'value' => $value, 'start' => $start, 'end' => $end, ); } /** * Failure result. * * @param string $why Detail. * @return array */ private static function fail( $why ) { return array( 'ok' => false, 'tokens' => array(), 'error' => $why, ); } }