*/ private $host_last = array(); /** * Check a URL. * * @param string $url Normalized absolute URL. */ public function check( string $url ): Response { $started = microtime( true ); $target = self::request_url( $url ); $response = $this->follow( $target, 'HEAD' ); if ( $this->needs_get_retry( $response ) ) { $response = $this->follow( $target, 'GET' ); } // A redirect to another host that ends in 2xx may be a parked/for-sale domain: look at the page. if ( '' === $response->error_type && null !== $response->code && $response->code < 300 && $response->redirect_count > 0 && '' === $response->body && Classifier::host_changed( $url, $response->final_url ) && UrlSafety::is_allowed( $response->final_url ) ) { $probe = $this->request( $response->final_url, 'GET', self::PARKING_PROBE_BYTES ); if ( ! is_wp_error( $probe ) ) { $response->body = substr( (string) wp_remote_retrieve_body( $probe ), 0, self::PARKING_PROBE_BYTES ); } } $response->response_ms = (int) round( ( microtime( true ) - $started ) * 1000 ); return $response; } /** * Whether a HEAD result should be retried with GET. * * @param Response $r HEAD result. */ private function needs_get_retry( Response $r ): bool { if ( '' !== $r->error_type ) { // Connection resets often mean "HEAD not supported". DNS, timeout and SSL errors would repeat. return 'connection' === $r->error_type; } if ( null === $r->code || 0 === $r->code ) { return true; } return $r->code >= 400 && ! in_array( $r->code, self::NO_GET_RETRY, true ); } /** * Follow up to MAX_REDIRECTS redirects manually. * * @param string $url Start URL. * @param string $method HEAD or GET. */ private function follow( string $url, string $method ): Response { $r = new Response(); $r->method = $method; $current = $url; $max_bytes = 'GET' === $method ? self::GET_MAX_BYTES : 0; $visited = array(); for ( $hop = 0; $hop <= self::MAX_REDIRECTS; $hop++ ) { $r->final_url = $current; // Checked for the first URL and for every redirect target a remote server hands us. if ( ! UrlSafety::is_allowed( $current ) ) { $r->error_type = 'unsafe'; $r->error_message = __( 'Not requested: the address is on a private or reserved network', 'unbreak' ); return $r; } $raw = $this->request( $current, $method, $max_bytes ); if ( is_wp_error( $raw ) ) { $r->error_message = substr( $raw->get_error_message(), 0, 255 ); $r->error_type = self::error_type( $raw ); return $r; } $code = (int) wp_remote_retrieve_response_code( $raw ); $r->code = $code > 0 ? $code : null; $r->headers = self::headers( $raw ); if ( 'GET' === $method ) { $r->body = substr( (string) wp_remote_retrieve_body( $raw ), 0, self::GET_MAX_BYTES ); } $location = $r->header( 'location' ); if ( $code >= 300 && $code < 400 && '' !== $location ) { $next = Url::resolve( $location, $current ); if ( null === $next ) { return $r; } $r->chain[] = array( $current, $code ); $visited[] = $current; if ( in_array( $next, $visited, true ) || self::MAX_REDIRECTS === $hop ) { $r->error_type = 'too_many_redirects'; $r->error_message = in_array( $next, $visited, true ) ? __( 'Redirect loop', 'unbreak' ) : __( 'Too many redirects', 'unbreak' ); $r->redirect_count = count( $r->chain ); return $r; } $current = $next; $r->redirect_count = count( $r->chain ); continue; } return $r; } return $r; } /** * One HTTP request without automatic redirects. * * @param string $url URL. * @param string $method HEAD or GET. * @param int $max_bytes Response size cap for GET (0 = no body). * @return array|\WP_Error */ protected function request( string $url, string $method, int $max_bytes ) { $headers = array( 'Accept' => 'text/html,*/*;q=0.8', 'Accept-Language' => 'en', ); $args = array( 'method' => $method, 'timeout' => self::TIMEOUT, 'redirection' => 0, 'user-agent' => (string) Settings::get( 'user_agent' ), 'headers' => $headers, 'sslverify' => true, 'reject_unsafe_urls' => false, 'decompress' => true, ); if ( 'GET' === $method ) { $range = self::PARKING_PROBE_BYTES === $max_bytes ? $max_bytes : self::GET_RANGE_BYTES; $args['headers']['Range'] = 'bytes=0-' . ( $range - 1 ); $args['limit_response_size'] = $max_bytes > 0 ? $max_bytes : self::GET_MAX_BYTES; } /** * Filters the request arguments used for link checks. * * @param array $args wp_remote_request() arguments. * @param string $url URL being checked. */ $args = (array) apply_filters( 'unbreak_check_request_args', $args, $url ); $this->pace( Url::host( $url ) ); return wp_remote_request( $url, $args ); } /** * Keep 1 request per host every 2 seconds also within one check (HEAD, GET retry, redirect hops). * * @param string $host Host. */ private function pace( string $host ): void { /** * Minimum seconds between two requests to the same host inside one check. * * @param float $gap Seconds. */ $gap = (float) apply_filters( 'unbreak_checker_host_gap', HostThrottle::MIN_GAP ); if ( $gap > 0 && isset( $this->host_last[ $host ] ) ) { $wait = $this->host_last[ $host ] + $gap - microtime( true ); if ( $wait > 0 ) { $this->sleep( $wait ); } } $this->host_last[ $host ] = microtime( true ); } /** * Sleep (overridable in subclasses). * * @param float $seconds Seconds. */ protected function sleep( float $seconds ): void { usleep( (int) ( $seconds * 1000000 ) ); } /** * Map a WP_Error from the HTTP API to an error category. * * @param \WP_Error $error Error. */ public static function error_type( \WP_Error $error ): string { if ( 'http_request_not_executed' === $error->get_error_code() ) { return 'blocked'; } $message = strtolower( $error->get_error_message() ); $checks = array( 'dns' => array( 'curl error 6:', 'could not resolve', 'name or service not known', 'getaddrinfo', 'no address associated', 'nodename nor servname' ), 'timeout' => array( 'curl error 28:', 'timed out', 'timeout was reached' ), 'ssl' => array( 'curl error 35:', 'curl error 51:', 'curl error 53:', 'curl error 54:', 'curl error 58:', 'curl error 59:', 'curl error 60:', 'curl error 77:', 'curl error 80:', 'curl error 82:', 'curl error 83:', 'ssl', 'certificate', 'tls' ), 'too_many_redirects' => array( 'too many redirects', 'curl error 47:' ), 'invalid_url' => array( 'a valid url was not provided', 'curl error 3:' ), ); foreach ( $checks as $type => $needles ) { foreach ( $needles as $needle ) { if ( false !== strpos( $message, $needle ) ) { return $type; } } } return 'connection'; } /** * Lowercased response headers as strings. * * @param array $raw HTTP API response. * @return array */ private static function headers( array $raw ): array { $out = array(); $headers = wp_remote_retrieve_headers( $raw ); if ( $headers instanceof \WpOrg\Requests\Utility\CaseInsensitiveDictionary ) { $headers = $headers->getAll(); } foreach ( (array) $headers as $name => $value ) { $out[ strtolower( (string) $name ) ] = is_array( $value ) ? (string) end( $value ) : (string) $value; } return $out; } /** * URL safe to hand to the HTTP API: IDN hosts in punycode, non-ASCII path bytes percent-encoded. * * @param string $url Normalized URL. */ public static function request_url( string $url ): string { $host = Url::host( $url ); if ( '' !== $host && preg_match( '/[^\x20-\x7e]/', $host ) && function_exists( 'idn_to_ascii' ) ) { $ascii = defined( 'INTL_IDNA_VARIANT_UTS46' ) ? idn_to_ascii( $host, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46 ) : idn_to_ascii( $host ); if ( is_string( $ascii ) && '' !== $ascii ) { $url = (string) preg_replace( '#^(https?://(?:[^@/]*@)?)' . preg_quote( $host, '#' ) . '#i', '$1' . $ascii, $url ); } } return (string) preg_replace_callback( '/[^\x21-\x7e]/', static function ( $m ) { return rawurlencode( $m[0] ); }, $url ); } }