0 ) { $hit = get_transient( $key ); if ( false !== $hit ) { return $hit; } // The app was unreachable a moment ago: say so again rather than wait on it again. $down = get_transient( self::DOWN_KEY ); if ( false !== $down ) { return $down; } } $error = self::fetch( $path, $args, $body, $timeout ); if ( null === $error ) { if ( $ttl > 0 ) { set_transient( $key, $body, $ttl ); } return $body; } /* * Failures are cached too, for longer than a success would be on the cheap calls, so a * broken connection costs one slow page every few minutes and not every page. An app that * could not be reached at all marks every call down, not just this one. A call that is not * cached (the editor panel, opened on purpose) still asks every time. */ if ( $ttl > 0 ) { set_transient( $key, $error, self::BACKOFF ); if ( in_array( $error->get_error_code(), array( 'silotree_unreachable', 'silotree_server_error' ), true ) ) { set_transient( self::DOWN_KEY, $error, self::BACKOFF ); } } return $error; } /** * One GET, uncached. * * @param array|null $body Set to the decoded answer on success. * @return WP_Error|null Null on success. */ private static function fetch( $path, $args, &$body, $timeout = 8 ) { $url = add_query_arg( $args, self::app_url() . $path ); $response = wp_remote_get( $url, array( // Short, because this blocks an admin page render. If the app is slow, the screen // should say so rather than hang on somebody's dashboard. 'timeout' => max( 1, (int) $timeout ), 'headers' => array( 'accept' => 'application/json' ) + SiloTree_Auth::headers_for( 'GET', $url ), ) ); if ( is_wp_error( $response ) ) { return new WP_Error( 'silotree_unreachable', __( 'Could not reach Silotree. Your site is fine; this panel is not.', 'silotree' ) ); } $code = wp_remote_retrieve_response_code( $response ); $body = json_decode( wp_remote_retrieve_body( $response ), true ); if ( 401 === $code ) { /* * The app's reason travels in the error's data: 'unknown_site' when no site by this name * holds a key there any more (deleted or disconnected), which Pro reads as its plan gone. */ return new WP_Error( 'silotree_unauthorised', __( 'Silotree does not recognise this site any more. Connect it again from Silotree → Settings.', 'silotree' ), array( 'status' => 401, 'reason' => is_array( $body ) && isset( $body['code'] ) && is_string( $body['code'] ) ? sanitize_key( $body['code'] ) : '', // Whether this site asked under the domain it was paired under (false for a clone or a moved site). 'paired' => (string) get_option( SiloTree_Auth::PAIRED_DOMAIN, '' ) === self::domain(), ) ); } if ( $code < 200 || $code >= 300 || ! is_array( $body ) ) { return new WP_Error( $code >= 500 ? 'silotree_server_error' : 'silotree_bad_response', sprintf( /* translators: %d: an HTTP status code. */ __( 'Silotree answered with %d.', 'silotree' ), (int) $code ) ); } return null; } /** The WP-Cron event that fills the cache for cached(). */ const REFRESH = 'silotree_api_refresh'; /** * A GET for a screen that is being drawn: the cached answer, or none, never a wait. * * The posts list, the editor and every Silotree screen are drawn on each load; asking the app * there put up to eight seconds on a page load whenever the cache had just expired. Here a miss * queues one WP-Cron event that asks the app and fills the cache, and the page is drawn * without the answer this time (a WP_Error, silotree_pending) and with it on the next load. * * @param string $path Endpoint path, under /api/plugin/. * @param array $args Query arguments. * @param int $ttl Seconds the refreshed answer is cached for. * @return array|WP_Error */ public static function cached( $path, $args = array(), $ttl = 120 ) { if ( ! SiloTree_Auth::connected() ) { return self::not_connected(); } $hit = get_transient( self::CACHE_PREFIX . md5( $path . wp_json_encode( $args ) ) ); if ( false !== $hit ) { return $hit; } $down = get_transient( self::DOWN_KEY ); if ( false !== $down ) { return $down; } $event = array( (string) $path, (array) $args, max( 1, (int) $ttl ) ); if ( ! wp_next_scheduled( self::REFRESH, $event ) ) { wp_schedule_single_event( time(), self::REFRESH, $event ); } return new WP_Error( 'silotree_pending', __( 'Asking Silotree. This appears on the next page load.', 'silotree' ) ); } /** The WP-Cron side of cached(): ask the app once, which caches the answer or the failure. */ public static function refresh( $path, $args = array(), $ttl = 120 ) { if ( is_string( $path ) && 0 === strpos( $path, '/api/plugin/' ) && is_array( $args ) ) { self::get( $path, $args, max( 1, (int) $ttl ) ); } } /** Everything the overview screen shows. */ public static function site() { return self::get( '/api/plugin/site', array(), 300 ); } /** Everything the editor panel shows for one post. */ public static function post_panel( $post_id ) { // Not cached: the editor panel is opened deliberately, and a stale answer there is worse // than a slow one — somebody is about to change the post it describes. return self::get( '/api/plugin/post', array( 'post_id' => (int) $post_id ), 0 ); } /** * A POST against the app. * * Same headers as GET, no cache: this is a write, and a stale 200 would hide that nothing * landed. Used to map post ids onto crawled pages without asking the site to be recrawled. * * @param string $path Endpoint path. * @param array $body JSON body. * @return array|WP_Error */ public static function post( $path, $body ) { if ( ! SiloTree_Auth::connected() ) { return self::not_connected(); } $url = self::app_url() . $path; $json = wp_json_encode( $body ); $response = wp_remote_post( $url, array( 'timeout' => 20, 'headers' => array( 'content-type' => 'application/json', 'accept' => 'application/json', ) + SiloTree_Auth::headers_for( 'POST', $url, $json ), 'body' => $json, ) ); if ( is_wp_error( $response ) ) { return new WP_Error( 'silotree_unreachable', __( 'Could not reach Silotree. Your site is fine; this panel is not.', 'silotree' ) ); } $code = wp_remote_retrieve_response_code( $response ); $data = json_decode( wp_remote_retrieve_body( $response ), true ); if ( $code < 200 || $code >= 300 || ! is_array( $data ) ) { return new WP_Error( 'silotree_bad_response', sprintf( /* translators: %d: an HTTP status code. */ __( 'Silotree answered with %d.', 'silotree' ), (int) $code ) ); } return $data; } /** Posts per request to the app's map endpoint. The endpoint takes up to 5,000. */ const MAP_BATCH = 500; /** Seconds one run of map_posts may spend before handing the rest to cron. */ const MAP_BUDGET = 20; /** * Tell the app which WordPress id is which URL, and the content hash it should expect. * * A crawl that ran before this plugin was connected has the pages and not the ids, so apply * 409s. This is the cheaper fix: the permalinks are already here. * * Paged by post id, so a site of fifty thousand posts is mapped whole rather than its first * 2,000 — which is where this used to stop, silently, leaving every later post unappliable. * Each run sends batches of MAP_BATCH until MAP_BUDGET seconds have passed, then schedules * itself to carry on from the last id sent. A failed batch keeps its place for the retry. * * @param int $cursor Map posts with an id above this. 0, the default, starts over. * @return array|WP_Error Totals for this run, with 'more' true when cron will carry on. */ public static function map_posts( $cursor = 0 ) { global $wpdb; $cursor = max( 0, (int) $cursor ); $started = microtime( true ); /** * Filters how many posts go to the app in one request. * * @param int $batch Posts per request, 500 by default; the app accepts up to 5,000. */ $batch = max( 1, min( 5000, (int) apply_filters( 'silotree_map_batch', self::MAP_BATCH ) ) ); $totals = array( 'mapped' => 0, 'unknown' => 0, 'received' => 0, 'more' => false, ); // A map from the start carries the site's category and tag pages once, with its first batch. $send_terms = 0 === $cursor && class_exists( 'SiloTree_Terms' ); for ( ;; ) { list( $types_in, $types ) = SiloTree_Scan::post_types_sql(); // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter -- {$wpdb->posts} is a table identifier; $types_in is a placeholder list bound through prepare(). $ids = array_map( 'intval', (array) $wpdb->get_col( $wpdb->prepare( "SELECT ID FROM {$wpdb->posts} WHERE post_status = 'publish' AND post_type IN ($types_in) AND ID > %d ORDER BY ID ASC LIMIT %d", array_merge( $types, array( $cursor, $batch ) ) ) ) ); // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter if ( ! $ids ) { break; } // One query for the batch's posts rather than one per get_post(). _prime_post_caches( $ids, false, false ); // And their meta (target keywords) in one more, rather than several per post. SiloTree_Keywords::prime( $ids ); $posts = array(); foreach ( $ids as $id ) { $post = get_post( $id ); $row = array( 'cms_post_id' => $id, 'url' => get_permalink( $id ), 'content_hash' => $post ? SiloTree_Linker::content_hash( $post->post_content ) : '', ); /* * Target keywords, when the post has any or somebody cleared them on purpose. * keywords_at is 0 for a list imported from the SEO plugin, which the app applies * only where nobody has set one. */ $keywords = SiloTree_Keywords::for_post( $id ); if ( $keywords['keywords'] || $keywords['at'] > 0 ) { $row['keywords'] = $keywords['keywords']; $row['keywords_at'] = (int) $keywords['at']; } $posts[] = $row; } $last = (int) end( $ids ); $done = count( $ids ) < $batch; $body = array( 'posts' => $posts, 'cursor' => $last, 'done' => $done, ); if ( $send_terms ) { // Categories and tags with posts, most posts first: { term_id, taxonomy, url, name, description, count }. $body['terms'] = SiloTree_Terms::for_map(); } $result = self::post( '/api/plugin/map', $body ); if ( is_wp_error( $result ) ) { // Keep the place: the next scheduled run starts from the batch that failed. self::schedule_map( $cursor ); return $result; } foreach ( array( 'mapped', 'unknown', 'received' ) as $key ) { $totals[ $key ] += isset( $result[ $key ] ) ? (int) $result[ $key ] : 0; } $cursor = $last; $send_terms = false; // The cache only fills: drop what this batch loaded before the next one. foreach ( $ids as $id ) { clean_post_cache( $id ); } if ( $done ) { break; } if ( microtime( true ) - $started > self::MAP_BUDGET ) { self::schedule_map( $cursor ); $totals['more'] = true; return $totals; } } update_option( 'silotree_last_map', time(), false ); delete_option( 'silotree_map_pending' ); return $totals; } /** Carry on mapping from $cursor in a moment, unless that is already queued. */ private static function schedule_map( $cursor ) { $args = array( (int) $cursor ); // So the screen does not start a second map from the top while this one is part-way. update_option( 'silotree_map_pending', (int) $cursor, false ); if ( ! wp_next_scheduled( 'silotree_map_posts', $args ) ) { wp_schedule_single_event( time() + 30, 'silotree_map_posts', $args ); } } /** The WP-Cron hook that sends a notice again when the first try did not get through. */ const NOTIFY_RETRY = 'silotree_notify_retry'; /** * Tell the app that something happened. Signed like every other call; sent only once the site * is connected. * * It waits for the answer, for three seconds at most. It used to be fire-and-forget * (`blocking => false`), and that loses notices: the request is handed to the transport and the * PHP request ends, closing the connection before the app has read it. Seen in the 0.4.1 * end-to-end run as an "aborted" request on the app's side and a new post that was never * woven. One failed try is sent again through WP-Cron a minute later; a second failure is left * to the next full scan, which finds the post anyway. * * A caller that keeps its own retry (the daily click send, say) passes $retried = true, so a * large body is never stored in a WP-Cron event's arguments. * * @param string $path The app's endpoint. * @param array $body What to send. * @param bool $retried True when this is the retry, or when the caller retries on its own. * @return int The app's HTTP status, or 0 when nothing was sent or it did not answer. */ public static function notify( $path, $body, $retried = false ) { if ( ! SiloTree_Auth::connected() ) { return 0; } $url = self::app_url() . $path; $json = wp_json_encode( $body ); $response = wp_remote_post( $url, array( 'timeout' => 3, 'headers' => array( 'content-type' => 'application/json' ) + SiloTree_Auth::headers_for( 'POST', $url, $json ), 'body' => $json, ) ); $code = is_wp_error( $response ) ? 0 : (int) wp_remote_retrieve_response_code( $response ); // A refusal (4xx) will be refused again; a network error or a 5xx is worth one more try. if ( ! $retried && ( 0 === $code || $code >= 500 ) ) { wp_schedule_single_event( time() + 60, self::NOTIFY_RETRY, array( $path, $body ) ); } return $code; } /** The WP-Cron side of a retried notice. */ public static function notify_retry( $path, $body ) { if ( is_string( $path ) && 0 === strpos( $path, '/api/plugin/' ) && is_array( $body ) ) { self::notify( $path, $body, true ); } } /** * Ask the app whether it recognises this site, and say precisely why not when it does not. * * Asked for by a person on the settings screen, so it runs whether or not the site is marked * connected — that is the question it answers. * * @return true|WP_Error */ public static function check() { $url = self::app_url() . '/api/plugin/site'; $response = wp_remote_get( $url, array( 'timeout' => 10, 'headers' => array( 'accept' => 'application/json' ) + SiloTree_Auth::headers_for( 'GET', $url ), ) ); if ( is_wp_error( $response ) ) { return new WP_Error( 'silotree_unreachable', sprintf( /* translators: %s: the network error. */ __( 'This site could not reach Silotree (%s). Your host may block outgoing requests.', 'silotree' ), $response->get_error_message() ) ); } $code = (int) wp_remote_retrieve_response_code( $response ); if ( 401 === $code ) { return new WP_Error( 'silotree_unauthorised', __( 'Silotree does not have this site\'s current key. Connect again.', 'silotree' ) ); } if ( $code < 200 || $code >= 300 ) { return new WP_Error( 'silotree_bad_response', sprintf( /* translators: %d: an HTTP status code. */ __( 'Silotree answered with %d.', 'silotree' ), $code ) ); } return true; } private static function not_connected() { return new WP_Error( 'silotree_not_connected', __( 'This site is not connected to Silotree yet.', 'silotree' ) ); } /** Drop everything cached, after a change that would make it wrong. */ public static function forget() { global $wpdb; // By name as well, for sites whose transients live in an object cache and not the table. delete_transient( self::DOWN_KEY ); $wpdb->query( $wpdb->prepare( "DELETE FROM {$wpdb->options} WHERE option_name LIKE %s", $wpdb->esc_like( '_transient_' . self::CACHE_PREFIX ) . '%' ) ); } }