array( until (a Unix time), reason, label ). */ const LANGUAGES_SET_ASIDE = 'langferry_priority_set_aside'; /** * next()'s reads within one request (ruling R42d): the main pages, tier 1's rows and tier 2's, * until the pages table changes (forget_memo()), and the table's last change they were read at. * * @var array */ private static $memo = array(); private static $memo_stamp = null; /** The report's limits, as the cloud keeps it (clean_priority in cloud/app/sites.py). */ const LABEL_MAX = 80; const COUNT_MAX = 1000000; const SET_ASIDE_MAX = 10; const REASONS = array( 'cache', 'unreachable', 'error', 'refused' ); /** * The main pages' keys, in order: "/" first (with a static front page and with latest posts), then * the menus' published, public pages in menu order, once each, through the filter * langferry_priority_pages. * * @return string[] */ public static function main_paths(): array { return array_keys( self::main_pages() ); } /** * The pair to translate next at $now: the first, in tier order, that isn't complete and isn't set * aside, in a language that isn't set aside. Null when there is none, when the site has no * languages, or when this call only moved a tier 3 cursor past a window with nothing waiting: * has_more() then says to call again. * * With a skip list (only `wp langferry translate` passes one: the pages it couldn't finish in its * run), those pairs are passed over. A tier 3 window whose only pairs waiting are skipped keeps its * cursor, so they are still there for the next run, and the call reads on, as many windows as it * takes: null then means nothing is left for this run. The worker never passes one, not even for a * pair a sign-in redirect set aside (that pair isn't waiting any more), so each of its calls reads * one window at most and its run's time check stays between them (rulings R28, R55). The languages * in $skip_langs (those the cloud paused, which the worker passes over for the rest of its run, * ruling R63) are left out after the languages set aside, and their cursors stay where they are. * * @param int $now A Unix time. * @param array $skip Pairs to pass over, keyed "lang path" ("es /about/"). * @param string $lang Only this language ('' for all). * @param string[] $skip_langs Languages to leave out, as codes. * @return array{lang: string, path: string, tier: int, label: string}|null path is a page key. */ public static function next( int $now, array $skip = array(), string $lang = '', array $skip_langs = array() ): ?array { $langs = self::without( self::active_languages( $now, $lang ), $skip_langs ); if ( ! $langs ) { return null; } $main = self::remembered( 'main', static function () { return self::main_pages(); } ); return self::find( $langs, $main, gmdate( 'Y-m-d H:i:s', $now ), true, $skip ); } /** * Forget next()'s reads (ruling R42d). Each write to the pages table in this request does it * (Pages clears its cache group's last change); the worker does it after each request too, as the * render it asked for wrote the table in a request of its own. */ public static function forget_memo(): void { self::$memo = array(); self::$memo_stamp = null; } /** * Whether tier 3 still has windows to read: the cursor of some language that isn't set aside isn't * past the sitemap's end (or the content changed, so every cursor starts again). The queue keeps * running while next() or this has something. * * @param int|null $now A Unix time; now when null. * @param string $lang Only this language ('' for all). * @param string[] $skip_langs Languages to leave out, as codes, as next() does. */ public static function has_more( ?int $now = null, string $lang = '', array $skip_langs = array() ): bool { $langs = self::without( self::active_languages( null === $now ? time() : $now, $lang ), $skip_langs ); return $langs && null !== self::current_language( $langs, self::cursors()['at'] ); } /** * Set a whole language aside until $until (a Unix time): the worker's requests to its own host got * no answer. The order skips it meanwhile; the report lists it once, with $reason and $label (the * pair it was at). Languages whose time is up, or that the site no longer has, are dropped. */ public static function set_language_aside( string $lang, int $until, string $reason, string $label ): void { $aside = self::languages_aside( time() ); $aside[ $lang ] = array( 'until' => $until, 'reason' => in_array( $reason, self::REASONS, true ) ? $reason : 'error', 'label' => self::label( $label, '' ), ); update_option( self::LANGUAGES_SET_ASIDE, array_intersect_key( $aside, array_flip( self::languages() ) ), false ); } /** * How many of the sitemap's entries a tier 3 window holds: langferry_priority_scan_limit, 2,000 by * default, kept between 100 and 10,000. */ public static function scan_limit(): int { /** * Filters how many of the sitemap's entries (its posts newest first, then its category and tag * pages) one call of the queue reads, from where the last one stopped. Kept between 100 and 10,000. * * @param int $limit Default 2,000. */ return max( self::SCAN_MIN, min( self::SCAN_MAX, (int) apply_filters( 'langferry_priority_scan_limit', self::SCAN_LIMIT ) ) ); } /** * Start tier 3 again from the sitemap's beginning, in one language or (null) in all: their rows are * gone ("Delete translations", Pages::forget()), so the windows they passed have work again. */ public static function restart( ?string $lang = null ): void { $saved = get_option( self::CURSOR ); if ( null === $lang || '' === $lang || ! is_array( $saved ) || ! isset( $saved['at'] ) || ! is_array( $saved['at'] ) ) { delete_option( self::CURSOR ); return; } if ( isset( $saved['at'][ $lang ] ) ) { unset( $saved['at'][ $lang ] ); update_option( self::CURSOR, $saved, false ); } } /** * The progress the hourly report sends as `priority` (all but `worker`, which the queue adds): * - main: each language => array( main pages complete, main pages ); * - next: the pair at the head of the queue, as array( lang, label, tier ), or null; * - then: the first language after next's whose main pages aren't all done (null when there is * none, or next isn't a main page); * - queued: pairs waiting per tier, "1", "2" and "3"; tier 3's is an estimate; * - set_aside: at most 10 pairs set aside, as array( lang, label, reason ); * - set_aside_total: how many are set aside in all (the languages set aside and the pairs), never * below the entries listed. * Counts stay between 0 and 1,000,000 and labels within 80 characters. main is an object (not an * empty list) when encoded as JSON, also when the site has no languages. */ public static function progress(): array { $at = gmdate( 'Y-m-d H:i:s' ); $all = self::languages(); $aside = array_intersect_key( self::languages_aside( time() ), array_flip( $all ) ); $langs = array_values( array_diff( $all, array_keys( $aside ) ) ); // The languages the order works on. $main = $all ? self::main_pages() : array(); $states = Pages::states( array_keys( $main ), $all ); $done = array(); $tier_1 = 0; foreach ( $all as $lang ) { $done[ $lang ] = 0; foreach ( array_keys( $main ) as $key ) { $row = $states[ $key ][ $lang ] ?? null; if ( null !== $row && 0 === $row['pending'] ) { ++$done[ $lang ]; } elseif ( ! isset( $aside[ $lang ] ) && self::waiting( $row, $at ) ) { ++$tier_1; } } } $next = $langs ? self::find( $langs, $main, $at, false ) : null; $then = null; if ( null !== $next && 1 === $next['tier'] ) { foreach ( array_slice( $langs, (int) array_search( $next['lang'], $langs, true ) + 1 ) as $lang ) { if ( $done[ $lang ] < count( $main ) ) { $then = $lang; break; } } } $tier_2 = iterator_count( self::asked( $langs, $at, $main, 1000 ) ); $counts = array(); foreach ( $done as $lang => $complete ) { $counts[ $lang ] = array( self::count( $complete ), self::count( count( $main ) ) ); } $set_aside = self::set_aside( $langs, $at, $main, $aside ); return array( 'main' => $counts ? $counts : new \stdClass(), 'next' => null === $next ? null : array( 'lang' => $next['lang'], 'label' => $next['label'], 'tier' => $next['tier'], ), 'then' => $then, 'queued' => array( '1' => self::count( $tier_1 ), '2' => self::count( $tier_2 ), '3' => self::count( self::estimate( $langs, $tier_1 + $tier_2 ) ), ), 'set_aside' => $set_aside, 'set_aside_total' => self::count( max( count( $set_aside ), count( $aside ) + Pages::set_aside_count( $langs, $at ) ) ), ); } /** * The first pair in tier order that is waiting at $at. * * @param string[] $langs * @param array $main main_pages(). * @param bool $move Whether tier 3 may move a cursor past a window with nothing * waiting (next() does; the report only looks). * @param array $skip Pairs to pass over, keyed "lang path". */ private static function find( array $langs, array $main, string $at, bool $move, array $skip = array() ): ?array { $read = static function () use ( $main, $langs ) { return Pages::states( array_keys( $main ), $langs ); }; $states = $move ? self::remembered( 'tier1:' . md5( (string) wp_json_encode( array( array_keys( $main ), $langs ) ) ), $read ) : $read(); foreach ( $langs as $lang ) { foreach ( $main as $key => $label ) { if ( self::waiting( $states[ $key ][ $lang ] ?? null, $at ) && ! isset( $skip[ "$lang $key" ] ) ) { return self::pair( $lang, $key, 1, $label ); } } } foreach ( self::asked( $langs, $at, $main, self::BATCH, $move ) as $pair ) { if ( ! isset( $skip[ "{$pair['lang']} {$pair['path']}" ] ) ) { return self::pair( $pair['lang'], $pair['path'], 2, self::label_for( $pair['path'], $main ) ); } } return self::sitemap_next( $langs, $main, $at, $move, $skip ); } /** * Tier 3, in its current language: the first, in the site's order, whose cursor isn't past the * sitemap's end. One window is read from that cursor, in batches, each checked in one query, and * the first pair waiting in it is next (a set-aside pair comes back in its place when its time is * up). Each batch with nothing waiting moves the cursor past it ($move), so a run that stops * halfway resumes at the next batch (ruling R42b); a window with nothing waiting gives null, with * has_more() true. Past the end, the language is done until the content changes. * * With a skip list ($move only), a window whose only pairs waiting are skipped keeps its cursor * from the first of them on and leaves its language for this call, and the call reads on: the next * window, or the next language, until a pair or the end. It ends: each window either gives a pair, * moves a cursor towards the sitemap's end, or leaves a language. * * @param string[] $langs * @param array $main * @param array $skip Pairs to pass over, keyed "lang path". */ private static function sitemap_next( array $langs, array $main, string $at, bool $move, array $skip = array() ): ?array { $cursors = self::cursors(); $held = array(); // Languages whose window holds only skipped pairs. $save = static function () use ( &$cursors ) { self::save_cursors( $cursors['seen'], $cursors['at'], self::languages() ); $cursors['changed'] = false; }; do { $lang = self::current_language( array_values( array_diff( $langs, $held ) ), $cursors['at'] ); if ( null === $lang ) { return null; } $window = self::window( $main, $cursors['at'][ $lang ] ?? array( 'posts', 0 ) ); $skipped = false; foreach ( $window as list( $batch, $after ) ) { $states = $batch ? Pages::states( array_keys( $batch ), array( $lang ) ) : array(); foreach ( $batch as $key => $label ) { if ( ! self::waiting( $states[ $key ][ $lang ] ?? null, $at ) ) { continue; } if ( isset( $skip[ "$lang $key" ] ) ) { $skipped = true; continue; } if ( $move && $cursors['changed'] ) { $save(); } return self::pair( $lang, $key, 3, $label ); } if ( $move && ! $skipped ) { $cursors['at'][ $lang ] = $after; // Nothing waiting in this batch: the cursor moves past it. $save(); } } if ( $skipped ) { $held[] = $lang; } elseif ( $move ) { $cursors['at'][ $lang ] = $window->getReturn(); $save(); } } while ( $move && $skip ); return null; } /** * The first language, in the site's order, whose cursor isn't past the sitemap's end. * * @param string[] $langs * @param array $at */ private static function current_language( array $langs, array $at ): ?string { foreach ( $langs as $lang ) { if ( 'end' !== ( $at[ $lang ][0] ?? 'posts' ) ) { return $lang; } } return null; } /** * One window of the sitemap from $from: scan_limit() entries at most, its posts newest first, then * its category and tag pages, newest first, in batches of up to 100. Each batch is yielded as * array( key => label, where the window goes on after it ), the batch empty when none of its * entries may be queued. Main pages (tier 1), excluded paths and addresses off the site are left * out, but counted. Its return value is where the next window starts: array( 'posts' or 'terms', * offset ), or array( 'end', 0 ) when this one reached the sitemap's end. * * A cursor array( 'new', offset, since, kind, offset ) first reads the posts published after * `since` (newest first, from offset), then goes on at the cursor it was before they were * published, moved on by their number (ruling R42c, cursors()). * * Category and tag pages are those with published posts: one an SEO plugin lists while it is * empty isn't queued. * * @param array $main * @param array $from A cursor (cursors()). * @return \Generator, 1: array}, mixed, array> */ private static function window( array $main, array $from ): \Generator { $limit = self::scan_limit(); $read = 0; $seen = $main; $kind = (string) $from[0]; $offset = (int) $from[1]; $batch = static function ( array $posts ) use ( &$seen ) { $out = array(); foreach ( $posts as $post ) { $key = self::own_key( (string) get_permalink( $post ) ); if ( null !== $key && ! isset( $seen[ $key ] ) ) { $seen[ $key ] = true; $out[ $key ] = self::label( $post->post_title, rawurldecode( $key ) ); } } return $out; }; if ( 'new' === $kind ) { $since = (string) $from[2]; $resume = array( (string) $from[3], (int) $from[4] ); $types = Seo::sitemap_post_types(); $args = $types ? self::post_args( $types, $since ) : array(); $done = ! $types; while ( ! $done && $read < $limit ) { $size = min( self::BATCH, $limit - $read ); list( $posts, $added ) = self::posts( $args, $offset, $size ); $read += count( $posts ); $offset += count( $posts ); $done = count( $posts ) < $size; $entries = $batch( $posts ); self::release( $added ); yield array( $entries, $done ? self::resumed( $resume, $offset ) : array( 'new', $offset, $since, $resume[0], $resume[1] ) ); } if ( ! $done ) { return array( 'new', $offset, $since, $resume[0], $resume[1] ); } list( $kind, $offset ) = self::resumed( $resume, $offset ); } if ( 'posts' === $kind ) { $types = Seo::sitemap_post_types(); $args = $types ? self::post_args( $types ) : array(); $done = ! $types; while ( ! $done && $read < $limit ) { $size = min( self::BATCH, $limit - $read ); list( $posts, $added ) = self::posts( $args, $offset, $size ); $read += count( $posts ); $offset += count( $posts ); $done = count( $posts ) < $size; $entries = $batch( $posts ); self::release( $added ); // Their keys are made: the window's posts aren't kept. yield array( $entries, $done ? array( 'terms', 0 ) : array( 'posts', $offset ) ); } if ( ! $done ) { return array( 'posts', $offset ); } $kind = 'terms'; $offset = 0; } if ( 'terms' === $kind ) { $taxonomies = Seo::sitemap_taxonomies(); $args = $taxonomies ? self::term_args( $taxonomies ) : array(); $done = ! $taxonomies; while ( ! $done && $read < $limit ) { $size = min( self::BATCH, $limit - $read ); $terms = get_terms( $args + array( 'orderby' => 'term_id', 'order' => 'DESC', 'number' => $size, 'offset' => $offset, ) ); $terms = is_array( $terms ) ? $terms : array(); $read += count( $terms ); $offset += count( $terms ); $done = count( $terms ) < $size; $entries = array(); foreach ( $terms as $term ) { $link = $term instanceof \WP_Term ? get_term_link( $term ) : ''; $key = is_string( $link ) ? self::own_key( $link ) : null; if ( null !== $key && ! isset( $seen[ $key ] ) ) { $seen[ $key ] = true; $entries[ $key ] = self::label( $term->name, rawurldecode( $key ) ); } } yield array( $entries, $done ? array( 'end', 0 ) : array( 'terms', $offset ) ); } if ( ! $done ) { return array( 'terms', $offset ); } } return array( 'end', 0 ); } /** * Where a cursor goes on once the posts newer than its `since` are read: the cursor it was before * they were published, moved on by their number ($count) while it is among the posts. Past the * posts (the category and tag pages, or the end) it stays where it was. * * @param array{0: string, 1: int} $resume * @return array{0: string, 1: int} */ private static function resumed( array $resume, int $count ): array { return 'posts' === $resume[0] ? array( 'posts', $resume[1] + $count ) : $resume; } /** * $size of the sitemap's posts from $offset (post_args()), read whole in one query: cache_results * is off and the query isn't split into IDs and cached posts. Their keys are made by get_permalink(), * which may read a post by its ID (get_the_category() under /%category%/, or a filter's * get_the_terms()), so the posts go into the object cache meanwhile, with their meta and terms (one * query each). window() takes out again what this added (release()) before it goes on. * * @return array{0: \WP_Post[], 1: array} The posts, and the IDs this added to the * object cache, by cache group: posts, post_meta and each {taxonomy}_relationships. */ private static function posts( array $args, int $offset, int $size ): array { $whole = static function ( $split, $query ) { return $query instanceof \WP_Query && $query->get( 'langferry_priority' ) ? false : $split; }; add_filter( 'split_the_query', $whole, 10, 2 ); $query = new \WP_Query( array_merge( $args, array( 'posts_per_page' => $size, 'offset' => $offset, ) ) ); remove_filter( 'split_the_query', $whole, 10 ); $posts = $query->posts; $ids = array_map( 'intval', wp_list_pluck( $posts, 'ID' ) ); $added = array(); if ( $ids ) { $groups = array( 'posts', 'post_meta' ); foreach ( get_object_taxonomies( $args['post_type'] ) as $taxonomy ) { $groups[] = $taxonomy . '_relationships'; } foreach ( $groups as $group ) { foreach ( wp_cache_get_multiple( $ids, $group ) as $id => $cached ) { if ( false === $cached ) { $added[ $group ][] = (int) $id; } } } update_post_cache( $posts ); update_meta_cache( 'post', $ids ); update_object_term_cache( $ids, $args['post_type'] ); } return array( $posts, $added ); } /** * Take out of the object cache what posts() added for a batch: the posts, their meta and their * term relationships (ruling R42a), so a window's thousands of posts never pile up in memory, nor * in a persistent object cache. What another read had cached before stays. * * @param array $added posts()'s IDs by cache group. */ private static function release( array $added ): void { foreach ( $added as $group => $ids ) { wp_cache_delete_multiple( $ids, $group ); } } /** * The query for tier 3's posts: the sitemap's (Seo::sitemap_query_args()), newest first, less the * shop's private pages; with $since, only those published after it (post_date_gmt, UTC). * * @param string[] $types Seo::sitemap_post_types(), not empty. */ private static function post_args( array $types, string $since = '' ): array { $args = Seo::sitemap_query_args( $types ); $args = array_merge( $args, array( 'post__not_in' => array_merge( (array) $args['post__not_in'], self::shop_pages() ), // phpcs:ignore WordPressVIPMinimum.Performance.WPQueryParams.PostNotIn_post__not_in -- What the SEO plugin leaves out of its sitemap, and the shop's private pages; filtering afterwards would break the offsets. 'fields' => '', 'orderby' => array( 'date' => 'DESC', 'ID' => 'DESC', ), 'no_found_rows' => true, 'cache_results' => false, 'update_post_meta_cache' => false, 'update_post_term_cache' => false, 'langferry_priority' => true, ) ); if ( '' !== $since ) { $args['date_query'] = array( array( 'column' => 'post_date_gmt', 'after' => $since, 'inclusive' => false, ), ); } return $args; } /** * WooCommerce's cart, checkout and account pages. They show a visitor's own data, so Langferry * never records them (Privacy::is_private_request()): queued, they could never be complete. * * @return int[] */ private static function shop_pages(): array { $ids = array(); if ( function_exists( 'wc_get_page_id' ) ) { foreach ( array( 'cart', 'checkout', 'myaccount' ) as $page ) { $id = (int) wc_get_page_id( $page ); if ( $id > 0 ) { $ids[] = $id; } } } return $ids; } /** * Tier 3's cursors, for the content as it is now (content_version()). * - Kept for it: as they were. * - Kept for the content before newer posts were published, and nothing else changed (ruling * R42c: the count of posts grew, the latest post_date_gmt grew, and the posts published after * the last one seen are exactly the new ones): each cursor first reads those posts, then goes on * where it was, moved on by their number (window()). A cursor at the very start reads them * anyway; one already reading newer posts reads them again from its own `since`. * - Otherwise none: every language starts again from the sitemap's beginning. * changed is true when they aren't the ones saved, so the first move saves them. * * @return array{seen: array, at: array, changed: bool} */ private static function cursors(): array { $seen = self::content_version(); $saved = get_option( self::CURSOR ); $valid = is_array( $saved ) && isset( $saved['version'], $saved['at'] ) && is_array( $saved['at'] ); $kept = $valid && $seen['version'] === $saved['version']; $newer = $valid && ! $kept && self::only_newer_posts( is_array( $saved['seen'] ?? null ) ? $saved['seen'] : array(), $seen ); $at = array(); foreach ( $kept || $newer ? $saved['at'] : array() as $lang => $spot ) { $spot = is_string( $lang ) ? self::spot( $spot ) : null; if ( null === $spot ) { continue; } if ( $newer && 'new' === $spot[0] ) { $spot = array( 'new', 0, $spot[2], $spot[3], $spot[4] ); } elseif ( $newer && ! ( 'posts' === $spot[0] && 0 === $spot[1] ) ) { $spot = array( 'new', 0, (string) $saved['seen']['published'], $spot[0], $spot[1] ); } $at[ $lang ] = $spot; } return array( 'seen' => $seen, 'at' => $at, 'changed' => ! $kept, ); } /** * A saved cursor, checked: array( 'posts' or 'terms', offset ), array( 'end', 0 ), or array( 'new', * offset, since, 'posts' / 'terms' / 'end', offset ); null for anything else. * * @param mixed $spot */ private static function spot( $spot ): ?array { if ( ! is_array( $spot ) ) { return null; } $kind = $spot[0] ?? null; if ( in_array( $kind, array( 'posts', 'terms', 'end' ), true ) ) { return array( $kind, max( 0, (int) ( $spot[1] ?? 0 ) ) ); } if ( 'new' === $kind && is_string( $spot[2] ?? null ) && preg_match( '/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/', $spot[2] ) && in_array( $spot[3] ?? null, array( 'posts', 'terms', 'end' ), true ) ) { return array( 'new', max( 0, (int) ( $spot[1] ?? 0 ) ), $spot[2], $spot[3], max( 0, (int) ( $spot[4] ?? 0 ) ) ); } return null; } /** * Keep the cursors of the site's languages, for this content (an option not autoloaded: only the * queue reads it). A language set aside keeps its cursor for when it comes back. * * @param array $seen content_version(). * @param array $at * @param string[] $langs */ private static function save_cursors( array $seen, array $at, array $langs ): void { update_option( self::CURSOR, array( 'version' => $seen['version'], 'seen' => $seen, 'at' => array_intersect_key( $at, array_flip( $langs ) ), ), false ); } /** * What tier 3's cursors were kept for, in one query: the sitemap's post types and taxonomies, how * many published posts it lists and the latest one's publication date, and how many of its * categories and tags have posts. version sums them up; the parts are kept with the cursors, so a * publish of newer posts can be told from any other change (cursors()). Editing a post changes * none of them: WooCommerce changes post_modified with every order, which would start the cursors * again all day. An edited page behind a cursor comes back when a person opens it (tier 2). * * @return array{version: string, scope: string, posts: int, published: string, terms: int} */ private static function content_version(): array { global $wpdb; $types = Seo::sitemap_post_types(); $taxes = Seo::sitemap_taxonomies(); $in_types = $types ? $types : array( '' ); $in_taxes = $taxes ? $taxes : array( '' ); $row = $wpdb->get_row( $wpdb->prepare( 'SELECT COUNT(*) AS posts, MAX(post_date_gmt) AS published, ( SELECT COUNT(*) FROM %i WHERE count > 0 AND taxonomy IN (' . implode( ',', array_fill( 0, count( $in_taxes ), '%s' ) ) . ') ) AS terms FROM %i WHERE post_status = %s AND post_password = %s AND post_type IN (' . implode( ',', array_fill( 0, count( $in_types ), '%s' ) ) . ')', array_merge( array( $wpdb->term_taxonomy ), $in_taxes, array( $wpdb->posts, 'publish', '' ), $in_types ) ), ARRAY_A ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery -- one query for three counts that WordPress has no single API for; read fresh, as it decides whether the cursors still hold. $row = is_array( $row ) ? $row : array(); return array( 'version' => md5( (string) wp_json_encode( array( $types, $taxes, $row ) ) ), 'scope' => md5( (string) wp_json_encode( array( $types, $taxes ) ) ), 'posts' => (int) ( $row['posts'] ?? 0 ), 'published' => (string) ( $row['published'] ?? '' ), 'terms' => (int) ( $row['terms'] ?? 0 ), ); } /** * Whether the content changed only because newer posts were published since $before (ruling * R42c): the same post types and taxonomies, the same categories and tags with posts, more posts, * a later latest one, and as many posts published after $before's latest as there are new posts * (one more query): not a backdated post, nor a post unpublished as another was published. * * @param array $before The content_version() saved with the cursors. * @param array $now content_version() now. */ private static function only_newer_posts( array $before, array $now ): bool { if ( ! isset( $before['scope'], $before['posts'], $before['published'], $before['terms'] ) || ! is_string( $before['published'] ) || ! preg_match( '/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/', $before['published'] ) ) { return false; } if ( $before['scope'] !== $now['scope'] || (int) $before['terms'] !== $now['terms'] || $now['posts'] <= (int) $before['posts'] || strcmp( $now['published'], $before['published'] ) <= 0 ) { return false; } global $wpdb; $types = Seo::sitemap_post_types(); $in_types = $types ? $types : array( '' ); $newer = (int) $wpdb->get_var( $wpdb->prepare( 'SELECT COUNT(*) FROM %i WHERE post_status = %s AND post_password = %s AND post_date_gmt > %s AND post_type IN (' . implode( ',', array_fill( 0, count( $in_types ), '%s' ) ) . ')', array_merge( array( $wpdb->posts, 'publish', '', $before['published'] ), $in_types ) ) ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery -- read fresh, with the content version above. return $newer === $now['posts'] - (int) $before['posts']; } /** * Tier 2's pairs, earliest asked first, read $size at a time: those in the site's languages, not * set aside, not main pages (tier 1) and not under an excluded path. * * @param string[] $langs * @param array $main * @param bool $memo Remember the reads, for next() (ruling R42d): each for its * time, so a pair coming back at its second is seen. * @return \Generator */ private static function asked( array $langs, string $at, array $main, int $size, bool $memo = false ): \Generator { for ( $offset = 0; $langs; $offset += $size ) { $read = static function () use ( $langs, $at, $offset, $size ) { return Pages::asked( $langs, $at, $offset, $size ); }; $rows = $memo ? self::remembered( 'tier2:' . md5( (string) wp_json_encode( array( $langs, $at, $offset, $size ) ) ), $read ) : $read(); foreach ( $rows as $row ) { if ( ! isset( $main[ $row['path'] ] ) && self::allowed( $row['path'] ) ) { yield $row; } } if ( count( $rows ) < $size ) { break; } } } /** * At most 10 entries set aside: each language set aside once, with the label of the pair it was at, * then the pairs set aside in the other languages, not under an excluded path; those coming back * first first. Each has a reason the report knows ("error" for any other). * * @param string[] $langs The languages not set aside. * @param array $main * @param array $aside languages_aside(). * @return array */ private static function set_aside( array $langs, string $at, array $main, array $aside ): array { $out = array(); uasort( $aside, static function ( $a, $b ) { return $a['until'] <=> $b['until']; } ); foreach ( array_slice( $aside, 0, self::SET_ASIDE_MAX, true ) as $lang => $entry ) { $out[] = array( 'lang' => (string) $lang, 'label' => $entry['label'], 'reason' => $entry['reason'], ); } for ( $offset = 0; $langs && count( $out ) < self::SET_ASIDE_MAX; $offset += self::BATCH ) { $rows = Pages::set_aside( $langs, $at, $offset, self::BATCH ); foreach ( $rows as $row ) { if ( count( $out ) < self::SET_ASIDE_MAX && self::allowed( $row['path'] ) ) { $out[] = array( 'lang' => $row['lang'], 'label' => self::label_for( $row['path'], $main ), 'reason' => in_array( $row['reason'], self::REASONS, true ) ? $row['reason'] : 'error', ); } } if ( count( $rows ) < self::BATCH ) { break; } } return $out; } /** * Tier 3's length, estimated (nothing shows it yet): the sitemap's entries and the home page, in * every language, less the pairs complete and those in tiers 1 and 2. * * @param string[] $langs * @param int $queued Pairs waiting in tiers 1 and 2. */ private static function estimate( array $langs, int $queued ): int { if ( ! $langs ) { return 0; } $entries = 1; // The home page. $types = Seo::sitemap_post_types(); if ( $types ) { $query = new \WP_Query( array_merge( self::post_args( $types ), array( 'fields' => 'ids', 'posts_per_page' => 1, 'no_found_rows' => false, ) ) ); $entries += (int) $query->found_posts; } $taxonomies = Seo::sitemap_taxonomies(); if ( $taxonomies ) { $terms = wp_count_terms( self::term_args( $taxonomies ) ); $entries += is_numeric( $terms ) ? (int) $terms : 0; } return max( 0, $entries * count( $langs ) - Pages::complete_count( $langs ) - $queued ); } /** * The main pages, key => label: "/" first, then the posts of Site_Size::main_ids() that visitors * can open, in menu order, once each by key (the static front page keys to "/"), as the filter * langferry_priority_pages leaves them. * * @return array */ private static function main_pages(): array { $links = Site_Size::menu_links(); $labels = array(); // Post ID => its first menu label. foreach ( $links as $link ) { $id = is_array( $link ) ? (int) ( $link['id'] ?? 0 ) : 0; $label = $id > 0 ? self::label( $link['label'] ?? '', '' ) : ''; if ( '' !== $label && ! isset( $labels[ $id ] ) ) { $labels[ $id ] = $label; } } $front = 'page' === get_option( 'show_on_front' ) ? (int) get_option( 'page_on_front' ) : 0; $pages = array( '/' => $labels[ $front ] ?? __( 'Home', 'langferry' ) ); $shop = self::shop_pages(); foreach ( Site_Size::main_ids( $links ) as $id ) { $post = get_post( $id ); if ( ! self::reachable( $post, $shop ) ) { continue; } $key = self::own_key( (string) get_permalink( $post ) ); if ( null !== $key && ! isset( $pages[ $key ] ) ) { $pages[ $key ] = $labels[ $id ] ?? self::label( $post->post_title, rawurldecode( $key ) ); } } /** * Filters the main pages, which the queue translates first: in each language, before anything * else. Add, drop or reorder them. * * @param string[] $keys The pages' keys: paths relative to the home page, such as "/about/" * (Langferry\Pages::key()). The home page, "/", comes first, then the * menus' pages in menu order. Anything but a path of this site is dropped. */ $keys = apply_filters( 'langferry_priority_pages', array_keys( $pages ) ); $out = array(); foreach ( is_array( $keys ) ? $keys : array() as $path ) { $key = self::path_key( $path ); if ( null !== $key && ! isset( $out[ $key ] ) ) { $out[ $key ] = $pages[ $key ] ?? self::label_for( $key, array() ); } } return $out; } /** * Whether visitors can open this post and Langferry can record it: published, without a password, * of a public post type, and not one of the shop's private pages (shop_pages()). * * @param mixed $post * @param int[] $shop shop_pages(). */ private static function reachable( $post, array $shop ): bool { if ( ! $post instanceof \WP_Post || 'publish' !== $post->post_status || '' !== (string) $post->post_password || in_array( (int) $post->ID, $shop, true ) ) { return false; } $type = get_post_type_object( $post->post_type ); return $type instanceof \WP_Post_Type && $type->public && is_post_type_viewable( $type ); } /** * The key of an address of this site that may be queued, or null: another host, a path outside * WordPress's home path, a resource or an excluded path gives null. Pages::key() ignores the host, * so an address is checked here before it is keyed. */ private static function own_key( string $url ): ?string { $parts = wp_parse_url( $url ); if ( ! is_array( $parts ) || ! isset( $parts['host'] ) ) { return null; } $router = Router::instance(); $host = strtolower( $parts['host'] . ( isset( $parts['port'] ) ? ':' . $parts['port'] : '' ) ); $path = isset( $parts['path'] ) ? (string) $parts['path'] : '/'; $home = $router->home_path(); if ( $host !== $router->home_host() || ( '' !== $home && $path !== $home && 0 !== strpos( $path, $home . '/' ) ) ) { return null; } $key = Pages::key( $url ); return self::allowed( $key ) ? $key : null; } /** * The key of a path the filter returned, or null: only a path of this site ("/about/", no host, * not "//host/…") is kept. * * @param mixed $path */ private static function path_key( $path ): ?string { if ( ! is_string( $path ) || '' === $path || '/' !== $path[0] || ( isset( $path[1] ) && ( '/' === $path[1] || '\\' === $path[1] ) ) ) { return null; } $key = Pages::key( $path ); return self::allowed( $key ) ? $key : null; } /** * Whether a key may be queued: one the table can hold, not a resource (files, feeds, the admin) * and not under an excluded path, which Langferry never translates. */ private static function allowed( string $key ): bool { if ( '' === $key || '/' !== $key[0] || mb_strlen( $key ) > Pages::MAX_PATH ) { return false; } $path = explode( '?', $key, 2 )[0]; $router = Router::instance(); return ! $router->is_resource_path( $path ) && ! $router->is_excluded_path( $path ); } /** * Whether a pair is waiting: no row, or something pending and not set aside at $at. * * @param array{pending: int, set_aside_until: ?string}|null $row */ private static function waiting( ?array $row, string $at ): bool { return null === $row || ( $row['pending'] > 0 && ! ( null !== $row['set_aside_until'] && $row['set_aside_until'] > $at ) ); } /** * A page's label when it isn't a main page: its post's title, found by its address, else its path; * "/" is "Home". * * @param array $main */ private static function label_for( string $key, array $main ): string { if ( isset( $main[ $key ] ) ) { return $main[ $key ]; } if ( '/' === $key ) { return __( 'Home', 'langferry' ); } $id = url_to_postid( home_url( $key ) ); return self::label( $id ? get_post_field( 'post_title', $id, 'raw' ) : '', rawurldecode( $key ) ); } /** * Plain text for the report (tags stripped, entities decoded, whitespace collapsed), at most 80 * characters: the first 79 and "…" when longer. $fallback, made plain the same way, when nothing * is left. * * @param mixed $text */ private static function label( $text, string $fallback ): string { $text = is_scalar( $text ) ? wp_check_invalid_utf8( wp_strip_all_tags( (string) $text ), true ) : ''; $text = trim( (string) preg_replace( '/[\p{Z}\p{Cc}]+/u', ' ', html_entity_decode( $text, ENT_QUOTES | ENT_HTML5, 'UTF-8' ) ) ); if ( '' === $text ) { return '' === $fallback ? '' : self::label( $fallback, '' ); } return mb_strlen( $text, 'UTF-8' ) > self::LABEL_MAX ? rtrim( mb_substr( $text, 0, self::LABEL_MAX - 1, 'UTF-8' ) ) . '…' : $text; } /** * The category and tag pages the sitemap lists that have posts, as get_terms() arguments. * * @param string[] $taxonomies Seo::sitemap_taxonomies(). */ private static function term_args( array $taxonomies ): array { return array( 'taxonomy' => $taxonomies, 'hide_empty' => true, 'hierarchical' => false, 'exclude' => Seo_Plugins::excluded_term_ids( $taxonomies ), // phpcs:ignore WordPressVIPMinimum.Performance.WPQueryParams.PostNotIn_exclude -- What the SEO plugin leaves out of its sitemap, as Seo::sitemap_terms() leaves it out. 'update_term_meta_cache' => false, ); } /** * @return string[] The site's languages, in its order (the switcher's), the original language not among them. */ private static function languages(): array { return array_values( array_filter( (array) Settings::effective()['languages'], 'is_string' ) ); } /** * The site's languages the order works on at $now: those not set aside, or only $only when it is * one of them. * * @return string[] */ private static function active_languages( int $now, string $only = '' ): array { $langs = array_values( array_diff( self::languages(), array_keys( self::languages_aside( $now ) ) ) ); return '' === $only ? $langs : array_values( array_intersect( $langs, array( $only ) ) ); } /** * $langs without the codes in $leave_out, in $langs' order. * * @param string[] $langs * @param array $leave_out Language codes; anything that isn't a string is ignored. * @return string[] */ private static function without( array $langs, array $leave_out ): array { $leave_out = array_filter( $leave_out, 'is_string' ); return $leave_out ? array_values( array_diff( $langs, $leave_out ) ) : $langs; } /** * The languages set aside at $now (set_language_aside()), as language => array( until, reason, label ). * * @return array */ private static function languages_aside( int $now ): array { $saved = get_option( self::LANGUAGES_SET_ASIDE ); $out = array(); foreach ( is_array( $saved ) ? $saved : array() as $lang => $entry ) { if ( is_string( $lang ) && is_array( $entry ) && (int) ( $entry['until'] ?? 0 ) > $now ) { $out[ $lang ] = array( 'until' => (int) $entry['until'], 'reason' => in_array( $entry['reason'] ?? '', self::REASONS, true ) ? (string) $entry['reason'] : 'error', 'label' => is_string( $entry['label'] ?? null ) ? $entry['label'] : '', ); } } return $out; } /** * $read(), remembered under $key until the pages table changes: a write in this request clears its * cache group's last change (Pages::flush()), and so does forget_memo(). * * @return mixed */ private static function remembered( string $key, callable $read ) { $stamp = wp_cache_get_last_changed( Pages::CACHE_GROUP ); if ( $stamp !== self::$memo_stamp ) { self::$memo = array(); self::$memo_stamp = $stamp; } if ( ! array_key_exists( $key, self::$memo ) ) { self::$memo[ $key ] = $read(); } return self::$memo[ $key ]; } private static function pair( string $lang, string $key, int $tier, string $label ): array { return array( 'lang' => $lang, 'path' => $key, 'tier' => $tier, 'label' => $label, ); } private static function count( int $n ): int { return max( 0, min( self::COUNT_MAX, $n ) ); } }