for each language of the current page that search engines may * index: the original language always, a translation only when it isn't hidden as incomplete * (Pages). The page is looked up by the key its render records, so a view makes one query for * both, through the cache. */ public static function hreflang_tags() { $config = Settings::effective(); if ( ! self::indexing_enabled() || ! $config['seo']['hreflang'] ) { return; } if ( is_404() || is_search() || is_preview() || is_feed() ) { return; } $router = Router::instance(); if ( $router->is_excluded_path( $router->request_path() ) ) { return; } $listed = Pages::listed_languages( Pages::key( $router->current_url_in( $config['default_language'] ) ), $config['languages'] ); echo "\n\n"; foreach ( Settings::all_languages() as $lang ) { if ( $lang !== $config['default_language'] && ! in_array( $lang, $listed, true ) ) { continue; } printf( '' . "\n", esc_attr( Languages::hreflang( $lang ) ), esc_url( $router->canonical_url_in( $lang ) ) ); } if ( $config['seo']['x_default'] ) { printf( '' . "\n", esc_url( $router->canonical_url_in( $config['default_language'] ) ) ); } } /** * Language attributes, og:locale and robots on a translated document. */ public static function apply_to_document( Html $doc, $lang, array $stats ) { $root = $doc->root(); if ( ! $root ) { return; } $config = Settings::effective(); $root->setAttribute( 'lang', Languages::hreflang( $lang ) ); if ( Languages::is_rtl( $lang ) ) { $root->setAttribute( 'dir', 'rtl' ); } elseif ( 'rtl' === strtolower( (string) $root->getAttribute( 'dir' ) ) ) { $root->setAttribute( 'dir', 'ltr' ); } $head = self::head( $doc ); if ( ! $head ) { return; } foreach ( $head->childNodes as $child ) { if ( 1 === $child->nodeType && 'meta' === Html::tag( $child ) && 'og:locale' === strtolower( (string) $child->getAttribute( 'property' ) ) ) { $child->setAttribute( 'content', Languages::locale( $lang ) ); } } $noindex = ! self::indexing_enabled() || ( $config['seo']['noindex_incomplete'] && self::should_noindex_incomplete( $stats ) ); if ( $noindex ) { self::set_robots( $doc, $head, 'noindex, follow' ); } } /** * Decide whether a translated page is too untranslated to be indexed. * * A page can be partly untranslated when the translation allowance runs out or the * cloud is unreachable on first view. Indexing it would show Google an * English page at a Spanish URL (a duplicate) and a poor first impression in * Spanish search results; noindexing it delays indexing until the next crawl. * * @param array $stats segments, translated, chars, translated_chars, error. */ public static function should_noindex_incomplete( array $stats ) { if ( empty( $stats['chars'] ) ) { return false; } $coverage = $stats['translated_chars'] / $stats['chars']; return $coverage < self::noindex_threshold( $stats ); } /** * The share of a page's text that must be translated for it to be indexed, and listed in * hreflang and the sitemaps (Pages). * * @param array $stats The page's stats (chars, translated_chars, pending, …), for the filter. */ public static function noindex_threshold( array $stats = array() ): float { // Business decision: below 80% translated text, hide the page from search // engines until it is complete. Tune via the filter below. /** * Filters the share of a page's text that must be translated for it to be indexed. * * @param float $threshold 0.8 by default. * @param array $stats The page's stats; empty when no page is in hand. */ return (float) apply_filters( 'langferry_noindex_coverage_threshold', 0.8, $stats ); } private static function head( Html $doc ) { foreach ( $doc->root()->childNodes as $child ) { if ( 1 === $child->nodeType && 'head' === Html::tag( $child ) ) { return $child; } } return null; } private static function set_robots( Html $doc, $head, $content ) { foreach ( $head->childNodes as $child ) { if ( 1 === $child->nodeType && 'meta' === Html::tag( $child ) && 'robots' === strtolower( (string) $child->getAttribute( 'name' ) ) ) { $child->setAttribute( 'content', $content ); return; } } $meta = $doc->create_element( 'meta' ); $meta->setAttribute( 'name', 'robots' ); $meta->setAttribute( 'content', $content ); $head->appendChild( $meta ); } /* ------------------------------------------------------------------ */ /* Sitemap */ /* ------------------------------------------------------------------ */ public static function sitemap_url() { return home_url( '/langferry-sitemap.xml' ); } public static function robots_txt( $output, $public ) { $config = Settings::effective(); if ( $public && self::indexing_enabled() && $config['seo']['sitemap'] ) { $output .= "\nSitemap: " . esc_url_raw( self::sitemap_url() ) . "\n"; } return $output; } /** * The sitemap index lists, for each language, its files of pages and then of category and tag * pages: /langferry-sitemap-es-1.xml and /langferry-sitemap-es-taxonomies-1.xml contain only * Spanish URLs (each with every hreflang alternate), so Search Console reports indexing per language. */ public static function maybe_serve_sitemap() { $response = self::sitemap_response( Router::instance()->request_path() ); if ( null === $response ) { return; } if ( 404 === $response[0] ) { status_header( 404 ); exit; } nocache_headers(); header( 'Content-Type: application/xml; charset=UTF-8' ); header( 'X-Robots-Tag: noindex, follow' ); echo $response[1]; // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped exit; } /** * The sitemap file a path asks for: array( 200, XML ); array( 404, '' ) for a sitemap address that * isn't served (the sitemap is off, or the site doesn't have that language); null for any other path. * * @param string $path Request path, relative to the site's home. * @return array{0: int, 1: string}|null */ public static function sitemap_response( $path ) { $file = self::parse_sitemap_path( $path ); if ( null === $file ) { return null; } list( $slug, $kind, $page ) = $file; $lang = '' === $slug ? null : Languages::from_url_slug( $slug, Settings::all_languages() ); if ( ! self::indexing_enabled() || ! Settings::effective()['seo']['sitemap'] || ( '' !== $slug && null === $lang ) ) { return array( 404, '' ); } if ( 'index' === $kind ) { return array( 200, self::sitemap_index() ); } return array( 200, 'taxonomies' === $kind ? self::sitemap_terms_page( $lang, $page ) : self::sitemap_page( $lang, $page ) ); } /** * What a sitemap address asks for, or null if it isn't one: the language's URL slug ('' for the * index), the kind of file ('index', 'posts' or 'taxonomies'), and the page (from 1; 0 for the index). * /langferry-sitemap.xml, /langferry-sitemap-es-1.xml, /langferry-sitemap-es-taxonomies-1.xml. * * @param string $path Request path, relative to the site's home. * @return array{0: string, 1: string, 2: int}|null */ public static function parse_sitemap_path( $path ) { if ( '/langferry-sitemap.xml' === $path ) { return array( '', 'index', 0 ); } if ( ! preg_match( '#^/langferry-sitemap-([a-z]{2,3}(?:-[a-z]{2})?)(-taxonomies)?-([1-9]\d*)\.xml$#', (string) $path, $m ) ) { return null; } return array( $m[1], '' === $m[2] ? 'posts' : 'taxonomies', (int) $m[3] ); } /** * URL of one language's sitemap file (used by the dashboard / Search Console sync). */ public static function language_sitemap_url( $lang, $page = 1, $kind = 'posts' ) { return home_url( '/langferry-sitemap-' . Languages::url_slug( $lang ) . ( 'taxonomies' === $kind ? '-taxonomies' : '' ) . '-' . (int) $page . '.xml' ); } /** * Post types the sitemap lists: those WordPress's own sitemap would list (public and * viewable, never media files, less any a plugin removes with wp_sitemaps_post_types), * less those the site's SEO plugin keeps out of its sitemap. * * @return string[] */ public static function sitemap_post_types() { $types = get_post_types( array( 'public' => true ), 'objects' ); unset( $types['attachment'] ); $types = array_filter( $types, 'is_post_type_viewable' ); /** This filter is documented in wp-includes/sitemaps/providers/class-wp-sitemaps-posts.php */ $types = (array) apply_filters( 'wp_sitemaps_post_types', $types ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- WordPress's own sitemap filter, so a plugin that removes a type there removes it here too. $names = array(); foreach ( array_keys( $types ) as $name ) { if ( Seo_Plugins::lists_post_type( $name ) ) { $names[ $name ] = $name; } } /** * Filters the post types the multilingual sitemap lists. * * @param array $types Post type names, keyed by name. */ return array_values( (array) apply_filters( 'langferry_sitemap_post_types', $names ) ); } /** * One page of the published posts the sitemap lists, and how many it lists in all. The * index and the files share this query, so they always agree. * * @return array{0: int[], 1: int} Post IDs, total. */ private static function sitemap_posts( $page ) { $types = self::sitemap_post_types(); if ( ! $types ) { return array( array(), 0 ); // An empty post_type would mean "post" to WP_Query. } $query = new \WP_Query( array_merge( self::sitemap_query_args( $types ), array( 'posts_per_page' => self::page_size(), 'paged' => max( 1, (int) $page ), ) ) ); return array( array_map( 'intval', $query->posts ), (int) $query->found_posts ); } /** * The query for the posts the sitemap lists, as IDs. Measuring the site (Site_Size) uses it too, so the * estimate counts exactly the pages the sitemap lists. * * @param string[] $types Post types the sitemap lists (not empty). */ public static function sitemap_query_args( array $types ) { return array( 'post_type' => $types, 'post_status' => 'publish', 'orderby' => 'ID', 'order' => 'ASC', 'fields' => 'ids', 'has_password' => false, 'post__not_in' => Seo_Plugins::excluded_post_ids( $types ), // phpcs:ignore WordPressVIPMinimum.Performance.WPQueryParams.PostNotIn_post__not_in -- What the SEO plugin leaves out; filtering afterwards would break the file count. 'ignore_sticky_posts' => true, 'suppress_filters' => false, ); } /** * Taxonomies whose category and tag pages the sitemap lists: those WordPress's own sitemap would * list (public and viewable, less any a plugin removes with wp_sitemaps_taxonomies), less those the * site's SEO plugin keeps out of its sitemap. * * @return string[] */ public static function sitemap_taxonomies() { $taxonomies = array_filter( get_taxonomies( array( 'public' => true ), 'objects' ), 'is_taxonomy_viewable' ); /** This filter is documented in wp-includes/sitemaps/providers/class-wp-sitemaps-taxonomies.php */ $taxonomies = (array) apply_filters( 'wp_sitemaps_taxonomies', $taxonomies ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- WordPress's own sitemap filter. $names = array(); foreach ( array_keys( $taxonomies ) as $name ) { if ( Seo_Plugins::lists_taxonomy( $name ) ) { $names[ $name ] = $name; } } /** * Filters the taxonomies whose category and tag pages the multilingual sitemap lists. * * @param array $taxonomies Taxonomy names, keyed by name. */ return array_values( (array) apply_filters( 'langferry_sitemap_taxonomies', $names ) ); } /** * One page of the category and tag pages the sitemap lists, and how many it lists in all. A term * with no published posts isn't listed, as in WordPress's own sitemap, unless the SEO plugin's * sitemap lists it (see Seo_Plugins::empty_terms()). * * @return array{0: int[], 1: int} Term IDs, total. */ private static function sitemap_terms( $page ) { $taxonomies = self::sitemap_taxonomies(); if ( ! $taxonomies ) { return array( array(), 0 ); // No taxonomy would mean every taxonomy to WP_Term_Query. } $empty = array( 'none' => array(), 'all' => array(), 'parents' => array(), 'ancestors' => array(), ); foreach ( $taxonomies as $taxonomy ) { $empty[ Seo_Plugins::empty_terms( $taxonomy ) ][] = $taxonomy; } $hide_empty = count( $empty['none'] ) === count( $taxonomies ); $exclude = Seo_Plugins::excluded_term_ids( $taxonomies ); if ( ! $hide_empty ) { // Some taxonomies list empty terms: leave out the ones the others don't. $exclude = array_merge( $exclude, self::empty_term_ids( $empty['none'] ), self::empty_term_ids( $empty['parents'], 'parents' ), self::empty_term_ids( $empty['ancestors'], 'ancestors' ) ); } $args = array( 'taxonomy' => $taxonomies, 'hide_empty' => $hide_empty, 'hierarchical' => false, 'exclude' => $exclude, // phpcs:ignore WordPressVIPMinimum.Performance.WPQueryParams.PostNotIn_exclude -- What the SEO plugin leaves out; filtering afterwards would break the file count. 'update_term_meta_cache' => false, ); $total = wp_count_terms( $args ); $ids = get_terms( $args + array( 'orderby' => 'term_id', 'order' => 'ASC', 'number' => self::page_size(), 'offset' => ( max( 1, (int) $page ) - 1 ) * self::page_size(), 'fields' => 'ids', ) ); return array( is_array( $ids ) ? array_map( 'intval', $ids ) : array(), is_numeric( $total ) ? (int) $total : 0 ); } /** * Terms of these taxonomies with no published posts, less, with $keep 'parents', those with a * direct child that has some, or with 'ancestors', those with posts anywhere below them. * * @param string[] $taxonomies Taxonomy names. * @param string $keep '', 'parents' or 'ancestors'. * @return int[] */ private static function empty_term_ids( array $taxonomies, $keep = '' ) { global $wpdb; if ( ! $taxonomies ) { return array(); } // phpcs:disable WordPress.DB.DirectDatabaseQuery -- one query for every empty term; WordPress has no API for it. if ( 'parents' === $keep ) { // Empty terms without a non-empty child. return array_map( 'intval', $wpdb->get_col( $wpdb->prepare( 'SELECT tt.term_id FROM %i tt WHERE tt.count = 0 AND tt.taxonomy IN (' . implode( ',', array_fill( 0, count( $taxonomies ), '%s' ) ) . ') AND NOT EXISTS ( SELECT 1 FROM %i c WHERE c.parent = tt.term_id AND c.taxonomy = tt.taxonomy AND c.count > 0 )', array_merge( array( $wpdb->term_taxonomy ), $taxonomies, array( $wpdb->term_taxonomy ) ) ) ) ); } if ( 'ancestors' !== $keep ) { return array_map( 'intval', $wpdb->get_col( $wpdb->prepare( 'SELECT tt.term_id FROM %i tt WHERE tt.count = 0 AND tt.taxonomy IN (' . implode( ',', array_fill( 0, count( $taxonomies ), '%s' ) ) . ')', array_merge( array( $wpdb->term_taxonomy ), $taxonomies ) ) ) ); } // phpcs:enable WordPress.DB.DirectDatabaseQuery $ids = array(); foreach ( $taxonomies as $taxonomy ) { $parent = array(); $empty = array(); foreach ( $wpdb->get_results( $wpdb->prepare( "SELECT term_id, parent, count FROM {$wpdb->term_taxonomy} WHERE taxonomy = %s", $taxonomy ) ) as $row ) { // phpcs:ignore WordPress.DB.DirectDatabaseQuery $parent[ (int) $row->term_id ] = (int) $row->parent; if ( 0 === (int) $row->count ) { $empty[ (int) $row->term_id ] = true; } } $kept = array(); foreach ( $parent as $id => $up ) { if ( isset( $empty[ $id ] ) ) { continue; } while ( $up && isset( $parent[ $up ] ) && ! isset( $kept[ $up ] ) ) { // Each ancestor once, so a loop in the data can't hang. $kept[ $up ] = true; $up = $parent[ $up ]; } } $ids = array_merge( $ids, array_keys( array_diff_key( $empty, $kept ) ) ); } return $ids; } /** * The sitemap index: one file per language and page. */ public static function sitemap_index() { list( , $posts ) = self::sitemap_posts( 1 ); list( , $terms ) = self::sitemap_terms( 1 ); $files = array( 'posts' => max( 1, (int) ceil( $posts / self::page_size() ) ), // The home page rides along on page 1. 'taxonomies' => (int) ceil( $terms / self::page_size() ), ); $xml = '' . "\n"; $xml .= '' . "\n"; foreach ( Settings::all_languages() as $lang ) { foreach ( $files as $kind => $pages ) { for ( $i = 1; $i <= $pages; $i++ ) { $xml .= "\t" . esc_url( self::language_sitemap_url( $lang, $i, $kind ) ) . "\n"; } } } return $xml . "\n"; } /** * One language's sitemap file: that language's address of each page listed in it, with its * alternates (urlset()). File 1 starts with the home page. */ public static function sitemap_page( $lang, $page ) { $urls = array(); if ( 1 === $page ) { $urls[] = array( home_url( '/' ), '' ); } list( $ids ) = self::sitemap_posts( $page ); foreach ( $ids as $id ) { $link = get_permalink( $id ); if ( $link && untrailingslashit( $link ) !== untrailingslashit( home_url( '/' ) ) ) { $urls[] = array( $link, get_post_modified_time( 'c', true, $id ) ); } } return self::urlset( $lang, $urls ); } /** * One language's file of category and tag pages listed in it (urlset()). No lastmod, as in * WordPress's own sitemap. */ public static function sitemap_terms_page( $lang, $page ) { $urls = array(); list( $ids ) = self::sitemap_terms( $page ); foreach ( $ids as $id ) { $link = get_term_link( $id ); if ( ! is_wp_error( $link ) ) { $urls[] = array( $link, '' ); } } return self::urlset( $lang, $urls ); } /** * A sitemap file: $lang's address of each URL listed in $lang, with its alternates in the * languages it is listed in. A translation hidden as incomplete isn't pointed at (Pages); the * original language lists every URL. A file left with no URL lists $lang's home page instead, so * every file the index lists stays readable from the day a language is switched on. That entry * names its own address among its alternates, as a sitemap URL must, even while the home page is * incomplete in $lang; its other alternates are filtered as usual. One query for the rows of every * URL in every language. * * @param string $lang Language code. * @param array $urls Each array( original-language URL, lastmod or '' ). */ private static function urlset( $lang, array $urls ) { $config = Settings::effective(); $router = Router::instance(); $languages = Settings::all_languages(); $default = $config['default_language']; if ( $config['seo']['translate_slugs'] ) { $paths = array_map( static function ( $u ) { return (string) wp_parse_url( $u[0], PHP_URL_PATH ); }, $urls ); Slugs::ensure( $paths, $config['languages'] ); } $entries = array(); foreach ( $urls as $entry ) { if ( ! $router->is_excluded_path( (string) wp_parse_url( $entry[0], PHP_URL_PATH ) ) ) { $entries[] = array( $entry[0], $entry[1], Pages::key( $entry[0] ) ); } } $home = array( home_url( '/' ), '', Pages::key( home_url( '/' ) ) ); $listed = Pages::listed_matrix( array_merge( array_column( $entries, 2 ), array( $home[2] ) ), $config['languages'] ); $shown = array(); foreach ( $entries as $entry ) { if ( $lang === $default || in_array( $lang, $listed[ $entry[2] ], true ) ) { $shown[] = $entry; } } $fallback = ! $shown; if ( $fallback ) { $shown[] = $home; } $xml = '' . "\n"; $xml .= '' . "\n"; foreach ( $shown as list( $url, $lastmod, $key ) ) { $alternates = array(); foreach ( $languages as $l ) { if ( $l === $default || ( $fallback && $l === $lang ) || in_array( $l, $listed[ $key ], true ) ) { $alternates[ $l ] = $router->translate_url( $url, $l ); } } $xml .= "\t\n\t\t" . esc_url( $router->translate_url( $url, $lang ) ) . "\n"; if ( $lastmod ) { $xml .= "\t\t" . esc_html( $lastmod ) . "\n"; } foreach ( $alternates as $alt_lang => $alt_url ) { $xml .= "\t\t" . '' . "\n"; } if ( $config['seo']['x_default'] ) { $xml .= "\t\t" . '' . "\n"; } $xml .= "\t\n"; } return $xml . "\n"; } }