Stem -> count map. Empty when the post has * no meaningful words (empty or whitespace-only * title and content). */ public static function build( string $title, string $content, WordTokenizer $tokenizer ): array { $plain = trim( $title . ' ' . self::strip_markup( $content ) ); if ( '' === $plain ) { return array(); } return $tokenizer->unique_stems( $plain ); } /** * Strips HTML tags, shortcodes, and Gutenberg block delimiters from content. * * The tokenizer would happily consume markup, but stems built from * tag names (`div`, `span`) would pollute the corpus. We strip in a * conservative way — anything between `<` and `>` goes, and obvious * shortcode and block-comment forms are removed too. * * @since 1.10.0 * * @param string $content Raw post content. * * @return string Plain text suitable for tokenisation. */ private static function strip_markup( string $content ): string { if ( '' === $content ) { return ''; } // Remove Gutenberg block delimiters first; they look like comments // and would otherwise be returned as bare text by strip_tags. $content = preg_replace( '//u', ' ', $content ); if ( null === $content ) { return ''; } // Drop shortcodes' brackets but keep their textual content where // reasonable — `[caption]Text[/caption]` should keep "Text". $content = preg_replace( '/\[\/?[A-Za-z0-9_-]+(?:\s+[^\]]*)?\]/u', ' ', $content ); if ( null === $content ) { return ''; } // Block tags with no whitespace between them (e.g. `

`, as // Gutenberg serialises adjacent blocks) would otherwise strip to // zero characters and glue the two sides together with no space — // corrupting sentence boundaries and anchor-in-sentence matching // downstream. A harmless extra space is preferable to a missing one. $content = str_replace( '><', '> <', $content ); // Strip remaining HTML tags. wp_strip_all_tags handles scripts/styles // and is preferred when WordPress is loaded. if ( function_exists( 'wp_strip_all_tags' ) ) { return (string) wp_strip_all_tags( $content ); } // phpcs:ignore WordPress.WP.AlternativeFunctions.strip_tags_strip_tags -- WordPress is not loaded (unit-test path); strip_tags is the only available fallback. return (string) strip_tags( $content ); } }