link text, for void tags.
* Original tags (with their attributes/URLs) never reach translators or machine services.
*
* @package Soft_ML
*/
defined( 'ABSPATH' ) || exit;
class Soft_ML_Segmenter {
/** Inline elements: stay inside the surrounding segment. */
const INLINE = array(
'a', 'abbr', 'acronym', 'b', 'bdi', 'bdo', 'big', 'br', 'cite', 'data', 'del', 'dfn', 'em', 'font',
'i', 'img', 'ins', 'kbd', 'label', 'mark', 'q', 's', 'small', 'span', 'strike', 'strong', 'sub',
'sup', 'time', 'u', 'wbr', 'input',
);
/** Elements whose entire content is never translated. */
const SKIP = array( 'head', 'title', 'script', 'style', 'svg', 'textarea', 'pre', 'code', 'noscript', 'iframe', 'object', 'template', 'math', 'select' );
const VOID = array( 'br', 'wbr', 'img', 'input', 'hr', 'meta', 'link', 'source', 'area', 'base', 'col', 'embed', 'param', 'track' );
const ATTRS = array( 'alt', 'title', 'placeholder', 'aria-label' );
const TOKEN_RE = '~(||"\'])*+>.*?|"\'])*+>.*?|"\'])*+>.*?|]*>|"\'])*+>)~is';
/**
* Extract unique segments.
*
* @param string $html Rendered HTML.
* @return array hash => [ 'hash', 'text', 'kind' (text|attr), 'tag', 'count' ]
*/
public static function extract( $html ) {
$found = array();
self::process(
$html,
static function ( $kind, $text, $tag ) use ( &$found ) {
$hash = self::hash( $text );
if ( isset( $found[ $hash ] ) ) {
++$found[ $hash ]['count'];
} else {
$found[ $hash ] = array( 'hash' => $hash, 'text' => $text, 'kind' => $kind, 'tag' => $tag, 'count' => 1 );
}
return null;
}
);
return $found;
}
/**
* Apply translations. Segments without a (valid) translation are left untouched.
*
* @param string $html Rendered HTML.
* @param array $translations hash => translated text (same placeholder format).
* @param array $stats Out: [ 'total' => n, 'translated' => n, 'rejected' => n ].
* @return string
*/
public static function apply( $html, array $translations, &$stats = null ) {
$stats = array( 'total' => 0, 'translated' => 0, 'rejected' => 0 );
return self::process(
$html,
static function ( $kind, $text, $tag ) use ( $translations, &$stats ) {
++$stats['total'];
$hash = self::hash( $text );
if ( ! isset( $translations[ $hash ] ) || '' === trim( $translations[ $hash ] ) ) {
return null;
}
$t = $translations[ $hash ];
if ( 'attr' === $kind ) {
if ( substr_count( $t, '<' ) > substr_count( $text, '<' ) || substr_count( $t, '>' ) > substr_count( $text, '>' ) ) {
++$stats['rejected'];
return null;
}
} elseif ( ! self::placeholders_match( $text, $t ) ) {
++$stats['rejected'];
return null;
}
++$stats['translated'];
return $t;
}
);
}
/**
* Stable key for a source text.
*
* @param string $text Normalised source text.
* @return string
*/
public static function hash( $text ) {
return sha1( $text );
}
/**
* Does $translation carry exactly the placeholders of $source and no other markup?
*
* @param string $source Source text.
* @param string $translation Translation.
* @return bool
*/
public static function placeholders_match( $source, $translation ) {
$re = '~~';
preg_match_all( $re, $source, $a );
preg_match_all( $re, $translation, $b );
$a = $a[0];
$b = $b[0];
sort( $a );
sort( $b );
if ( $a !== $b ) {
return false;
}
// Stray angle brackets exist in some legacy source text; allow no more than the source had.
$s = preg_replace( $re, '', $source );
$t = preg_replace( $re, '', $translation );
return substr_count( $t, '<' ) <= substr_count( $s, '<' ) && substr_count( $t, '>' ) <= substr_count( $s, '>' );
}
/**
* Core walker. $cb( kind, text, blockTag ) returns a replacement string or null.
*
* @param string $html Input.
* @param callable $cb Callback.
* @return string
*/
public static function process( $html, callable $cb ) {
$tokens = preg_split( self::TOKEN_RE, $html, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY );
if ( false === $tokens ) {
return $html; // PCRE limit hit: fail safe, untranslated.
}
$out = array();
$run = array();
$skip_name = '';
$skip_n = 0;
$block_tag = 'body';
$n = count( $tokens );
for ( $i = 0; $i < $n; $i++ ) {
$tok = $tokens[ $i ];
if ( '<' !== $tok[0] || ( 1 !== preg_match( '~^<(?:!|/?[a-zA-Z])~', $tok ) ) ) {
if ( $skip_n ) {
$out[] = $tok;
} else {
$run[] = array( 't', $tok );
}
continue;
}
// Comments, doctype, CDATA, raw script/style/svg blocks.
if ( '' === substr( $tok, -2 );
if ( $skip_n ) {
if ( $name === $skip_name && ! $void ) {
$skip_n += $closing ? -1 : 1;
}
$out[] = $tok;
continue;
}
if ( ! $closing && ! $void && ( in_array( $name, self::SKIP, true ) || self::is_no_translate( $tok ) ) ) {
self::flush( $run, $out, $cb, $block_tag );
$skip_name = $name;
$skip_n = 1;
$out[] = $tok;
continue;
}
if ( ! $closing ) {
$tok = self::translate_attrs( $tok, $name, $cb, $block_tag );
}
if ( in_array( $name, self::INLINE, true ) ) {
$run[] = array( 'g', $tok, $name, $closing, $void );
} else {
self::flush( $run, $out, $cb, $block_tag );
$out[] = $tok;
if ( ! $closing ) {
$block_tag = $name;
}
}
}
self::flush( $run, $out, $cb, $block_tag );
return implode( '', $out );
}
/**
* Element opted out of translation: translate="no" (HTML standard) or class "notranslate".
*
* @param string $tag Opening tag.
* @return bool
*/
private static function is_no_translate( $tag ) {
return 1 === preg_match( '~\stranslate\s*=\s*["\']no["\']~i', $tag )
|| 1 === preg_match( '~\sclass\s*=\s*["\'][^"\']*\bnotranslate\b~i', $tag );
}
/**
* Translate alt/title/placeholder/aria-label (and button-like input values) inside one tag.
*/
private static function translate_attrs( $tag, $name, callable $cb, $block_tag ) {
if ( false === strpos( $tag, '=' ) ) {
return $tag;
}
$attrs = self::ATTRS;
if ( 'input' === $name && 1 === preg_match( '~\stype\s*=\s*["\']?(submit|button|reset)\b~i', $tag ) ) {
$attrs[] = 'value';
}
$re = '~(\s(?:' . implode( '|', array_map( 'preg_quote', $attrs ) ) . ')\s*=\s*)("([^"]*)"|\'([^\']*)\')~i';
return preg_replace_callback(
$re,
static function ( $m ) use ( $cb, $block_tag ) {
$quote = $m[2][0];
$value = isset( $m[4] ) ? $m[4] : $m[3];
$norm = trim( preg_replace( '/\s+/u', ' ', $value ) );
if ( '' === $norm || ! self::has_letters( $norm ) ) {
return $m[0];
}
$res = $cb( 'attr', $norm, $block_tag );
if ( null === $res ) {
return $m[0];
}
$res = str_replace( $quote, '"' === $quote ? '"' : ''', $res );
return $m[1] . $quote . $res . $quote;
},
$tag
);
}
/**
* Turn the pending inline run into a segment (or pass it through) and append to $out.
*
* @param array $run Pending tokens (by reference, emptied).
* @param array $out Output pieces (by reference).
* @param callable $cb Callback.
* @param string $block_tag Enclosing block tag (context).
*/
private static function flush( array &$run, array &$out, callable $cb, $block_tag ) {
if ( ! $run ) {
return;
}
$tokens = $run;
$run = array();
$has_text = false;
foreach ( $tokens as $t ) {
if ( 't' === $t[0] && '' !== trim( $t[1] ) ) {
$has_text = true;
break;
}
}
if ( ! $has_text ) {
foreach ( $tokens as $t ) {
$out[] = $t[1];
}
return;
}
$lead = 0;
$trail = count( $tokens );
$prefix = '';
$suffix = '';
// Peel edge whitespace, void tags and comments, then matched wrapper tags (.. around everything).
$peel = static function () use ( &$tokens, &$lead, &$trail, &$prefix, &$suffix ) {
while ( $lead < $trail && self::is_edge_filler( $tokens[ $lead ] ) ) {
$prefix .= $tokens[ $lead ][1];
++$lead;
}
while ( $trail > $lead && self::is_edge_filler( $tokens[ $trail - 1 ] ) ) {
$suffix = $tokens[ $trail - 1 ][1] . $suffix;
--$trail;
}
};
$peel();
while ( $trail - $lead >= 2 ) {
$first = $tokens[ $lead ];
$last = $tokens[ $trail - 1 ];
if ( 'g' === $first[0] && 'g' === $last[0] && ! $first[3] && ! $first[4] && $last[3] && $first[2] === $last[2]
&& self::wraps_all( $tokens, $lead, $trail ) ) {
$prefix .= $first[1];
$suffix = $last[1] . $suffix;
++$lead;
--$trail;
$peel();
continue;
}
break;
}
// Build source text with numbered placeholders.
$src = '';
$map = array(); // placeholder string => original tag string
$ids = 0;
$stack = array();
for ( $k = $lead; $k < $trail; $k++ ) {
$t = $tokens[ $k ];
if ( 't' === $t[0] ) {
$src .= $t[1];
} elseif ( 'r' === $t[0] ) {
// Comments inside a segment would be lost on replacement: keep them as void placeholders.
++$ids;
$map[ "" ] = $t[1];
$src .= "";
} elseif ( $t[4] ) {
++$ids;
$map[ "" ] = $t[1];
$src .= "";
} elseif ( ! $t[3] ) {
++$ids;
$stack[] = array( $t[2], $ids );
$map[ "" ] = $t[1];
$src .= "";
} else {
$id = 0;
for ( $s = count( $stack ) - 1; $s >= 0; $s-- ) {
if ( $stack[ $s ][0] === $t[2] ) {
$id = $stack[ $s ][1];
array_splice( $stack, $s, 1 );
break;
}
}
if ( ! $id ) {
++$ids;
$id = $ids;
$map[ "" ] = '';
}
$map[ "" ] = $t[1];
$src .= "";
}
}
$norm = trim( preg_replace( '/\s+/u', ' ', $src ) );
$text = trim( preg_replace( '~~', '', $norm ) );
if ( '' === $text || ! self::has_letters( $text ) ) {
foreach ( $tokens as $t ) {
$out[] = $t[1];
}
return;
}
$res = $cb( 'text', $norm, $block_tag );
if ( null === $res ) {
foreach ( $tokens as $t ) {
$out[] = $t[1];
}
return;
}
$out[] = $prefix . strtr( $res, $map ) . $suffix;
}
/**
* @param array $t Token.
* @return bool Whitespace text, comment, or void inline tag.
*/
private static function is_edge_filler( array $t ) {
if ( 't' === $t[0] ) {
return '' === trim( $t[1] );
}
if ( 'r' === $t[0] ) {
return true;
}
return 'g' === $t[0] && $t[4];
}
/**
* Does the opening tag at $lead close exactly at $trail-1?
*/
private static function wraps_all( array $tokens, $lead, $trail ) {
$name = $tokens[ $lead ][2];
$depth = 0;
for ( $k = $lead; $k < $trail; $k++ ) {
$t = $tokens[ $k ];
if ( 'g' !== $t[0] || $t[2] !== $name || $t[4] ) {
continue;
}
$depth += $t[3] ? -1 : 1;
if ( 0 === $depth ) {
return $k === $trail - 1;
}
}
return false;
}
/**
* Letters present once entities are removed?
*
* @param string $s Text.
* @return bool
*/
private static function has_letters( $s ) {
return 1 === preg_match( '/\p{L}/u', preg_replace( '/&#?\w+;/', '', $s ) );
}
}