<?php
/**
 * Fetches key public pages and records every tracking tag, third-party script, pixel and embed.
 *
 * @package GoConsent
 */

namespace GoConsent\Scanner;

use GoConsent\Frontend\LocalFonts;
use GoConsent\Settings;

defined( 'ABSPATH' ) || exit;

/**
 * Site scanner.
 */
class Scanner {

	const OPTION      = 'goconsent_scan';
	const TOKEN_KEY   = 'goconsent_scan_token';
	const POLICY_LIVE = 'goconsent_policy_live';

	/**
	 * Settings.
	 *
	 * @var Settings
	 */
	private $settings;

	/**
	 * Classifier.
	 *
	 * @var Classifier
	 */
	public $classifier;

	/**
	 * Constructor.
	 *
	 * @param Settings $settings Settings.
	 */
	public function __construct( Settings $settings ) {
		$this->settings   = $settings;
		$this->classifier = new Classifier( $settings );
	}

	/**
	 * Last scan result.
	 *
	 * @return array{time:int,pages:string[],errors:string[],items:array}
	 */
	public function last() {
		$r = get_option( self::OPTION, array() );
		return wp_parse_args(
			is_array( $r ) ? $r : array(),
			array(
				'time'   => 0,
				'pages'  => array(),
				'errors' => array(),
				'reason' => '',
				'items'  => array(),
			)
		);
	}

	/**
	 * Whether a request carries a valid scan token (the page is then served unfiltered).
	 *
	 * @return bool
	 */
	public static function is_scan_request() {
		if ( ! isset( $_GET['goconsent_scan'] ) ) { // phpcs:ignore WordPress.Security.NonceVerification.Recommended -- token checked below.
			return false;
		}
		$token = get_transient( self::TOKEN_KEY );
		$given = sanitize_text_field( wp_unslash( $_GET['goconsent_scan'] ) ); // phpcs:ignore WordPress.Security.NonceVerification.Recommended
		return $token && hash_equals( (string) $token, $given );
	}

	/**
	 * Scan the site now.
	 *
	 * @return array
	 */
	public function run() {
		// One scan at a time: each page fetch holds a PHP worker on this server.
		if ( get_transient( 'goconsent_scan_lock' ) ) {
			return $this->last();
		}
		set_transient( 'goconsent_scan_lock', 1, 5 * MINUTE_IN_SECONDS );

		$token = wp_generate_password( 24, false );
		set_transient( self::TOKEN_KEY, $token, 5 * MINUTE_IN_SECONDS );

		$items   = array();
		$errors  = array();
		$reason  = '';
		$pages   = array();
		$pp      = $this->settings->privacy_page_id();
		$pp_url = $pp ? get_permalink( $pp ) : '';

		$fails   = 0;
		$started = microtime( true );
		foreach ( $this->urls() as $url ) {
			/*
			 * Loopback safety: on small hosting plans every page fetch needs a free PHP
			 * worker. Stop after two failed or slow fetches, or after 45 seconds, so the
			 * scan can never tie up the site.
			 */
			if ( $fails >= 2 || microtime( true ) - $started > 45 ) {
				$errors[] = $url;
				continue;
			}
			$t0  = microtime( true );
			$res = self::fetch( add_query_arg( 'goconsent_scan', $token, $url ) );
			if ( is_wp_error( $res ) || 200 !== (int) wp_remote_retrieve_response_code( $res ) ) {
				$errors[] = $url;
				if ( '' === $reason ) {
					$reason = self::explain( $res );
				}
				++$fails;
				continue;
			}
			if ( microtime( true ) - $t0 > 6 ) {
				++$fails; // Slow response: the server is busy, back off.
			}
			$pages[] = $url;
			$html    = (string) wp_remote_retrieve_body( $res );
			$this->parse( $html, $url, $items );
			if ( $pp_url && $url === $pp_url ) {
				update_option(
					self::POLICY_LIVE,
					array(
						'text' => self::page_text( $html ),
						'time' => time(),
						'page' => $pp,
					),
					false
				);
			}
		}
		delete_transient( self::TOKEN_KEY );
		delete_transient( 'goconsent_scan_lock' );

		$result = array(
			'time'   => time(),
			'pages'  => $pages,
			'errors' => $errors,
			'reason' => substr( sanitize_text_field( $reason ), 0, 300 ),
			'items'  => $items,
		);
		update_option( self::OPTION, $result, false );
		return $result;
	}

	/**
	 * Visible main text of a page, as visitors read it (theme templates included).
	 *
	 * @param string $html Page HTML.
	 * @return string
	 */
	public static function page_text( $html ) {
		$html = preg_replace( '#<(script|style|noscript|template|svg|header|footer|nav|dialog)\b[^>]*>.*?</\1>#is', ' ', $html );
		$html = preg_replace( '#<div[^>]+class="[^"]*\bgc-(banner|footer-link)\b[^"]*"[^>]*>.*?</div>#is', ' ', $html );
		if ( preg_match( '#<main\b[^>]*>(.*)</main>#is', $html, $m ) ) {
			$html = $m[1];
		} elseif ( preg_match( '#<body\b[^>]*>(.*)</body>#is', $html, $m ) ) {
			$html = $m[1];
		}
		$text = html_entity_decode( wp_strip_all_tags( str_replace( '<', ' <', $html ) ), ENT_QUOTES, 'UTF-8' );
		return substr( trim( preg_replace( '/\s+/u', ' ', $text ) ), 0, 100000 );
	}

	/**
	 * Re-read only the privacy page as visitors see it (after someone edits it).
	 *
	 * @return bool Whether the page was read.
	 */
	public function refresh_policy() {
		$pp = $this->settings->privacy_page_id();
		if ( ! $pp ) {
			return false;
		}
		$token = wp_generate_password( 24, false );
		set_transient( self::TOKEN_KEY, $token, MINUTE_IN_SECONDS );
		$res = self::fetch( add_query_arg( 'goconsent_scan', $token, get_permalink( $pp ) ) );
		delete_transient( self::TOKEN_KEY );
		if ( is_wp_error( $res ) || 200 !== (int) wp_remote_retrieve_response_code( $res ) ) {
			// Don't keep checking an old copy: fall back to the saved page text.
			delete_option( self::POLICY_LIVE );
			return false;
		}
		$text  = self::page_text( (string) wp_remote_retrieve_body( $res ) );
		$title = trim( wp_strip_all_tags( html_entity_decode( get_the_title( $pp ), ENT_QUOTES, 'UTF-8' ) ) );
		// A maintenance, coming-soon or login page answered instead of the privacy page.
		if ( '' !== $title && false === stripos( $text, $title ) ) {
			delete_option( self::POLICY_LIVE );
			return false;
		}
		update_option(
			self::POLICY_LIVE,
			array(
				'text' => $text,
				'time' => time(),
				'page' => $pp,
			),
			false
		);
		return true;
	}

	/**
	 * Plain-English reason a page couldn't be read, with the technical detail after it.
	 *
	 * @param array|\WP_Error $res Response.
	 * @return string
	 */
	public static function explain( $res ) {
		if ( is_wp_error( $res ) ) {
			$msg = (string) $res->get_error_message();
			$map = array(
				'/cURL error 28|timed out/i'                         => __( 'The site took too long to answer itself (server busy or slow).', 'goconsent' ),
				'/cURL error (35|51|58|59|60|77)|SSL|certificate/i'  => __( 'This server can’t verify its own SSL certificate.', 'goconsent' ),
				'/cURL error (6|7)\b|resolve|couldn.t connect|refused/i' => __( 'This server can’t connect to its own web address (hosting or DNS setting).', 'goconsent' ),
				'/blocked|WP_HTTP_BLOCK_EXTERNAL/i'                  => __( 'Outgoing requests are blocked in wp-config.php (WP_HTTP_BLOCK_EXTERNAL).', 'goconsent' ),
			);
			foreach ( $map as $re => $text ) {
				if ( preg_match( $re, $msg ) ) {
					return $text . ' (' . $msg . ')';
				}
			}
			return $msg;
		}
		$code = (int) wp_remote_retrieve_response_code( $res );
		$page = self::block_text( $res );
		if ( in_array( $code, array( 403, 406, 429, 503 ), true ) ) {
			$who = self::blocker( $res );
			if ( $who ) {
				/* translators: 1: firewall name and fix, 2: HTTP status code. */
				return sprintf( __( '%1$s (HTTP %2$d)', 'goconsent' ), $who, $code ) . $page;
			}
		}
		$map  = array(
			401 => __( 'The site is password protected.', 'goconsent' ),
			403 => __( 'A firewall or security plugin blocked the scanner.', 'goconsent' ),
			404 => __( 'The page wasn’t found.', 'goconsent' ),
			429 => __( 'A firewall rate-limited the scanner.', 'goconsent' ),
			503 => __( 'The site is in maintenance mode or overloaded.', 'goconsent' ),
		);
		$text = isset( $map[ $code ] ) ? $map[ $code ] : ( $code >= 500 ? __( 'The page had a server error.', 'goconsent' ) : ( $code >= 300 && $code < 400 ? __( 'The page redirects too many times.', 'goconsent' ) : __( 'The page didn’t load normally.', 'goconsent' ) ) );
		/* translators: %d: HTTP status code. */
		return $text . ' ' . sprintf( __( '(HTTP %d)', 'goconsent' ), $code ) . $page;
	}

	/**
	 * The visible text of an error page, quoted, so the person can tell who blocked the scan.
	 *
	 * @param array $res Response.
	 * @return string
	 */
	private static function block_text( $res ) {
		$html = (string) wp_remote_retrieve_body( $res );
		$html = preg_replace( '#<(script|style|noscript)\b[^>]*>.*?</\1>#is', ' ', substr( $html, 0, 20000 ) );
		$text = trim( preg_replace( '/\s+/u', ' ', html_entity_decode( wp_strip_all_tags( str_replace( '<', ' <', $html ) ), ENT_QUOTES, 'UTF-8' ) ) );
		$srv  = sanitize_text_field( (string) wp_remote_retrieve_header( $res, 'server' ) );
		if ( '' === $text && '' === $srv ) {
			return '';
		}
		$text = function_exists( 'mb_substr' ) ? mb_substr( $text, 0, 160 ) : substr( $text, 0, 160 );
		/* translators: 1: error page text, 2: web server name. */
		return ' ' . sprintf( __( 'Error page: “%1$s” (server: %2$s).', 'goconsent' ), sanitize_text_field( $text ), $srv ? $srv : '?' );
	}

	/**
	 * Name the firewall that blocked the scanner, from the block page and headers, plus what to allow.
	 *
	 * @param array $res Response.
	 * @return string
	 */
	private static function blocker( $res ) {
		$body   = strtolower( substr( (string) wp_remote_retrieve_body( $res ), 0, 20000 ) );
		$server = strtolower( (string) wp_remote_retrieve_header( $res, 'server' ) );
		$ip     = isset( $_SERVER['SERVER_ADDR'] ) ? sanitize_text_field( wp_unslash( $_SERVER['SERVER_ADDR'] ) ) : '';
		/* translators: %s: IP address. */
		$allow = $ip ? ' ' . sprintf( __( 'Allowlist this server’s IP address (%s) there.', 'goconsent' ), $ip ) : '';
		$has   = static function ( $needles ) use ( $body ) {
			foreach ( (array) $needles as $n ) {
				if ( false !== strpos( $body, $n ) ) {
					return true;
				}
			}
			return false;
		};
		if ( $has( array( 'defender', 'wpdef', 'wd_lockout', 'wp-defender' ) ) ) {
			$why = $has( array( 'country', 'location', 'region' ) ) ? __( 'its Location banning (country block)', 'goconsent' ) : ( $has( array( 'user agent', 'user-agent', 'bot' ) ) ? __( 'its bot or User Agent rules', 'goconsent' ) : __( 'its firewall', 'goconsent' ) );
			/* translators: %s: which Defender feature. */
			return sprintf( __( 'Defender (Site Defender) blocked the scanner with %s.', 'goconsent' ), $why ) . $allow;
		}
		if ( $has( 'wordfence' ) ) {
			return __( 'Wordfence blocked the scanner.', 'goconsent' ) . $allow;
		}
		if ( $has( array( 'ithemes', 'solid security', 'itsec' ) ) ) {
			return __( 'Solid Security (iThemes) blocked the scanner.', 'goconsent' ) . $allow;
		}
		if ( $has( 'sucuri' ) || wp_remote_retrieve_header( $res, 'x-sucuri-id' ) ) {
			return __( 'The Sucuri firewall blocked the scanner.', 'goconsent' ) . $allow;
		}
		if ( false !== strpos( $server, 'cloudflare' ) || wp_remote_retrieve_header( $res, 'cf-ray' ) ) {
			return __( 'Cloudflare blocked the scanner (a WAF rule or Bot Fight Mode).', 'goconsent' ) . $allow;
		}
		if ( $has( array( 'mod_security', 'modsecurity', 'not acceptable' ) ) ) {
			return __( 'The web host’s ModSecurity firewall blocked the scanner.', 'goconsent' ) . $allow;
		}
		if ( $has( array( 'all in one wp security', 'aiowps' ) ) ) {
			return __( 'All In One WP Security blocked the scanner.', 'goconsent' ) . $allow;
		}
		if ( strlen( $body ) < 1500 && ( false !== strpos( $server, 'nginx' ) || false !== strpos( $server, 'apache' ) || false !== strpos( $server, 'litespeed' ) ) ) {
			/* translators: %s: web server name. */
			return sprintf( __( 'The web host’s firewall blocked the scanner before WordPress loaded (%s).', 'goconsent' ), $server ) . ( $ip ? ' ' . sprintf( /* translators: %s: IP address. */ __( 'Allowlist this server’s IP address (%s) in the hosting panel’s firewall.', 'goconsent' ), $ip ) : '' );
		}
		if ( defined( 'DEFENDER_VERSION' ) || defined( 'WP_DEFENDER_FILE' ) || class_exists( 'WP_Defender\\Bootstrap', false ) ) {
			return __( 'Defender (Site Defender) likely blocked the scanner. Check Firewall > Local Firewall > Malicious bot detector and Banning locations.', 'goconsent' ) . $allow;
		}
		if ( preg_match( '#<title>(.{1,80}?)</title>#', $body, $m ) ) {
			/* translators: %s: block page title. */
			return sprintf( __( 'A firewall blocked the scanner. Its block page says “%s”.', 'goconsent' ), sanitize_text_field( $m[1] ) ) . $allow;
		}
		return '';
	}

	/**
	 * Fetch one of this site's own pages, the way WordPress core's own loopback checks do.
	 *
	 * SSL is checked as core does for requests to itself ("https_local_ssl_verify", off by
	 * default): many hosts route a site's requests to itself internally, where the
	 * certificate can't be verified even though visitors see a valid one.
	 *
	 * @param string $url URL on this site.
	 * @return array|\WP_Error
	 */
	private static function fetch( $url ) {
		return wp_remote_get(
			$url,
			array(
				'timeout'     => 8,
				'redirection' => 3,
				'sslverify'   => (bool) apply_filters( 'https_local_ssl_verify', false ), // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- WordPress core filter.
				// WordPress's own user agent, as its loopback checks use: some host firewalls block
				// anything that calls itself a "scanner". Scan requests are recognised by the token.
				'user-agent'  => 'WordPress/' . get_bloginfo( 'version' ) . '; ' . home_url(),
				// Standard request headers: security plugins (e.g. Defender's "Empty headers") block requests without them.
				'headers'     => array(
					'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
					'Accept-Language' => 'en-US,en;q=0.9',
					'Cache-Control'   => 'no-cache',
					'Referer'         => home_url( '/' ),
				),
				'cookies'     => array(),
			)
		);
	}

	/**
	 * Pages to scan: home, key flows, privacy page, two most recently edited pages, one item per custom post type.
	 *
	 * @return string[]
	 */
	public function urls() {
		$urls = array( home_url( '/' ) );
		foreach ( array( 'book-now', 'booking', 'contact', 'contact-us', 'checkout', 'shop', 'cart' ) as $slug ) {
			$p = get_page_by_path( $slug );
			if ( $p && 'publish' === $p->post_status ) {
				$urls[] = get_permalink( $p );
			}
		}
		$pp = $this->settings->privacy_page_id();
		if ( $pp ) {
			$urls[] = get_permalink( $pp );
		}
		$recent = get_posts(
			array(
				'post_type'      => 'page',
				'posts_per_page' => 2,
				'orderby'        => 'modified',
				'post_status'    => 'publish',
				'fields'         => 'ids',
			)
		);
		foreach ( $recent as $id ) {
			$urls[] = get_permalink( $id );
		}
		// One recent item from up to three custom post types (listings, vehicles, products...),
		// where maps and booking widgets usually live.
		$types = array_slice(
			array_values(
				get_post_types(
					array(
						'public'   => true,
						'_builtin' => false,
					)
				)
			),
			0,
			3
		);
		foreach ( $types as $type ) {
			$one = get_posts(
				array(
					'post_type'      => $type,
					'posts_per_page' => 1,
					'post_status'    => 'publish',
					'fields'         => 'ids',
				)
			);
			if ( $one ) {
				$urls[] = get_permalink( $one[0] );
			}
		}
		/**
		 * Filter the URLs the scanner checks.
		 *
		 * @param string[] $urls URLs.
		 */
		$urls = apply_filters( 'goconsent_scan_urls', $urls );
		return array_slice( array_values( array_unique( array_filter( $urls ) ) ), 0, 10 );
	}

	/**
	 * Record what one page loads.
	 *
	 * @param string $html  Page HTML.
	 * @param string $url   Page URL.
	 * @param array  $items Items found so far (by reference).
	 */
	public function parse( $html, $url, array &$items ) {
		// Scripts. One tag can match several services (e.g. GA + Ads in one block).
		if ( preg_match_all( '#<script\b([^>]*)>(.*?)</script>#is', $html, $m, PREG_SET_ORDER | PREG_OFFSET_CAPTURE ) ) {
			$source = new Source( $html, $url );
			foreach ( $m as $mm ) {
				$s = array( $mm[0][0], $mm[1][0], $mm[2][0] );
				if ( false !== stripos( $s[1], 'data-goconsent-skip' ) || preg_match( '#\btype\s*=\s*["\']?(application/ld\+json|text/template|text/x-template)#i', $s[1] ) ) {
					continue;
				}
				$hay   = $s[1] . ' ' . $s[2];
				$found = false;
				$from  = null;
				foreach ( Signatures::all() as $id => $sig ) {
					foreach ( $sig[2] as $needle ) {
						if ( false !== stripos( $hay, $needle ) ) {
							$from = null === $from ? $source->detect( $s[1], $mm[0][1] ) : $from;
							$this->add( $items, $id, $sig[0], $sig[1], 'script', $needle, $url, true, $from );
							$found = true;
							break;
						}
					}
				}
				if ( $found ) {
					continue;
				}
				$src = Classifier::attr( $s[1], 'src' );
				if ( $src && $this->classifier->is_third_party( $src ) ) {
					$host = strtolower( (string) wp_parse_url( ( 0 === strpos( $src, '//' ) ? 'https:' : '' ) . html_entity_decode( $src, ENT_QUOTES | ENT_HTML5 ), PHP_URL_HOST ) );
					$this->add( $items, 'host:' . $host, $host, Classifier::guess( $src ), 'script', $host, $url, false, $source->detect( $s[1], $mm[0][1] ) );
				}
			}
		}

		// Google Fonts: copied to the site only when "Host Google Fonts on this site" is on.
		if ( preg_match_all( '#(?:href\s*=\s*["\']|@import\s+(?:url\()?\s*["\']?)((?:https?:)?//fonts\.googleapis\.com/css[^"\')\s]*)#i', $html, $fm ) ) {
			foreach ( array_unique( $fm[1] ) as $href ) {
				$local = LocalFonts::localize( $href, 15 );
				$this->add( $items, 'fonts:google', 'Google Fonts', 'necessary', $local ? 'fonts-local' : 'fonts-remote', 'fonts.googleapis.com', $url, true );
				if ( ! $local ) {
					$items['fonts:google']['kind'] = 'fonts-remote';
				}
			}
		}

		// Embeds.
		if ( preg_match_all( '#<iframe\b[^>]*\bsrc\s*=\s*["\']([^"\']+)#i', $html, $im ) ) {
			foreach ( $im[1] as $src ) {
				$src  = html_entity_decode( $src, ENT_QUOTES | ENT_HTML5 );
				$host = strtolower( (string) wp_parse_url( $src, PHP_URL_HOST ) );
				if ( ! $host || Classifier::same_site( $host ) || 'www.googletagmanager.com' === $host ) {
					continue;
				}
				if ( false !== strpos( $host, 'youtube' ) ) {
					$this->add( $items, 'embed:youtube', 'YouTube videos', 'necessary', 'embed', $host, $url, true );
				} elseif ( false !== strpos( $src, 'google.com/maps' ) || false !== strpos( $host, 'maps.google' ) ) {
					$this->add( $items, 'embed:gmaps', 'Google Maps', 'necessary', 'embed', $host, $url, true );
				} elseif ( false !== strpos( $host, 'vimeo' ) ) {
					$this->add( $items, 'embed:vimeo', 'Vimeo videos', 'necessary', 'embed', $host, $url, true );
				} else {
					$this->add( $items, 'embed:' . $host, $host, 'necessary', 'embed-other', $host, $url, true );
				}
			}
		}
		// Click-to-load placeholders keep the original iframe inside a <template>.
		if ( false !== strpos( $html, 'data-goconsent-embed="map"' ) ) {
			$this->add( $items, 'embed:gmaps', 'Google Maps', 'necessary', 'embed', 'www.google.com', $url, true );
		}

		// Google Maps API loaded from a data attribute or by a script (e.g. data-mlsg-maps-src).
		if ( preg_match( '#maps\.(?:googleapis|google)\.com/maps/api/js#i', $html ) ) {
			$this->add( $items, 'maps_js', 'Google Maps API (loads on click)', 'necessary', 'script', 'maps.googleapis.com', $url, true );
		}

		// Image pixels outside scripts (e.g. <noscript><img src="facebook.com/tr">).
		if ( preg_match_all( '#<img\b[^>]*\bsrc\s*=\s*["\']([^"\']+)#i', $html, $pm ) ) {
			foreach ( $pm[1] as $src ) {
				foreach ( Signatures::all() as $id => $sig ) {
					if ( 'necessary' === $sig[1] ) {
						continue;
					}
					foreach ( $sig[2] as $needle ) {
						if ( false !== stripos( $src, $needle ) ) {
							$this->add( $items, $id, $sig[0], $sig[1], 'pixel', $needle, $url, true );
							break 2;
						}
					}
				}
			}
		}
	}

	/**
	 * Add or extend a found item.
	 *
	 * @param array  $items    Items (by reference).
	 * @param string $id       Item id.
	 * @param string $label    Display label.
	 * @param string $category Default category.
	 * @param string $kind     script|pixel|embed|embed-other|fonts-local|fonts-remote.
	 * @param string $matched  What matched.
	 * @param string $url      Page URL.
	 * @param bool   $known    Whether it is a known service.
	 * @param array  $source   What put it on the page (see Source::detect()).
	 */
	private function add( array &$items, $id, $label, $category, $kind, $matched, $url, $known, $source = null ) {
		if ( ! isset( $items[ $id ] ) ) {
			$items[ $id ] = array(
				'label'   => $label,
				'default' => $category,
				'kind'    => $kind,
				'match'   => $matched,
				'pages'   => array(),
				'known'   => $known,
				'sources' => array(),
			);
		}
		// Keep up to three distinct sources (a tag can be added by more than one tool).
		if ( is_array( $source ) ) {
			$key = $source['type'] . '|' . $source['slug'] . '|' . $source['name'] . '|' . $source['detail'];
			if ( ! isset( $items[ $id ]['sources'][ $key ] ) && count( $items[ $id ]['sources'] ) < 3 ) {
				$items[ $id ]['sources'][ $key ] = $source;
			}
		}
		$path = (string) wp_parse_url( $url, PHP_URL_PATH );
		$path = '' === $path ? '/' : $path;
		if ( ! in_array( $path, $items[ $id ]['pages'], true ) ) {
			$items[ $id ]['pages'][] = $path;
		}
	}
}