<?php
if ( ! defined( 'ABSPATH' ) ) {
	exit;
}

// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- The searchable copy of the website is stored in the plugin's own tables, which have no WordPress API; it is rebuilt in batches, so caching would serve stale data.

/**
 * Reads the website so the chatbot can answer from it (no AI involved).
 *
 * Builds a searchable copy of every public page, split into short passages,
 * and collects question-and-answer pairs already written on the site (FAQ
 * headings, accordions, toggles, FAQ blocks). Works on any WordPress site.
 *
 * Each page is requested over HTTP with no cookies, exactly as a logged-out
 * visitor sees it. Pages that redirect (for example to /login/), return an
 * error status, are private, drafts, password-protected, excluded in the
 * settings, or restricted by a membership plugin are never indexed. Users,
 * orders, customers, bookings and form entries are never read.
 */
class Replyvane_Knowledge {

	const QUEUE     = 'replyvane_kb_queue';
	const LOCK      = 'replyvane_kb_lock';
	const MAX_CHUNK = 1400;

	/**
	 * Pages the chatbot has read (public pages only), with title and address.
	 *
	 * @return array<int, array{title:string, url:string}>
	 */
	public static function page_list( $limit = 40 ) {
		global $wpdb;
		$cache = wp_cache_get( 'page_list_' . (int) $limit . '_' . get_option( 'replyvane_kb_version', '0' ), 'replyvane' );
		if ( is_array( $cache ) ) {
			return $cache;
		}
		$rows = $wpdb->get_results( $wpdb->prepare( 'SELECT MIN(title) AS title, url FROM %i GROUP BY url ORDER BY MIN(title) ASC LIMIT %d', self::table(), (int) $limit ), ARRAY_A );
		$out  = array();
		foreach ( (array) $rows as $r ) {
			if ( '' !== trim( (string) $r['title'] ) ) {
				$out[] = array(
					'title' => (string) $r['title'],
					'url'   => (string) $r['url'],
				);
			}
		}
		wp_cache_set( 'page_list_' . (int) $limit . '_' . get_option( 'replyvane_kb_version', '0' ), $out, 'replyvane', HOUR_IN_SECONDS );
		return $out;
	}

	public static function table() {
		global $wpdb;
		return $wpdb->prefix . 'replyvane_content';
	}

	public static function harvest_table() {
		global $wpdb;
		return $wpdb->prefix . 'replyvane_harvest';
	}

	public static function create_tables() {
		global $wpdb;
		require_once ABSPATH . 'wp-admin/includes/upgrade.php';
		$charset = $wpdb->get_charset_collate();
		$content = self::table();
		$harvest = self::harvest_table();
		dbDelta(
			"CREATE TABLE {$content} (
			id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
			post_id bigint(20) unsigned NOT NULL DEFAULT 0,
			url varchar(500) NOT NULL DEFAULT '',
			title varchar(255) NOT NULL DEFAULT '',
			heading varchar(255) NOT NULL DEFAULT '',
			content text NOT NULL,
			hash char(32) NOT NULL DEFAULT '',
			source varchar(20) NOT NULL DEFAULT '',
			indexed_at datetime NULL DEFAULT NULL,
			PRIMARY KEY  (id),
			KEY post_id (post_id),
			KEY hash (hash),
			FULLTEXT KEY search (title,heading,content)
		) {$charset};"
		);
		dbDelta(
			"CREATE TABLE {$harvest} (
			id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
			post_id bigint(20) unsigned NOT NULL DEFAULT 0,
			url varchar(500) NOT NULL DEFAULT '',
			page_title varchar(255) NOT NULL DEFAULT '',
			question text NOT NULL,
			answer text NOT NULL,
			PRIMARY KEY  (id),
			KEY post_id (post_id)
		) {$charset};"
		);
	}

	/** When the kept-out pages change, remove them now and re-read the site. */
	public static function settings_changed( $old, $new ) {
		$before = array( is_array( $old ) && isset( $old['exclude'] ) ? $old['exclude'] : '', is_array( $old ) && isset( $old['exclude_ids'] ) ? (array) $old['exclude_ids'] : array() );
		$after  = array( is_array( $new ) && isset( $new['exclude'] ) ? $new['exclude'] : '', is_array( $new ) && isset( $new['exclude_ids'] ) ? (array) $new['exclude_ids'] : array() );
		sort( $before[1] );
		sort( $after[1] );
		if ( $before !== $after ) {
			// Uses the new settings, so kept-out pages are dropped immediately.
			add_filter( 'pre_option_' . Replyvane_Plugin::OPTION, function () use ( $new ) { return $new; } );
			self::start_full_reindex();
			self::bump_version();
		}
	}

	public static function hooks() {
		add_action( 'update_option_' . Replyvane_Plugin::OPTION, array( __CLASS__, 'settings_changed' ), 10, 2 );
		add_action( 'replyvane_kb_daily', array( __CLASS__, 'start_full_reindex' ) );
		add_action( 'replyvane_kb_process', array( __CLASS__, 'cron_process' ) );
		add_action( 'replyvane_kb_single', array( __CLASS__, 'index_single' ) );
		add_action( 'save_post', array( __CLASS__, 'on_save' ), 20, 2 );
		add_action( 'transition_post_status', array( __CLASS__, 'on_status_change' ), 10, 3 );
		add_action( 'before_delete_post', array( __CLASS__, 'remove_post' ) );
	}

	/* ------------------------------------------------------------------ */
	/* What counts as public                                               */
	/* ------------------------------------------------------------------ */

	public static function post_types() {
		$types = get_post_types( array( 'public' => true ), 'names' );
		unset( $types['attachment'], $types['elementor_library'] );
		return array_values( (array) apply_filters( 'replyvane_kb_post_types', $types ) );
	}

	/** Runs a callback as a logged-out visitor so no plugin reveals member content. */
	private static function as_visitor( $callback ) {
		$previous = get_current_user_id();
		wp_set_current_user( 0 );
		try {
			return call_user_func( $callback );
		} finally {
			wp_set_current_user( $previous );
		}
	}

	public static function public_ids() {
		$query = new WP_Query(
			array(
				'post_type'           => self::post_types(),
				'post_status'         => 'publish',
				'has_password'        => false,
				'posts_per_page'      => -1,
				'fields'              => 'ids',
				'no_found_rows'       => true,
				'orderby'             => 'ID',
				'order'               => 'ASC',
				'ignore_sticky_posts' => true,
			)
		);
		$ids = array_map( 'intval', $query->posts );
		return array_values( array_filter( $ids, array( __CLASS__, 'is_public' ) ) );
	}

	/** Addresses of login and account pages, always kept out. */
	public static function default_patterns() {
		return array( '/login/', '/my-account/', '/cart/', '/checkout/', '/account/', '/dashboard/', '/register/', '/wp-admin/', '/lost-password/' );
	}

	/**
	 * Why a published page is kept out automatically, or '' if it isn't.
	 * Shown next to pre-ticked, locked pages in Settings → Pages to keep out.
	 */
	public static function auto_reason( $post ) {
		$post = get_post( $post );
		if ( ! $post ) {
			return '';
		}
		// WooCommerce account, cart and checkout pages.
		if ( function_exists( 'wc_get_page_id' ) ) {
			foreach ( array( 'myaccount' => 'WooCommerce account page', 'cart' => 'WooCommerce cart', 'checkout' => 'WooCommerce checkout' ) as $page => $label ) {
				if ( (int) wc_get_page_id( $page ) === (int) $post->ID ) {
					return $label;
				}
			}
		}
		// Hidden products.
		if ( 'product' === $post->post_type && function_exists( 'wc_get_product' ) ) {
			$product = wc_get_product( $post->ID );
			if ( ! $product || ! $product->is_visible() ) {
				return 'Hidden from the shop';
			}
		}
		// Login and account addresses (checked by address and by the page's own
		// path, so it works with any permalink setting).
		$url = get_permalink( $post );
		$uri = get_page_uri( $post );
		if ( ( $url && self::is_excluded_url( $url ) ) || ( $uri && self::is_excluded_url( home_url( '/' . $uri . '/' ) ) ) ) {
			return 'Login or account page';
		}
		// Pages marked noindex in Yoast or Rank Math.
		if ( '1' === (string) get_post_meta( $post->ID, '_yoast_wpseo_meta-robots-noindex', true ) ) {
			return 'Hidden from search engines (Yoast)';
		}
		$rank_math = get_post_meta( $post->ID, 'rank_math_robots', true );
		if ( is_array( $rank_math ) && in_array( 'noindex', $rank_math, true ) ) {
			return 'Hidden from search engines (Rank Math)';
		}
		// Membership and content-restriction plugins, checked as a logged-out visitor.
		$current = get_current_user_id();
		wp_set_current_user( 0 );
		$members = ( function_exists( 'wc_memberships_is_post_content_restricted' ) && wc_memberships_is_post_content_restricted( $post->ID ) )
			|| ( function_exists( 'pmpro_has_membership_access' ) && ! pmpro_has_membership_access( $post->ID, 0 ) )
			|| ( function_exists( 'rcp_is_restricted_content' ) && rcp_is_restricted_content( $post->ID ) )
			|| ( class_exists( 'MeprRule' ) && method_exists( 'MeprRule', 'is_locked' ) && MeprRule::is_locked( $post ) );
		wp_set_current_user( $current );
		if ( $members ) {
			return 'Members only';
		}
		// Found to need a login (or to be unavailable) when the site was last read.
		if ( in_array( (int) $post->ID, array_map( 'intval', (array) get_option( 'replyvane_kb_restricted', array() ) ), true ) ) {
			return 'Needs a login to view';
		}
		return '';
	}

	public static function is_public( $id ) {
		$post = get_post( $id );
		if ( ! $post || 'publish' !== $post->post_status || '' !== (string) $post->post_password ) {
			return false;
		}
		if ( ! in_array( $post->post_type, self::post_types(), true ) ) {
			return false;
		}
		// Pages ticked in Settings → "Pages to keep out", and their sub-pages.
		$ticked = array_map( 'intval', (array) Replyvane_Plugin::get( 'exclude_ids' ) );
		if ( $ticked && array_intersect( array_merge( array( (int) $post->ID ), array_map( 'intval', get_post_ancestors( $post ) ) ), $ticked ) ) {
			return false;
		}
		// Kept out automatically (login pages, shop pages, noindex, members only).
		// "Needs a login" is decided by actually reading the page, so it is not
		// used here; otherwise such a page could never be re-checked.
		$reason = self::auto_reason( $post );
		if ( '' !== $reason && 'Needs a login to view' !== $reason ) {
			return false;
		}

		/**
		 * Last word on whether a post may be used by the assistant.
		 * Return false to keep a post out of the chatbot.
		 */
		return (bool) apply_filters( 'replyvane_kb_is_public', true, $post );
	}

	public static function is_excluded_url( $url ) {
		return self::matches_patterns( $url, self::default_patterns() );
	}

	/** True when the address starts with one of the patterns (e.g. "/members/"). */
	public static function matches_patterns( $url, array $patterns ) {
		$path = strtolower( untrailingslashit( (string) wp_parse_url( $url, PHP_URL_PATH ) ) );
		foreach ( $patterns as $rule ) {
			$rule = trim( (string) $rule );
			if ( '' === $rule ) {
				continue;
			}
			$rule_path = wp_parse_url( $rule, PHP_URL_PATH );
			$rule      = strtolower( untrailingslashit( $rule_path ? $rule_path : $rule ) );
			if ( '' === $rule ) {
				continue;
			}
			if ( $path === $rule || 0 === strpos( $path . '/', $rule . '/' ) ) {
				return true;
			}
		}
		return false;
	}

	/* ------------------------------------------------------------------ */
	/* Queue                                                               */
	/* ------------------------------------------------------------------ */

	public static function start_full_reindex() {
		$ids = self::as_visitor( array( __CLASS__, 'public_ids' ) );
		update_option( self::QUEUE, $ids, false );
		update_option( 'replyvane_kb_total', count( $ids ), false );
		self::purge_except( $ids );
		self::schedule_next( 5 );
		return count( $ids );
	}

	public static function cron_process() {
		$remaining = self::process_batch( 5 );
		if ( $remaining > 0 ) {
			self::schedule_next( 10 );
		}
	}

	private static function schedule_next( $delay ) {
		if ( ! wp_next_scheduled( 'replyvane_kb_process' ) ) {
			wp_schedule_single_event( time() + (int) $delay, 'replyvane_kb_process' );
		}
	}

	public static function remaining() {
		$queue = get_option( self::QUEUE, array() );
		return is_array( $queue ) ? count( $queue ) : 0;
	}

	/** Indexes up to $count queued posts. Returns how many are still queued. */
	public static function process_batch( $count ) {
		if ( get_transient( self::LOCK ) ) {
			return self::remaining();
		}
		set_transient( self::LOCK, 1, 2 * MINUTE_IN_SECONDS );

		$queue = get_option( self::QUEUE, array() );
		$queue = is_array( $queue ) ? $queue : array();
		$batch = array_splice( $queue, 0, (int) $count );
		update_option( self::QUEUE, $queue, false );

		foreach ( $batch as $id ) {
			self::index_post( (int) $id );
		}

		if ( empty( $queue ) ) {
			update_option( 'replyvane_kb_last', current_time( 'mysql' ), false );
			self::build_site_map();
			self::bump_version();
			delete_transient( self::LOCK );
			/** Fires when every page has been read, e.g. to create starter questions. */
			do_action( 'replyvane_kb_scan_complete' );
			return 0;
		}

		delete_transient( self::LOCK );
		return count( $queue );
	}

	/* ------------------------------------------------------------------ */
	/* Keeping the index fresh                                             */
	/* ------------------------------------------------------------------ */

	public static function on_save( $post_id, $post ) {
		if ( wp_is_post_revision( $post_id ) || wp_is_post_autosave( $post_id ) ) {
			return;
		}
		if ( ! in_array( $post->post_type, self::post_types(), true ) ) {
			return;
		}
		$args = array( (int) $post_id );
		if ( ! wp_next_scheduled( 'replyvane_kb_single', $args ) ) {
			// Wait a moment so caches and page builders finish saving.
			wp_schedule_single_event( time() + 30, 'replyvane_kb_single', $args );
		}
	}

	public static function index_single( $post_id ) {
		self::index_post( (int) $post_id );
		self::build_site_map();
		self::bump_version();
	}

	/** Cached answers are tied to this version, so they expire whenever content changes. */
	public static function bump_version() {
		update_option( 'replyvane_kb_version', (string) time(), false );
	}

	public static function on_status_change( $new_status, $old_status, $post ) {
		if ( 'publish' === $old_status && 'publish' !== $new_status ) {
			self::remove_post( $post->ID );
		}
	}

	public static function remove_post( $post_id ) {
		global $wpdb;
		$wpdb->delete( self::table(), array( 'post_id' => (int) $post_id ), array( '%d' ) );
		$wpdb->delete( self::harvest_table(), array( 'post_id' => (int) $post_id ), array( '%d' ) );
	}

	private static function purge_except( array $ids ) {
		global $wpdb;
		foreach ( array( self::table(), self::harvest_table() ) as $table ) {
			if ( empty( $ids ) ) {
				$wpdb->query( $wpdb->prepare( "DELETE FROM %i", $table ) );
				continue;
			}
			$ids = array_map( 'intval', $ids );
			$wpdb->query(
				$wpdb->prepare(
					'DELETE FROM %i WHERE post_id NOT IN (' . implode( ',', array_fill( 0, count( $ids ), '%d' ) ) . ')',
					array_merge( array( $table ), $ids )
				)
			);
		}
	}

	/* ------------------------------------------------------------------ */
	/* Indexing one page                                                   */
	/* ------------------------------------------------------------------ */

	public static function index_post( $post_id ) {
		return self::as_visitor(
			function () use ( $post_id ) {
				self::remove_post( $post_id );
				if ( ! self::is_public( $post_id ) ) {
					return 0;
				}
				$url   = get_permalink( $post_id );
				$title = html_entity_decode( wp_strip_all_tags( get_the_title( $post_id ) ), ENT_QUOTES, 'UTF-8' );

				$fetched = self::fetch_public_html( $url );
				$restricted = array_map( 'intval', (array) get_option( 'replyvane_kb_restricted', array() ) );
				$was        = in_array( (int) $post_id, $restricted, true );
				if ( 'restricted' === $fetched['status'] && ! $was ) {
					$restricted[] = (int) $post_id;
					update_option( 'replyvane_kb_restricted', $restricted, false );
				} elseif ( 'restricted' !== $fetched['status'] && $was ) {
					update_option( 'replyvane_kb_restricted', array_values( array_diff( $restricted, array( (int) $post_id ) ) ), false );
				}
				if ( 'restricted' === $fetched['status'] ) {
					return 0;
				}
				if ( 'ok' === $fetched['status'] ) {
					$html   = $fetched['html'];
					$source = 'rendered';
				} else {
					// The site could not request itself (loopback blocked). Render the
					// page content in PHP instead, still as a logged-out visitor.
					$html   = self::render_locally( $post_id );
					$source = 'database';
				}

				self::harvest( $post_id, $url, $title, $html );
				$sections = self::extract_sections( $html );
				self::harvest_headings( $post_id, $url, $title, $sections );
				if ( 'product' === get_post_type( $post_id ) ) {
					$facts = self::product_facts( $post_id );
					if ( $facts ) {
						array_unshift( $sections, array( 'heading' => $title, 'text' => $facts ) );
					}
				}

				return self::store( $post_id, $url, $title, self::chunk( $sections ), $source );
			}
		);
	}

	/**
	 * Requests a page with no cookies. Returns status "ok" with HTML,
	 * "restricted" when a visitor would not see it, or "error" when the
	 * request itself failed.
	 */
	private static function fetch_public_html( $url, $depth = 0 ) {
		$response = wp_remote_get(
			$url,
			array(
				'timeout'     => 25,
				'redirection' => 0,
				'cookies'     => array(),
				'sslverify'   => (bool) Replyvane_Plugin::get( 'sslverify' ),
				'user-agent'  => 'FAQ-Chatbot-Reader/' . REPLYVANE_VERSION . ' (+' . home_url( '/' ) . ')',
				'headers'     => array(
					'Accept'        => 'text/html',
					'Cache-Control' => 'no-cache',
				),
			)
		);

		if ( is_wp_error( $response ) ) {
			return array( 'status' => 'error' );
		}

		$code = (int) wp_remote_retrieve_response_code( $response );

		if ( $code >= 300 && $code < 400 ) {
			$location = (string) wp_remote_retrieve_header( $response, 'location' );
			if ( 0 === $depth && $location && self::same_page( $url, $location ) ) {
				return self::fetch_public_html( self::absolute_url( $location ), 1 );
			}
			return array( 'status' => 'restricted' ); // e.g. redirected to /login/.
		}

		if ( 200 === $code ) {
			$type = wp_remote_retrieve_header( $response, 'content-type' );
			$type = is_array( $type ) ? implode( ';', $type ) : (string) $type;
			if ( false === stripos( $type, 'html' ) ) {
				return array( 'status' => 'restricted' );
			}
			return array(
				'status' => 'ok',
				'html'   => (string) wp_remote_retrieve_body( $response ),
			);
		}

		// Firewalls and overloaded servers: treat as a failed request, not as private.
		if ( in_array( $code, array( 0, 403, 429 ), true ) || $code >= 500 ) {
			return array( 'status' => 'error' );
		}

		return array( 'status' => 'restricted' ); // 401, 404, 410 and similar.
	}

	private static function absolute_url( $location ) {
		if ( 0 === strpos( $location, '/' ) && 0 !== strpos( $location, '//' ) ) {
			return home_url( $location );
		}
		return $location;
	}

	/** True when a redirect only changes the scheme, www, or trailing slash. */
	private static function same_page( $a, $b ) {
		$b    = self::absolute_url( $b );
		$norm = function ( $url ) {
			$host = strtolower( (string) wp_parse_url( $url, PHP_URL_HOST ) );
			$host = preg_replace( '/^www\./', '', $host );
			$path = untrailingslashit( (string) wp_parse_url( $url, PHP_URL_PATH ) );
			return $host . $path;
		};
		return $norm( $a ) === $norm( $b );
	}

	private static function render_locally( $post_id ) {
		global $post;
		$original = $post;
		$post     = get_post( $post_id ); // phpcs:ignore WordPress.WP.GlobalVariablesOverride.Prohibited
		setup_postdata( $post );

		$html = '';
		if ( class_exists( '\Elementor\Plugin' ) && 'builder' === get_post_meta( $post_id, '_elementor_edit_mode', true ) ) {
			$html = \Elementor\Plugin::$instance->frontend->get_builder_content_for_display( $post_id );
		}
		if ( '' === trim( (string) $html ) ) {
			$html = apply_filters( 'the_content', $post->post_content ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- WordPress core filter, used to render page content.
		}

		$post = $original; // phpcs:ignore WordPress.WP.GlobalVariablesOverride.Prohibited
		wp_reset_postdata();

		return '<html><body><main><h1>' . esc_html( get_the_title( $post_id ) ) . '</h1>' . $html . '</main></body></html>';
	}

	private static function product_facts( $post_id ) {
		if ( ! function_exists( 'wc_get_product' ) ) {
			return '';
		}
		$product = wc_get_product( $post_id );
		if ( ! $product ) {
			return '';
		}
		$clean = function ( $html ) {
			return trim( preg_replace( '/\s+/u', ' ', html_entity_decode( wp_strip_all_tags( (string) $html ), ENT_QUOTES, 'UTF-8' ) ) );
		};

		$lines   = array( 'Product: ' . $clean( $product->get_name() ) );
		$price   = $clean( $product->get_price_html() );
		$cats    = $clean( wc_get_product_category_list( $post_id, ', ' ) );
		$summary = $clean( $product->get_short_description() );

		if ( $price ) {
			$lines[] = 'Price: ' . $price;
		}
		if ( $cats ) {
			$lines[] = 'Category: ' . $cats;
		}
		$lines[] = 'Availability: ' . ( $product->is_in_stock() ? 'In stock' : 'Out of stock' );
		if ( $summary ) {
			$lines[] = 'Summary: ' . $summary;
		}
		$lines[] = 'Product page: ' . get_permalink( $post_id );

		return implode( "\n", $lines );
	}

	/* ------------------------------------------------------------------ */
	/* Question-and-answer pairs already written on the site               */
	/* ------------------------------------------------------------------ */

	/** Plain text of a node, keeping line breaks between blocks. */
	private static function node_text( $node ) {
		$out    = '';
		$blocks = array( 'p', 'div', 'li', 'br', 'tr', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'section', 'dd', 'dt', 'blockquote' );
		foreach ( $node->childNodes as $child ) {
			if ( XML_TEXT_NODE === $child->nodeType ) {
				$out .= $child->nodeValue;
			} elseif ( XML_ELEMENT_NODE === $child->nodeType ) {
				$tag = strtolower( $child->nodeName );
				if ( in_array( $tag, array( 'script', 'style', 'svg', 'button', 'form' ), true ) ) {
					continue;
				}
				$inner = self::node_text( $child );
				if ( 'li' === $tag ) {
					$inner = '- ' . ltrim( $inner );
				}
				$out .= in_array( $tag, $blocks, true ) ? "\n" . $inner . "\n" : $inner;
			}
		}
		return $out;
	}

	private static function clean_text( $text ) {
		$lines = array();
		foreach ( preg_split( '/\n+/', (string) $text ) as $line ) {
			$line = trim( preg_replace( '/[ \t\x{00A0}]+/u', ' ', $line ) );
			if ( '' !== $line && '-' !== $line ) {
				$lines[] = $line;
			}
		}
		return implode( "\n", $lines );
	}

	/**
	 * Finds FAQ-style blocks: <details>/<summary>, Elementor accordions and
	 * toggles, and Yoast / Rank Math FAQ blocks. Skips site headers, footers
	 * and popups.
	 */
	private static function harvest( $post_id, $url, $title, $html ) {
		if ( ! class_exists( 'DOMDocument' ) || '' === trim( (string) $html ) ) {
			return;
		}
		$previous = libxml_use_internal_errors( true );
		$dom      = new DOMDocument();
		$dom->loadHTML( '<?xml encoding="UTF-8">' . $html );
		libxml_clear_errors();
		libxml_use_internal_errors( $previous );
		$xp = new DOMXPath( $dom );

		$skip  = 'ancestor::*[@data-elementor-type="header" or @data-elementor-type="footer" or @data-elementor-type="popup"] or ancestor::header or ancestor::footer or ancestor::nav';
		$class = function ( $name ) {
			return 'contains(concat(" ", normalize-space(@class), " "), " ' . $name . ' ")';
		};
		$pairs = array();

		// <details><summary>Question</summary>Answer</details> (also Elementor's newer accordion).
		foreach ( $xp->query( '//details[not(' . $skip . ')]' ) as $d ) {
			$summary = $xp->query( './summary', $d )->item( 0 );
			if ( ! $summary ) {
				continue;
			}
			$q = self::clean_text( $summary->textContent );
			$d->removeChild( $summary );
			$pairs[] = array( $q, self::clean_text( self::node_text( $d ) ) );
		}
		// Elementor accordion / toggle (classic widgets).
		foreach ( $xp->query( '//*[(' . $class( 'elementor-accordion-item' ) . ' or ' . $class( 'elementor-toggle-item' ) . ') and not(' . $skip . ')]' ) as $item ) {
			$t = $xp->query( './/*[' . $class( 'elementor-tab-title' ) . ']', $item )->item( 0 );
			$c = $xp->query( './/*[' . $class( 'elementor-tab-content' ) . ']', $item )->item( 0 );
			if ( $t && $c ) {
				$pairs[] = array( self::clean_text( $t->textContent ), self::clean_text( self::node_text( $c ) ) );
			}
		}
		// Yoast FAQ block and Rank Math FAQ block.
		foreach ( array( array( 'schema-faq-section', 'schema-faq-question', 'schema-faq-answer' ), array( 'rank-math-list-item', 'rank-math-question', 'rank-math-answer' ) ) as $cfg ) {
			foreach ( $xp->query( '//*[' . $class( $cfg[0] ) . ' and not(' . $skip . ')]' ) as $item ) {
				$t = $xp->query( './/*[' . $class( $cfg[1] ) . ']', $item )->item( 0 );
				$c = $xp->query( './/*[' . $class( $cfg[2] ) . ']', $item )->item( 0 );
				if ( $t && $c ) {
					$pairs[] = array( self::clean_text( $t->textContent ), self::clean_text( self::node_text( $c ) ) );
				}
			}
		}
		self::save_pairs( $post_id, $url, $title, $pairs );
	}

	/** Headings phrased as questions ("How long does it take?") with the text under them. */
	private static function harvest_headings( $post_id, $url, $title, array $sections ) {
		$pairs = array();
		foreach ( $sections as $section ) {
			$parts = explode( ' › ', (string) $section['heading'] );
			$last  = trim( end( $parts ) );
			if ( '?' === mb_substr( $last, -1 ) ) {
				// Drop call-to-action link addresses such as "Book now [https://…]".
				$pairs[] = array( $last, preg_replace( '/\s*\[https?:[^\]]+\]/', '', $section['text'] ) );
			}
		}
		self::save_pairs( $post_id, $url, $title, $pairs );
	}

	private static function save_pairs( $post_id, $url, $title, array $pairs ) {
		global $wpdb;
		$seen = array();
		foreach ( $pairs as $pair ) {
			list( $q, $a ) = $pair;
			$q = trim( preg_replace( '/\s+/u', ' ', (string) $q ) );
			$a = trim( (string) $a );
			$len_q = mb_strlen( $q );
			if ( $len_q < 8 || $len_q > 200 || mb_strlen( $a ) < 15 ) {
				continue;
			}
			$key = mb_strtolower( $q );
			if ( isset( $seen[ $key ] ) ) {
				continue;
			}
			$seen[ $key ] = true;
			$exists = $wpdb->get_var( $wpdb->prepare( 'SELECT id FROM %i WHERE post_id = %d AND question = %s', self::harvest_table(), $post_id, $q ) );
			if ( $exists ) {
				continue;
			}
			$wpdb->insert(
				self::harvest_table(),
				array(
					'post_id'    => (int) $post_id,
					'url'        => mb_substr( $url, 0, 500 ),
					'page_title' => mb_substr( $title, 0, 250 ),
					'question'   => $q,
					'answer'     => mb_substr( $a, 0, 3000 ),
				),
				array( '%d', '%s', '%s', '%s', '%s' )
			);
		}
	}

	/* ------------------------------------------------------------------ */
	/* HTML to text                                                        */
	/* ------------------------------------------------------------------ */

	/**
	 * Turns a page into sections of plain text, each labelled with its
	 * heading path (e.g. "Our Services › Comprehensive Scan"). Site header,
	 * footer, menus, forms, popups and scripts are removed.
	 */
	public static function extract_sections( $html ) {
		$html = (string) $html;
		if ( '' === trim( $html ) || ! class_exists( 'DOMDocument' ) ) {
			$text = trim( preg_replace( '/\s+/u', ' ', wp_strip_all_tags( $html ) ) );
			return $text ? array( array( 'heading' => '', 'text' => $text ) ) : array();
		}

		$previous = libxml_use_internal_errors( true );
		$dom      = new DOMDocument();
		$dom->loadHTML( '<?xml encoding="UTF-8">' . $html );
		libxml_clear_errors();
		libxml_use_internal_errors( $previous );
		$xpath = new DOMXPath( $dom );

		$noise = array(
			'//script', '//style', '//noscript', '//svg', '//iframe', '//template', '//form',
			'//select', '//button', '//header', '//footer', '//nav', '//aside',
			'//*[@data-elementor-type="header"]',
			'//*[@data-elementor-type="footer"]',
			'//*[@data-elementor-type="popup"]',
			'//*[contains(concat(" ", normalize-space(@class), " "), " screen-reader-text ")]',
			'//*[contains(concat(" ", normalize-space(@class), " "), " elementor-screen-only ")]',
			'//*[contains(concat(" ", normalize-space(@class), " "), " site-header ")]',
			'//*[contains(concat(" ", normalize-space(@class), " "), " site-footer ")]',
			'//*[contains(concat(" ", normalize-space(@class), " "), " elementor-menu-cart__container ")]',
			'//*[contains(concat(" ", normalize-space(@class), " "), " woocommerce-notices-wrapper ")]',
		);
		foreach ( $noise as $query ) {
			$nodes = $xpath->query( $query );
			if ( ! $nodes ) {
				continue;
			}
			$remove = array();
			foreach ( $nodes as $node ) {
				$remove[] = $node;
			}
			foreach ( $remove as $node ) {
				if ( $node->parentNode ) {
					$node->parentNode->removeChild( $node );
				}
			}
		}

		$root = $xpath->query( '//main' )->item( 0 );
		if ( ! $root ) {
			$root = $xpath->query( '//*[@id="content"]' )->item( 0 );
		}
		if ( ! $root ) {
			$root = $dom->getElementsByTagName( 'body' )->item( 0 );
		}
		if ( ! $root ) {
			return array();
		}

		// Keep short internal call-to-action links, e.g. "JOIN NOW [https://…]".
		$home_host = preg_replace( '/^www\./', '', strtolower( (string) wp_parse_url( home_url(), PHP_URL_HOST ) ) );
		$anchors   = array();
		foreach ( $root->getElementsByTagName( 'a' ) as $a ) {
			$anchors[] = $a;
		}
		foreach ( $anchors as $a ) {
			$href = trim( (string) $a->getAttribute( 'href' ) );
			$text = trim( preg_replace( '/\s+/u', ' ', $a->textContent ) );
			if ( '' === $href || '#' === $href[0] || '' === $text || mb_strlen( $text ) > 40 ) {
				continue;
			}
			$href = self::absolute_url( $href );
			$host = preg_replace( '/^www\./', '', strtolower( (string) wp_parse_url( $href, PHP_URL_HOST ) ) );
			if ( $host !== $home_host || 0 === stripos( $href, 'mailto:' ) ) {
				continue;
			}
			$a->appendChild( $dom->createTextNode( ' [' . $href . ']' ) );
		}

		// Mark headings and block boundaries so the text keeps its structure.
		$elements = array();
		foreach ( $root->getElementsByTagName( '*' ) as $el ) {
			$elements[] = $el;
		}
		$blocks = array( 'p', 'div', 'section', 'article', 'li', 'ul', 'ol', 'table', 'tr', 'td', 'th', 'blockquote', 'dd', 'dt', 'figcaption', 'h5', 'h6', 'details', 'summary' );
		foreach ( $elements as $el ) {
			$tag = strtolower( $el->nodeName );
			// Collapsible FAQ questions (<summary>, accordion and toggle titles,
			// FAQ block questions) are treated as headings, so each answer is
			// filed under its own question.
			$is_question = 'summary' === $tag || preg_match( '/(^|\s)(elementor-tab-title|e-n-accordion-item-title|schema-faq-question|rank-math-question)(\s|$)/', (string) $el->getAttribute( 'class' ) );
			if ( $is_question ) {
				$tag = 'h3';
			}
			if ( preg_match( '/^h([1-4])$/', $tag, $m ) ) {
				$text = trim( preg_replace( '/\s+/u', ' ', $el->textContent ) );
				while ( $el->firstChild ) {
					$el->removeChild( $el->firstChild );
				}
				$el->appendChild( $dom->createTextNode( "\n\x01" . $m[1] . '|' . $text . "\n" ) );
			} elseif ( 'br' === $tag ) {
				if ( $el->parentNode ) {
					$el->parentNode->insertBefore( $dom->createTextNode( "\n" ), $el );
				}
			} elseif ( in_array( $tag, $blocks, true ) ) {
				$el->appendChild( $dom->createTextNode( "\n" ) );
			}
		}

		$lines    = preg_split( '/\n+/', $root->textContent );
		$sections = array();
		$stack    = array();
		$current  = array( 'heading' => '', 'lines' => array() );

		$flush = function () use ( &$sections, &$current ) {
			if ( ! empty( $current['lines'] ) ) {
				$sections[] = array(
					'heading' => $current['heading'],
					'text'    => implode( "\n", $current['lines'] ),
				);
			}
		};

		foreach ( $lines as $line ) {
			$line = trim( preg_replace( '/[ \t\x{00A0}]+/u', ' ', $line ) );
			if ( '' === $line ) {
				continue;
			}
			if ( "\x01" === $line[0] ) {
				$parts = explode( '|', substr( $line, 1 ), 2 );
				$level = (int) $parts[0];
				$text  = isset( $parts[1] ) ? trim( $parts[1] ) : '';
				if ( '' === $text ) {
					continue;
				}
				$flush();
				foreach ( array_keys( $stack ) as $lvl ) {
					if ( $lvl >= $level ) {
						unset( $stack[ $lvl ] );
					}
				}
				$stack[ $level ] = $text;
				ksort( $stack );
				$current = array(
					'heading' => implode( ' › ', $stack ),
					'lines'   => array(),
				);
				continue;
			}
			if ( mb_strlen( $line ) < 2 ) {
				continue;
			}
			if ( end( $current['lines'] ) === $line ) {
				continue;
			}
			$current['lines'][] = $line;
		}
		$flush();

		return $sections;
	}

	/** Groups sections into chunks of roughly MAX_CHUNK characters. */
	public static function chunk( array $sections ) {
		$chunks  = array();
		$buffer  = '';
		$heading = '';

		$flush = function () use ( &$chunks, &$buffer, &$heading ) {
			if ( '' !== trim( $buffer ) ) {
				$chunks[] = array(
					'heading' => $heading,
					'content' => trim( $buffer ),
				);
			}
			$buffer  = '';
			$heading = '';
		};

		foreach ( $sections as $section ) {
			$label = $section['heading'] ? '## ' . $section['heading'] . "\n" : '';
			$block = $label . $section['text'];

			if ( mb_strlen( $block ) > self::MAX_CHUNK ) {
				$flush();
				$piece = $label;
				foreach ( explode( "\n", $section['text'] ) as $line ) {
					if ( mb_strlen( $piece ) + mb_strlen( $line ) > self::MAX_CHUNK && trim( $piece ) !== trim( $label ) ) {
						$chunks[] = array( 'heading' => $section['heading'], 'content' => trim( $piece ) );
						$piece    = $label . '(continued)' . "\n";
					}
					$piece .= mb_substr( $line, 0, self::MAX_CHUNK ) . "\n";
				}
				if ( trim( $piece ) !== trim( $label ) ) {
					$chunks[] = array( 'heading' => $section['heading'], 'content' => trim( $piece ) );
				}
				continue;
			}

			if ( mb_strlen( $buffer ) + mb_strlen( $block ) > self::MAX_CHUNK ) {
				$flush();
			}
			if ( '' === $buffer ) {
				$heading = $section['heading'];
			}
			$buffer .= ( '' === $buffer ? '' : "\n\n" ) . $block;
		}
		$flush();

		return $chunks;
	}

	private static function store( $post_id, $url, $title, array $chunks, $source ) {
		global $wpdb;
		$table  = self::table();
		$stored = 0;

		foreach ( $chunks as $chunk ) {
			$hash = md5( strtolower( preg_replace( '/\s+/u', ' ', $chunk['content'] ) ) );

			// Skip blocks repeated across pages (shared banners, reused sections).
			$exists = $wpdb->get_var(
				$wpdb->prepare( "SELECT id FROM %i WHERE hash = %s AND post_id <> %d LIMIT 1", $table, $hash, $post_id )
			);
			if ( $exists ) {
				continue;
			}

			$wpdb->insert(
				$table,
				array(
					'post_id'    => (int) $post_id,
					'url'        => mb_substr( $url, 0, 500 ),
					'title'      => mb_substr( $title, 0, 250 ),
					'heading'    => mb_substr( (string) $chunk['heading'], 0, 250 ),
					'content'    => $chunk['content'],
					'hash'       => $hash,
					'source'     => $source,
					'indexed_at' => current_time( 'mysql' ),
				),
				array( '%d', '%s', '%s', '%s', '%s', '%s', '%s', '%s' )
			);
			$stored++;
		}
		return $stored;
	}

	/** A short list of page titles and links, given to the assistant for navigation help. */
	public static function build_site_map() {
		global $wpdb;
		$table = self::table();
		$rows  = $wpdb->get_results( $wpdb->prepare( "SELECT url, MIN(title) AS title FROM %i GROUP BY url ORDER BY MIN(id) ASC LIMIT 80", $table ), ARRAY_A );
		update_option( 'replyvane_kb_site_map', $rows ? $rows : array(), false );
	}

	public static function stats() {
		global $wpdb;
		$table = self::table();
		return array(
			'pages'        => (int) $wpdb->get_var( $wpdb->prepare( "SELECT COUNT(DISTINCT post_id) FROM %i", $table )),
			'chunks'       => (int) $wpdb->get_var( $wpdb->prepare( "SELECT COUNT(*) FROM %i", $table )),
			'remaining'    => self::remaining(),
			'total'        => (int) get_option( 'replyvane_kb_total', 0 ),
			'last_indexed' => (string) get_option( 'replyvane_kb_last', '' ),
		);
	}
}
// phpcs:enable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching