store = $store; } /** * Returns the module slug. * * @since 1.1.0 * * @return string Module slug. */ public function slug() { return self::SLUG; } /** * Attaches the module's WordPress hooks. * * All three hooks are attached unconditionally and the enable setting is * read inside the logging callback instead. Two reasons: * * - Hooks are registered once per request, during bootstrap. Deciding there * would freeze the answer for the whole request, so a test - or an admin * saving the setting - could not observe the module stopping and resuming * without a new request. * - Retention cleanup and the aggregates must keep working while logging is * off. Existing rows stay owned by the retention policy, so switching * collection off must not strand data the site owner expects to expire. * * @since 1.1.0 * * @return void */ public function register_hooks() { add_action( 'template_redirect', array( $this, 'maybe_log_visit' ), 1 ); add_action( Activator::CRON_CLEANUP_LOGS, array( $this, 'cleanup' ) ); add_filter( 'aive_dashboard_stats', array( $this, 'add_dashboard_stats' ) ); } /** * Records the current request when it is an eligible AI crawler visit. * * Every rejection is silent and side-effect free: this runs on a live * frontend request, so a malformed header or a database fault must change * nothing about the response the visitor receives. * * @since 1.1.0 * * @return bool True when a row was recorded. */ public function maybe_log_visit() { if ( ! Helpers::is_module_enabled( self::SLUG ) ) { return false; } if ( ! $this->is_loggable_request() ) { return false; } $post = $this->resolve_published_post(); if ( ! $post instanceof WP_Post ) { return false; } $user_agent = $this->read_user_agent(); if ( null === $user_agent ) { return false; } // The registry is the only thing that may identify a crawler. See the class comment. $crawler = Helpers::detect_ai_bot( $user_agent ); if ( null === $crawler ) { return false; } $stored_user_agent = $this->prepare_user_agent( $user_agent ); if ( null === $stored_user_agent ) { return false; } /* * Probed last, so the cost - one `SHOW COLUMNS` round trip per request * that reaches it - falls only on requests that are about to produce a * row, not on ordinary page views. See * Database::crawler_log_is_canonical() for why a database the migration * has not finished with must record nothing at all rather than a row * with no canonical instant. */ if ( ! Database::crawler_log_is_canonical() ) { return false; } /* * One clock reading, two renderings of it. Calling the clock twice could * straddle a second boundary and leave the row's two columns describing * instants a second apart, one of which would then be the authority and * the other a visibly disagreeing display value. */ $now = current_datetime(); return $this->store->insert( array( 'bot_name' => $this->limit( $crawler['name'], self::MAX_BOT_NAME ), 'bot_type' => $this->limit( $crawler['type'], self::MAX_BOT_TYPE ), 'request_uri' => $this->read_request_path(), 'request_method' => 'GET', 'user_agent' => $stored_user_agent, 'ip_address' => $this->read_client_ip(), 'visited_at' => $now->format( 'Y-m-d H:i:s' ), 'visited_at_utc' => self::utc_string( $now ), 'post_id' => $post->ID, ) ); } /** * Deletes rows that have outlived the configured retention period. * * Runs on the `aive_cleanup_crawler_logs` daily event. The cutoff is exactly * `retention_days` x 24 hours of elapsed time before now, expressed as a UTC * instant, and deletion is strictly older than it, so a row landing on the * boundary survives. Rows at or after the cutoff are untouched and no other * AIVE table is read or written, which makes repeated runs a no-op. * * Declines to run before the canonical column exists: an elapsed-time window * has nothing to compare against until then, and deleting on the strength of * the local column is what this increment exists to stop. * * @since 1.1.0 * * @return int|null Rows deleted, or null when the query failed or the schema is not ready. */ public function cleanup() { if ( ! Database::crawler_log_is_canonical() ) { return null; } return $this->store->delete_older_than( self::retention_cutoff() ); } /** * Adds the crawler aggregates to the shared dashboard stats array. * * The two keys are published unconditionally, including while logging is * disabled, because they describe the rows that exist rather than whether * collection is running. A consumer that needs to present "unavailable" * reads `enable_crawler_log` for itself; this module does not decide that on * its behalf. A failed query contributes zero rather than a null that every * consumer would have to re-check. * * @since 1.1.0 * * @param array|mixed $stats Stats collected so far. * @return array Stats including this module's keys. */ public function add_dashboard_stats( $stats ) { if ( ! is_array( $stats ) ) { $stats = array(); } $total = null; $unique = null; // Zeroes rather than a count read from a column that may not be there yet. if ( Database::crawler_log_is_canonical() ) { $since = self::stats_window_start(); $total = $this->store->count_since( $since ); $unique = $this->store->count_distinct_bots_since( $since ); } $stats['crawler_total_visits_7d'] = null === $total ? 0 : $total; $stats['crawler_unique_bots_7d'] = null === $unique ? 0 : $unique; return $stats; } /** * Returns the effective retention period in whole days. * * Clamped on read as well as on save. The settings sanitizer cannot be the * only guard: it runs on `admin_init`, so a value written programmatically * outside an admin request never passes through it, and the retention window * decides what gets deleted. * * @since 1.1.0 * * @return int Retention period, between RETENTION_MIN and RETENTION_MAX. */ public static function retention_days() { return self::sanitize_retention_days( Helpers::get_setting( 'log_retention_days', self::RETENTION_DEFAULT ) ); } /** * Reduces any submitted value to a retention period inside the allowed range. * * Pure logic with no WordPress dependency, so the Unit suite exercises it * directly. * * The contract fixes two behaviours that need reconciling. Decimal input * "follows absint() semantics", and a nonnumeric, negative, zero or * out-of-range submission "becomes the nearest integer bound". Taking * absint()'s absolute value would send -5 to 5 - an in-range value, so no * bound would apply and the second rule would never fire for a negative * number. The integer cast that absint() performs is therefore applied, and * the absolute value is not: 12.7 truncates to 12 as specified, while -5, 0 * and "abc" all reach the nearest bound, RETENTION_MIN. * * A non-scalar is not a submission shape at all - no integer can be derived * from it and nothing about it indicates which bound is "nearest" - so the * documented default stands rather than the most destructive bound. * * @since 1.1.0 * * @param mixed $value Submitted value. * @return int Retention period, between RETENTION_MIN and RETENTION_MAX. */ public static function sanitize_retention_days( $value ) { if ( ! is_scalar( $value ) ) { return self::RETENTION_DEFAULT; } $days = (int) $value; return max( self::RETENTION_MIN, min( self::RETENTION_MAX, $days ) ); } /** * Returns the retention cutoff as a canonical UTC datetime string. * * Exactly `retention_days` x 24 hours of elapsed time before now. See * `minus_days_utc()` for why that is not the same as the same wall-clock * time that many calendar days ago. * * @since 1.1.0 * * @return string UTC `Y-m-d H:i:s` cutoff. */ public static function retention_cutoff() { return self::minus_days_utc( current_datetime(), self::retention_days() ); } /** * Returns the lower bound of the seven-day reporting window. * * @since 1.1.0 * * @return string UTC `Y-m-d H:i:s` lower bound. */ public static function stats_window_start() { return self::minus_days_utc( current_datetime(), self::STATS_WINDOW_DAYS ); } /** * Returns the instant a whole number of 24-hour days before the one given, * as a UTC datetime string. * * Two separate things make this correct, and both are needed. * * The subtraction is done on the Unix timestamp, because both windows are * defined as absolute elapsed time: `n` x 24 hours, not the same wall-clock * reading `n` calendar days ago. `DateTimeImmutable::modify()` with a * relative number of days, like a `PD` interval, moves the local clock * fields and then re-resolves the offset, so in a timezone that observes * daylight saving it is an hour short across the spring transition - * deleting rows early - and an hour long across the autumn one. * * The result is then rendered in UTC, because that is the basis rows are * compared in. Rendering a correct instant as a site-local string would * undo the arithmetic inside a fall-back hour: the cutoff instant and an * older row can render to local strings whose order is the reverse of the * instants' order, so the comparison would keep what it should delete. * * `now` is a parameter rather than read here so that the arithmetic can be * verified at a fixed transition-crossing instant, not only at whatever * date the tests happen to run on. * * @since 1.1.0 * * @param DateTimeImmutable $now Instant to measure back from. * @param int $days Whole 24-hour days to subtract. * @return string UTC `Y-m-d H:i:s` datetime. */ public static function minus_days_utc( DateTimeImmutable $now, $days ) { $seconds = (int) $days * DAY_IN_SECONDS; return self::utc_string( $now->setTimestamp( $now->getTimestamp() - $seconds ) ); } /** * Renders an instant as a canonical UTC datetime string. * * The one place a `DateTimeImmutable` becomes a storable or comparable * canonical value, so "canonical means UTC" is asserted once. * * @since 1.1.0 * * @param DateTimeImmutable $instant Instant to render. * @return string UTC `Y-m-d H:i:s` datetime. */ public static function utc_string( DateTimeImmutable $instant ) { return $instant->setTimezone( new DateTimeZone( 'UTC' ) )->format( 'Y-m-d H:i:s' ); } /** * Reports whether the current request may be logged at all. * * Checked before the User-Agent is read, so a request this module has no * business recording costs nothing beyond these tests. The order runs * cheapest-and-broadest first: request context, then method, then the * WordPress query conditionals, then the request path. * * @since 1.1.0 * * @return bool True when the request is a frontend content GET. */ private function is_loggable_request() { if ( is_admin() || wp_doing_cron() || wp_doing_ajax() || wp_is_json_request() ) { return false; } if ( defined( 'REST_REQUEST' ) && REST_REQUEST ) { return false; } if ( 'GET' !== $this->read_request_method() ) { return false; } /* * These are WordPress's own answers about what it resolved the request * to, which is why the check happens on template_redirect rather than * earlier: a path test alone cannot tell a feed or a 404 from a page. */ if ( is_feed() || is_robots() || is_404() || is_preview() ) { return false; } // Core serves its sitemaps through a query var rather than a conditional tag. if ( '' !== (string) get_query_var( 'sitemap', '' ) || '' !== (string) get_query_var( 'sitemap-stylesheet', '' ) ) { return false; } return $this->is_loggable_path(); } /** * Reports whether the request path is one this module records. * * Only the path component is examined. A query string cannot turn an * eligible path into an excluded one - `/?s=sitemap` is a search request, * not a sitemap - and reading the query here would hand any visitor control * over whether their visit is recorded. * * Matched against the full path rather than the 500-character value that * gets stored, so a long path cannot push `sitemap` past the storage bound * and out of reach of the exclusion. * * @since 1.1.0 * * @return bool True when the path is loggable. */ private function is_loggable_path() { $path = strtolower( $this->raw_request_path() ); if ( false !== strpos( $path, '/wp-json/' ) ) { return false; } if ( false !== strpos( $path, 'robots.txt' ) ) { return false; } if ( false !== strpos( $path, 'sitemap' ) ) { return false; } return ! str_ends_with( $path, '.xml' ); } /** * Returns the request method, uppercased. * * Uppercased and nothing else. HTTP method names are case-sensitive tokens * with no surrounding whitespace, so a padded value is not a GET that needs * tidying - it is a request this module declines to interpret. A missing or * non-scalar value yields an empty string, which no comparison accepts, so * an unusual server environment fails closed the same way. * * @since 1.1.0 * * @return string Uppercased request method, or an empty string. */ private function read_request_method() { if ( ! isset( $_SERVER['REQUEST_METHOD'] ) || ! is_scalar( $_SERVER['REQUEST_METHOD'] ) ) { return ''; } return strtoupper( (string) wp_unslash( $_SERVER['REQUEST_METHOD'] ) ); // phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized -- Compared against a fixed literal and never stored or rendered. } /** * Returns the sanitised path component of the request, unbounded. * * The query string and fragment are dropped rather than sanitised away: a * query string can carry search terms, session identifiers, tokens or * personal data, none of which this module has a reason to retain. What is * left is run through `esc_url_raw()`, which removes control characters and * whitespace while leaving legitimate percent-encoding intact - * `sanitize_text_field()` would strip every `%XX` sequence and silently * mangle a non-ASCII path. * * @since 1.1.0 * * @return string Request path, defaulting to `/`. */ private function raw_request_path() { if ( ! isset( $_SERVER['REQUEST_URI'] ) || ! is_scalar( $_SERVER['REQUEST_URI'] ) ) { return '/'; } $raw = (string) wp_unslash( $_SERVER['REQUEST_URI'] ); // phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized -- Reduced to its path and sanitised below. $path = wp_parse_url( $raw, PHP_URL_PATH ); if ( ! is_string( $path ) || '' === $path ) { return '/'; } $path = $this->scrub_utf8( esc_url_raw( $path ) ); if ( '' === $path || ! str_starts_with( $path, '/' ) ) { /* * Anything that does not survive as an absolute path is reported as * the site root rather than dropped: the visit is real and the bound * the contract sets for an unparsable URI is "/", not "no row". */ return '/'; } return $path; } /** * Returns the request path bounded to its column length. * * @since 1.1.0 * * @return string Storable request path. */ private function read_request_path() { return $this->limit( $this->raw_request_path(), self::MAX_REQUEST_URI ); } /** * Returns the raw User-Agent header, or null when there is nothing usable. * * This is the value matched against the registry, so it is deliberately not * sanitised here - a sanitiser could change the token and so change which * crawler matched. `prepare_user_agent()` bounds the copy that is stored. * * @since 1.1.0 * * @return string|null Raw header value, or null. */ private function read_user_agent() { if ( ! isset( $_SERVER['HTTP_USER_AGENT'] ) || ! is_scalar( $_SERVER['HTTP_USER_AGENT'] ) ) { return null; } $user_agent = (string) wp_unslash( $_SERVER['HTTP_USER_AGENT'] ); // phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized -- Matched against the registry as sent; the stored copy is normalised by prepare_user_agent(). return '' === trim( $user_agent ) ? null : $user_agent; } /** * Bounds and normalises the User-Agent header for storage. * * Control characters - CR, LF, NUL and the rest of the C0 range, plus DEL - * become spaces. They have no meaning in a header value and a stored CR/LF * pair is the shape that turns a later log line or header into two. Invalid * UTF-8 is then stripped, because a byte sequence the column's charset * cannot represent would otherwise be truncated at the bad byte by $wpdb or * rejected outright by the database. * * The value is never rendered unescaped; the admin tables run every stored * field through `esc_html()`. * * @since 1.1.0 * * @param string $user_agent Raw header value. * @return string|null Storable value, or null when nothing usable remains. */ private function prepare_user_agent( $user_agent ) { /* * Byte-wise on purpose: the /u modifier would make preg_replace() return * null for a header that is not valid UTF-8, and that is exactly the * input this needs to handle. No multibyte character contains a byte in * the C0 range, so replacing those bytes cannot split one. */ $normalized = preg_replace( '/[\x00-\x1F\x7F]/', ' ', $user_agent ); if ( ! is_string( $normalized ) ) { return null; } $normalized = trim( $this->scrub_utf8( $normalized ) ); if ( '' === $normalized ) { return null; } return $this->limit( $normalized, self::MAX_USER_AGENT ); } /** * Replaces byte sequences that are not valid UTF-8. * * A header or request target arrives as bytes and carries no encoding * guarantee, while the log columns are utf8mb4. Left alone, an invalid * sequence is either truncated at the bad byte by $wpdb or rejected by the * database, so a crafted header could drop the row - or the part of it that * made the request worth recording. * * `wp_check_invalid_utf8( $value, true )` is the WordPress helper for this * and is deliberately not used: with stripping on it calls * `iconv( 'utf-8', 'utf-8', ... )`, which raises an E_NOTICE * ("Detected an illegal character in input string") on exactly the input it * exists to clean. That notice would land on a live frontend request, which * the contract rules out, and on a site with display_errors on it would be * printed into the response. `mb_convert_encoding()` substitutes the * invalid bytes instead and says nothing. * * @since 1.1.0 * * @param string $value Raw byte string. * @return string Value containing only valid UTF-8. */ private function scrub_utf8( $value ) { $scrubbed = mb_convert_encoding( (string) $value, 'UTF-8', 'UTF-8' ); return is_string( $scrubbed ) ? $scrubbed : ''; } /** * Returns the direct peer address, or null when it is not a valid IP. * * `REMOTE_ADDR` only. `X-Forwarded-For`, `Forwarded`, `CF-Connecting-IP` and * every similar header is attacker-controlled on a site that is not behind a * proxy that overwrites it, so trusting one would let any visitor write an * arbitrary address into the log. The cost is that a site behind a CDN or * reverse proxy records the proxy's address rather than the crawler's, which * the admin page and the README both state. * * @since 1.1.0 * * @return string|null Valid IPv4 or IPv6 address, or null. */ private function read_client_ip() { if ( ! isset( $_SERVER['REMOTE_ADDR'] ) || ! is_scalar( $_SERVER['REMOTE_ADDR'] ) ) { return null; } $address = trim( (string) wp_unslash( $_SERVER['REMOTE_ADDR'] ) ); // phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized -- Validated as an IP address below, which is stricter than any sanitiser. if ( false === filter_var( $address, FILTER_VALIDATE_IP ) ) { return null; } return $this->limit( $address, self::MAX_IP_ADDRESS ); } /** * Resolves the published, publicly viewable post this request serves. * * Reads the queried object rather than the queried object id, and requires a * WP_Post. `get_queried_object_id()` returns a term id on a taxonomy archive * and a user id on an author archive, either of which could collide with a * real post id and attribute a visit to unrelated content. An archive, * search or date view has no post of its own and is simply not recorded. * * `url_to_postid()` is deliberately not used as a fallback: it would attach * a visit to a post WordPress did not serve on this request. * * A queried `WP_Post` is necessary but not sufficient, so `is_singular()` * is required as well. WordPress hands the blog index the configured posts * page as its queried object, and that view is a list of other people's * content, not a visit to the page itself - `is_home()` is true and * `is_singular()` is false. Without the conditional, a site with a posts * page would record every crawl of its index as a visit to that page. A * static front page is a genuine singular view of a published page and is * still recorded. * * @since 1.1.0 * * @return WP_Post|null Published public post, or null. */ private function resolve_published_post() { if ( ! is_singular() ) { return null; } $queried = get_queried_object(); if ( ! $queried instanceof WP_Post ) { return null; } if ( 'publish' !== $queried->post_status ) { return null; } if ( 'attachment' === $queried->post_type ) { return null; } // A password-protected post is not public content, even when published. if ( '' !== (string) $queried->post_password ) { return null; } $type = get_post_type_object( $queried->post_type ); if ( ! $type instanceof WP_Post_Type || ! $type->public ) { return null; } return $queried; } /** * Truncates a value to a column's character bound. * * Characters, not bytes: the columns are `VARCHAR(n)` under a utf8mb4 * collation, so the limit counts characters and `substr()` could both cut a * multibyte character in half and reject a value that fits. * * @since 1.1.0 * * @param string $value Value to bound. * @param int $length Maximum characters. * @return string Bounded value. */ private function limit( $value, $length ) { return mb_substr( (string) $value, 0, $length, 'UTF-8' ); } }