, open: bool, exact: bool}|null */ private ?array $mark = null; /** * Whether the last run stopped without finishing its chunk, so every step is marked before it runs. * * @var bool */ private bool $recovering = false; /** * When, and for which unit, the mark was last written in this run. * * @var float */ private float $marked_at = 0.0; /** * Unit of the last mark written in this run. * * @var int|null */ private ?int $marked_in = null; /** * Whether this run wrote the mark; a mark it only read stays as the last crash left it. * * @var bool */ private bool $marked_here = false; /** * Whether the mark this run read is only close to where the last run stopped, so any step after it may have been started once already. * * @var bool */ private bool $uncertain = false; /** * Constructor. * * @param ServiceRegistry $registry Service registry. * @param Repository $settings Settings repository. * @param Inventory $inventory Inventory the findings go into. * @param OptionInspector $options Option inspection. * @param callable $units Returns the units of a scan, see {@see Sources::units()}. * @param string $common_hosts_file Absolute path of `common-hosts.json`. * @param callable|null $dump Returns the registered scripts (`handle`, `src`, `inline`) or null when the page cannot be fetched; the dump is skipped, with a notice, without it. */ public function __construct( ServiceRegistry $registry, Repository $settings, Inventory $inventory, OptionInspector $options, callable $units, string $common_hosts_file, ?callable $dump = null ) { $this->registry = $registry; $this->settings = $settings; $this->inventory = $inventory; $this->options = $options; $this->units = $units; $this->common_hosts_file = $common_hosts_file; $this->dump = $dump; } /** * Begin a new scan; one in progress is left alone. The state option is inserted, or an unusable one replaced, in * one conditional query, so of several callers at once exactly one begins the scan. * * @return bool Whether this call began the scan. */ public function start(): bool { global $wpdb; $raw = $this->raw_state(); if ( null !== $raw && null !== self::validate( maybe_unserialize( $raw ) ) ) { return false; } delete_option( self::CRASH_OPTION ); $fresh = maybe_serialize( array( 'run_id' => wp_generate_uuid4(), 'started' => time(), 'units' => array_values( (array) call_user_func( $this->units ) ), 'unit' => 0, 'file' => 0, 'services' => array(), 'hosts' => array(), 'notices' => array(), 'lock' => 0, ) ); if ( null === $raw ) { $won = 1 === $wpdb->query( $wpdb->prepare( "INSERT IGNORE INTO {$wpdb->options} (option_name, option_value, autoload) VALUES (%s, %s, 'no')", self::STATE_OPTION, $fresh ) ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Atomic insert; the option API cannot say who won. } else { $won = $this->swap_state( $raw, $fresh ); } $this->forget_state_cache(); return $won; } /** * Work on the scan for up to a time budget, beginning one when none is in progress. At least one step is done. * * @param float $budget Seconds to spend. * @return bool Whether the scan is complete; false when it needs another run, or another run is working on it. */ public function advance( float $budget = self::BUDGET ): bool { if ( null === $this->state() ) { $this->start(); } $state = $this->claim( $budget ); if ( null === $state ) { return false; } $deadline = microtime( true ) + $budget; $detector = $this->detector(); $total = count( $state['units'] ); $files = null; $this->mark = $this->crash_mark( $state['run_id'] ); $this->recovering = null !== $this->mark && $this->mark['open']; $this->uncertain = $this->recovering && ! $this->mark['exact']; $this->marked_at = 0.0; $this->marked_in = null; $this->marked_here = false; do { if ( $state['unit'] >= $total ) { break; } $unit = $state['units'][ $state['unit'] ]; if ( Sources::DUMP === $unit['type'] ) { if ( $this->may_start( $state, self::NOTICE_PAGE_CHECK_SKIPPED ) ) { $this->inspect_dump( $detector, $state ); } ++$state['unit']; continue; } if ( Sources::OPTIONS === $unit['type'] ) { if ( $this->may_start( $state, self::NOTICE_OPTIONS_SKIPPED ) ) { $this->inspect_options( $state ); } ++$state['unit']; continue; } $files ??= self::files( $unit['path'] ); if ( $state['file'] >= count( $files ) ) { ++$state['unit']; $state['file'] = 0; $files = null; continue; } if ( $this->may_start( $state, self::NOTICE_SKIPPED_FILE . str_replace( trailingslashit( WP_CONTENT_DIR ), '', $files[ $state['file'] ] ) ) ) { $this->scan_file( $files[ $state['file'] ], $unit['label'], $detector, $state ); } ++$state['file']; } while ( microtime( true ) < $deadline ); if ( $state['unit'] >= $total ) { if ( ! $this->may_start( $state, null ) ) { // Finishing stopped the scan twice; a new scan can start, this one cannot be completed. delete_option( self::STATE_OPTION ); delete_option( self::CRASH_OPTION ); return false; } $this->finish( $state ); return true; } $state['lock'] = 0; $this->save( $state ); $this->settle_mark(); return false; } /** * Whether a scan is in progress. */ public function running(): bool { return null !== $this->state(); } /** * Whether another run is working on the scan right now. * * @phpstan-impure */ public function locked(): bool { $state = $this->state(); return null !== $state && $state['lock'] > time(); } /** * Where the scan stands. * * @return array{running: bool, started: int, units_total: int, units_done: int, last_completed: int, last_services: int, last_browsing: int, last_hosts: int, last_notices: list} */ public function status(): array { $state = $this->state(); $results = $this->results(); $services = $this->inventory->get()['services']; return array( 'running' => null !== $state, 'started' => null !== $state ? $state['started'] : 0, 'units_total' => null !== $state ? count( $state['units'] ) : 0, 'units_done' => null !== $state ? $state['unit'] : 0, 'last_completed' => (int) ( $results['completed'] ?? 0 ), 'last_services' => count( $services ), 'last_browsing' => count( array_filter( $services, static fn ( array $service ): bool => 'browsing' === ( $service['origin'] ?? '' ) ) ), 'last_hosts' => count( (array) ( $results['hosts'] ?? array() ) ), 'last_notices' => array_values( array_filter( (array) ( $results['notices'] ?? array() ), 'is_string' ) ), ); } /** * The findings of the last completed scan. * * @return array{run_id: string, started: int, completed: int, units: int, services: array>, hosts: array>, notices?: list}|null Null when no scan has completed. */ public function results(): ?array { $results = get_option( self::RESULTS_OPTION, null ); return is_array( $results ) && is_array( $results['services'] ?? null ) && is_array( $results['hosts'] ?? null ) ? $results : null; } /** * The scan in progress; null when there is none, or it was abandoned a day ago. * * @return array{run_id: string, started: int, units: list, unit: int, file: int, services: array>, hosts: array>, lock: int}|null */ private function state(): ?array { return self::validate( get_option( self::STATE_OPTION, null ) ); } /** * A stored scan state, or null when it is unusable or was abandoned a day ago. * * @param mixed $state Stored value. * @return array{run_id: string, started: int, units: list, unit: int, file: int, services: array>, hosts: array>, lock: int}|null */ private static function validate( $state ): ?array { if ( ! is_array( $state ) || ! is_int( $state['started'] ?? null ) || ! is_array( $state['units'] ?? null ) || ! is_int( $state['unit'] ?? null ) || ! is_int( $state['file'] ?? null ) || ! is_array( $state['services'] ?? null ) || ! is_array( $state['hosts'] ?? null ) || $state['started'] < time() - DAY_IN_SECONDS ) { return null; } $state['lock'] = (int) ( $state['lock'] ?? 0 ); return $state; } /** * Take the scan's lock: set `lock` in the stored state with one conditional query that only matches the state as it * was read, so of several runs at once exactly one gets it. * * @param float $budget Seconds the run may spend. * @return array{run_id: string, started: int, units: list, unit: int, file: int, services: array>, hosts: array>, lock: int}|null The locked state; null when there is no scan or another run holds it. */ private function claim( float $budget ): ?array { $raw = $this->raw_state(); $state = null === $raw ? null : self::validate( maybe_unserialize( $raw ) ); if ( null === $raw || null === $state || $state['lock'] > time() ) { return null; } $state['lock'] = time() + (int) ceil( $budget ) + MINUTE_IN_SECONDS; $won = $this->swap_state( $raw, maybe_serialize( $state ) ); $this->forget_state_cache(); return $won ? $state : null; } /** * The stored state exactly as the database holds it, bypassing the option cache. */ private function raw_state(): ?string { global $wpdb; $raw = $wpdb->get_var( $wpdb->prepare( "SELECT option_value FROM {$wpdb->options} WHERE option_name = %s", self::STATE_OPTION ) ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- A cached copy could be stale. return is_string( $raw ) ? $raw : null; } /** * Replace the stored state when it still is what the caller read. * * @param string $old Stored value as read. * @param string $replacement Value to store. */ private function swap_state( string $old, string $replacement ): bool { global $wpdb; return 1 === $wpdb->query( $wpdb->prepare( "UPDATE {$wpdb->options} SET option_value = %s WHERE option_name = %s AND option_value = %s", $replacement, self::STATE_OPTION, $old ) ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Compare-and-swap. } /** * Drop the cached copies of the state after it was written around the option API. */ private function forget_state_cache(): void { wp_cache_delete( self::STATE_OPTION, 'options' ); wp_cache_delete( 'notoptions', 'options' ); wp_cache_delete( 'alloptions', 'options' ); } /** * Store the scan in progress. * * @param array $state State. */ private function save( array $state ): void { update_option( self::STATE_OPTION, $state, false ); } /** * Detector for the services and the admin's rules, necessary services included. */ private function detector(): Detector { $hosts = Detector::common_hosts( $this->common_hosts_file ); foreach ( array( home_url(), site_url() ) as $url ) { $host = wp_parse_url( $url, PHP_URL_HOST ); if ( is_string( $host ) ) { $hosts[] = $host; } } return new Detector( $this->registry->detection_rules(), $hosts ); } /** * The file the scan last started on, with the times it started on it; null when it belongs to another scan. * * @param string $run_id Id of the scan in progress. * @return array{run_id: string, unit: int, file: int, tries: int, skipped: list, open: bool, exact: bool}|null */ private function crash_mark( string $run_id ): ?array { $mark = get_option( self::CRASH_OPTION, null ); if ( ! is_array( $mark ) || ( $mark['run_id'] ?? null ) !== $run_id || ! is_int( $mark['unit'] ?? null ) || ! is_int( $mark['file'] ?? null ) || ! is_int( $mark['tries'] ?? null ) ) { return null; } $skipped = is_array( $mark['skipped'] ?? null ) ? array_values( array_filter( $mark['skipped'], 'is_string' ) ) : array(); return array( 'run_id' => $mark['run_id'], 'unit' => $mark['unit'], 'file' => $mark['file'], 'tries' => $mark['tries'], 'skipped' => $skipped, 'open' => false !== ( $mark['open'] ?? true ), 'exact' => false !== ( $mark['exact'] ?? true ), ); } /** * Say whether the scan may start on the step at its position (a file, the options, the home page, or finishing), and * record where it is so a crash can be traced. A step the scan started on twice without finishing it has stopped the * scan twice: it is skipped, for the rest of the scan, with a notice. * * The position is stored before the step runs, because a crash loses the scan's own state back to the start of its * chunk. Storing it costs a write, so a run does it once per unit and then at most once a second; after a crash the * next run knows the position is only close and marks every step until its chunk completes. A step that may have * started before the stale position counts as started once already. * * @param array $state State, changed in place. * @param string|null $notice Notice to record when the step is skipped; none for finishing. */ private function may_start( array &$state, ?string $notice ): bool { $key = $state['unit'] . ':' . $state['file']; $skipped = $this->mark['skipped'] ?? array(); if ( in_array( $key, $skipped, true ) ) { return $this->skip( $state, $notice ); } if ( ! $this->recovering ) { $this->mark_in_passing( $state, $skipped ); return true; } $order = -1; if ( null !== $this->mark ) { $order = $this->mark['unit'] <=> $state['unit']; if ( 0 === $order ) { $order = $this->mark['file'] <=> $state['file']; } } if ( $order > 0 ) { // Reading again what came before the step that crashed. return true; } $tries = 0 === $order && null !== $this->mark ? $this->mark['tries'] : (int) $this->uncertain; $this->mark = array( 'run_id' => (string) $state['run_id'], 'unit' => (int) $state['unit'], 'file' => (int) $state['file'], 'tries' => min( $tries + 1, self::MAX_ATTEMPTS + 1 ), 'skipped' => $skipped, 'open' => true, 'exact' => true, ); $this->marked_here = true; if ( $tries < self::MAX_ATTEMPTS ) { update_option( self::CRASH_OPTION, $this->mark, false ); return true; } $this->mark['skipped'][] = $key; $this->mark['tries'] = $tries; update_option( self::CRASH_OPTION, $this->mark, false ); return $this->skip( $state, $notice ); } /** * Record the notice of a skipped step. * * @param array $state State, changed in place. * @param string|null $notice Notice; none for finishing. */ private function skip( array &$state, ?string $notice ): bool { if ( null !== $notice ) { $state['notices'] = array_values( array_unique( array_merge( (array) ( $state['notices'] ?? array() ), array( $notice ) ) ) ); } return false; } /** * Store the position, when the unit is a new one or the last write is a second old. * * @param array $state State. * @param array $skipped Steps skipped so far. */ private function mark_in_passing( array $state, array $skipped ): void { $now = microtime( true ); if ( $this->marked_in === $state['unit'] && $now - $this->marked_at < 1.0 ) { return; } $this->marked_in = (int) $state['unit']; $this->marked_at = $now; $this->mark = array( 'run_id' => (string) $state['run_id'], 'unit' => (int) $state['unit'], 'file' => (int) $state['file'], 'tries' => 1, 'skipped' => $skipped, 'open' => true, 'exact' => false, ); update_option( self::CRASH_OPTION, $this->mark, false ); $this->marked_here = true; } /** * Close the mark this run wrote when its chunk completed: it goes unless it carries skipped steps, which the scan keeps. */ private function settle_mark(): void { if ( null === $this->mark || ! $this->marked_here ) { return; } if ( array() === $this->mark['skipped'] ) { delete_option( self::CRASH_OPTION ); return; } $this->mark['open'] = false; update_option( self::CRASH_OPTION, $this->mark, false ); } /** * Note the services other plugins' options switch on. * * @param array $state State, changed in place. */ private function inspect_options( array &$state ): void { $known = $this->registry->services(); foreach ( $this->options->inspect() as $slug => $names ) { if ( isset( $known[ $slug ] ) ) { foreach ( $names as $name ) { self::note( $state['services'], $slug, 'option:' . $name ); } } } } /** * Note what the scripts the front page registers refer to; without the dump, record that the page check was skipped. * * @param Detector $detector Detector. * @param array $state State, changed in place. */ private function inspect_dump( Detector $detector, array &$state ): void { $scripts = null === $this->dump ? null : call_user_func( $this->dump ); if ( null === $scripts ) { $state['notices'] = array_values( array_unique( array_merge( (array) ( $state['notices'] ?? array() ), array( self::NOTICE_PAGE_CHECK_SKIPPED ) ) ) ); return; } foreach ( $scripts as $script ) { $this->record_found( $detector->scan_script( $script['src'], $script['inline'] ), 'enqueue:' . $script['handle'], $state ); } } /** * Note what one file refers to. * * @param string $file Absolute path. * @param string $label Label of the unit the file belongs to. * @param Detector $detector Detector. * @param array $state State, changed in place. */ private function scan_file( string $file, string $label, Detector $detector, array &$state ): void { $size = is_file( $file ) ? filesize( $file ) : false; $code = false !== $size && $size > 0 && $size <= self::MAX_FILE_BYTES ? file_get_contents( $file ) : false; // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents, WordPressVIPMinimum.Performance.FetchingRemoteData.FileGetContentsUnknown -- Local file of an active plugin or theme. if ( false === $code ) { return; } $this->record( $detector, $code, $label, $state ); } /** * Note the services and hosts a piece of code refers to. * * @param Detector $detector Detector. * @param string $code Source code. * @param string $label Where the code comes from. * @param array $state State, changed in place. */ private function record( Detector $detector, string $code, string $label, array &$state ): void { $this->record_found( $detector->scan( $code ), $label, $state ); } /** * Note what a detector found. * * @param array{services: list, hosts: list} $found Findings. * @param string $label Label of the unit. * @param array $state State, changed in place. */ private function record_found( array $found, string $label, array &$state ): void { foreach ( $found['services'] as $key ) { self::note( $state['services'], $key, $label ); } foreach ( $found['hosts'] as $host ) { if ( isset( $state['hosts'][ $host ] ) || count( $state['hosts'] ) < self::MAX_HOSTS ) { self::note( $state['hosts'], $host, $label ); } } } /** * Remember where a service or host was seen, a few places at most. * * @param array> $bucket Findings by key, changed in place. * @param string $key Service key or host. * @param string $label Where it was seen. */ private static function note( array &$bucket, string $key, string $label ): void { $places = $bucket[ $key ] ?? array(); if ( count( $places ) < self::MAX_WHERE && ! in_array( $label, $places, true ) ) { $places[] = $label; } $bucket[ $key ] = $places; } /** * Store the findings, hand them to the inventory and announce the scan. * * @param array $state State of the finished scan. */ private function finish( array $state ): void { $results = array( 'run_id' => $state['run_id'], 'started' => $state['started'], 'completed' => time(), 'units' => count( $state['units'] ), 'services' => $state['services'], 'hosts' => $state['hosts'], 'notices' => array_values( array_unique( (array) ( $state['notices'] ?? array() ) ) ), ); update_option( self::RESULTS_OPTION, $results, false ); delete_option( self::STATE_OPTION ); delete_option( self::CRASH_OPTION ); $this->record_last_run( $results['completed'] ); $this->inventory->rebuild(); /** * Fires after a static scan finished and the inventory was rebuilt from it. * * @param array $results Findings: `run_id`, `started`, `completed`, `units`, `services` (service slug or * `user:` => where it was seen) and `hosts` (script and iframe hosts no rule claims => where they were seen) and `notices` (`page_check_skipped` * when the site could not fetch its own front page). */ do_action( 'consentfold_scan_completed', $results ); } /** * Record the end of the scan in the `scanner.last_run` setting. * * @param int $time Unix time. */ private function record_last_run( int $time ): void { $site = $this->settings->get_site(); $scanner = is_array( $site['scanner'] ?? null ) ? $site['scanner'] : array(); try { $this->settings->set( array_merge( $site, array( 'scanner' => array_merge( $scanner, array( 'last_run' => $time ) ) ) ) ); } catch ( InvalidSettingsException $e ) { // The stored settings no longer validate; the admin sees that where they edit them. The scan still counts. unset( $e ); } } /** * The scannable files of a plugin or theme, in a stable order. * * @param string $path Directory, or a single file. * @return list */ private static function files( string $path ): array { if ( is_file( $path ) ) { return array( $path ); } if ( ! is_dir( $path ) ) { return array(); } $files = array(); try { $filter = new RecursiveCallbackFilterIterator( new RecursiveDirectoryIterator( $path, FilesystemIterator::SKIP_DOTS ), static function ( SplFileInfo $item ): bool { return $item->isDir() ? ! in_array( $item->getFilename(), self::SKIPPED_DIRS, true ) : in_array( strtolower( $item->getExtension() ), self::EXTENSIONS, true ); } ); foreach ( new RecursiveIteratorIterator( $filter, RecursiveIteratorIterator::LEAVES_ONLY, RecursiveIteratorIterator::CATCH_GET_CHILD ) as $item ) { $files[] = $item->getPathname(); } } catch ( UnexpectedValueException $e ) { // An unreadable top directory has nothing to scan. unset( $e ); } sort( $files ); return $files; } }