get_var( $wpdb->prepare( "SELECT group_id FROM {$wpdb->prefix}actionscheduler_groups WHERE slug = %s", 'wardpair' ) ); if ( $group_id ) { // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- The placeholder list is generated from the number of values, and every value is still bound by $wpdb->prepare(). $killed += (int) $wpdb->query( $wpdb->prepare( "UPDATE {$wpdb->prefix}actionscheduler_actions SET status = 'failed', extended_args = NULL, claim_id = 0 WHERE group_id = %d AND hook IN ({$transfer_ph}) AND status IN ('in-progress', 'pending')", array_merge( array( $group_id ), $transfer_hooks ) ) ); // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- The placeholder list is generated from the number of values, and every value is still bound by $wpdb->prepare(). // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- The placeholder list is generated from the number of values, and every value is still bound by $wpdb->prepare(). $killed += (int) $wpdb->query( $wpdb->prepare( "UPDATE {$wpdb->prefix}actionscheduler_actions SET status = 'failed', extended_args = NULL, claim_id = 0 WHERE group_id = %d AND hook NOT IN ({$transfer_ph}) AND status IN ('in-progress', 'pending') AND scheduled_date_gmt < DATE_SUB( UTC_TIMESTAMP(), INTERVAL %d MINUTE )", array_merge( array( $group_id ), $transfer_hooks, array( self::STALE_JOB_MIN_AGE_MINUTES ) ) ) ); // phpcs:enable WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared } // Belt-and-braces: the named list too, in case the group lookup // above found nothing (an Action Scheduler storage backend that // doesn't use the groups table). Same split as Step 1's group-based // kill: transfer hooks unconditionally, the rest age-guarded. // // Built as plain args arrays (extra values appended at the end) // rather than `...self::OWN_ACTIONS, $extra` — this plugin's stated // minimum is PHP 7.4, and a positional argument after a spread isn't // reliably supported before PHP 8.1. $other_hooks = array_values( array_diff( self::OWN_ACTIONS, $transfer_hooks ) ); $other_ph = implode( ', ', array_fill( 0, count( $other_hooks ), '%s' ) ); // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- The placeholder list is generated from the number of values, and every value is still bound by $wpdb->prepare(). // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- The placeholder list is generated from the number of values, and every value is still bound by $wpdb->prepare(). $wpdb->query( $wpdb->prepare( "UPDATE {$wpdb->prefix}actionscheduler_actions SET status = 'failed', extended_args = NULL WHERE hook IN ({$transfer_ph}) AND status IN ('in-progress', 'pending')", $transfer_hooks ) ); // phpcs:enable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- The placeholder list is generated from the number of values, and every value is still bound by $wpdb->prepare(). $wpdb->query( $wpdb->prepare( "UPDATE {$wpdb->prefix}actionscheduler_actions SET status = 'failed', extended_args = NULL WHERE hook IN ({$other_ph}) AND status IN ('in-progress', 'pending') AND scheduled_date_gmt < DATE_SUB( UTC_TIMESTAMP(), INTERVAL %d MINUTE )", array_merge( $other_hooks, array( self::STALE_JOB_MIN_AGE_MINUTES ) ) ) ); // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared // ── Step 2: Kill any stale backup chunk jobs from previous clone ────── foreach ( array( 'wardpair_push_catchup', 'wardpair_file_backup_chunk', 'wardpair_db_backup_chunk' ) as $hook ) { $wpdb->query( $wpdb->prepare( "UPDATE {$wpdb->prefix}actionscheduler_actions SET status = 'failed' WHERE hook = %s AND status IN ('in-progress', 'pending')", $hook ) ); } // ── Step 3: Kill stuck AS migration hook ────────────────────────────── $wpdb->query( $wpdb->prepare( "UPDATE %i SET status = 'failed' WHERE hook = 'action_scheduler/migration_hook' AND status IN ('in-progress', 'pending')", $wpdb->prefix . 'actionscheduler_actions' ) ); // ── Step 4: Release stale claims (actions stuck > 10 min) ──────────── $wpdb->query( $wpdb->prepare( "UPDATE %i a JOIN %i c ON a.claim_id = c.claim_id SET a.status = 'failed', a.claim_id = 0 WHERE a.status = 'in-progress' AND c.date_created_gmt < DATE_SUB(UTC_TIMESTAMP(), INTERVAL 10 MINUTE)", $wpdb->prefix . 'actionscheduler_actions', $wpdb->prefix . 'actionscheduler_claims' ) ); // ── Step 5: Delete orphaned claim records ───────────────────────────── $wpdb->query( $wpdb->prepare( "DELETE c FROM %i c WHERE NOT EXISTS ( SELECT 1 FROM %i a WHERE a.claim_id = c.claim_id AND a.status = 'in-progress' )", $wpdb->prefix . 'actionscheduler_claims', $wpdb->prefix . 'actionscheduler_actions' ) ); // ── Step 6: Cancel remaining pending clone actions via AS API ───────── // Hook only — no args, no group. Passing group with an empty args // array made Action Scheduler 3.8.0 skip its bulk-cancel shortcut and // fall through to as_unschedule_action(), which matches on // `a.args = '[]'` (exact match). Every chunk job carries real args // (filename, chunk index, total), so none of them matched and this // step silently cancelled nothing for the hooks that matter most. // Hook names here are all Wardpair-owned and globally unique, so // there's no risk of an unqualified hook match catching someone // else's action. if ( function_exists( 'as_unschedule_all_actions' ) ) { foreach ( self::OWN_ACTIONS as $action ) { as_unschedule_all_actions( $action ); } } // ── Step 7: Wipe clone retry flags ──────────────────────────────────── $wpdb->query( $wpdb->prepare( "DELETE FROM %i WHERE option_name LIKE %s OR option_name LIKE %s", $wpdb->options, 'wardpair_clone_retry_%', 'wardpair_clone_file_retry_%' ) ); // ── Step 7b: Wipe extraction state ─────────────────────────────────── $wpdb->query( $wpdb->prepare( "DELETE FROM %i WHERE option_name LIKE %s", $wpdb->options, 'wardpair_extract_%' ) ); // ── Step 8: Reset all clone state flags ─────────────────────────────── $reset_keys = array( 'wardpair_clone_primary_prefix', 'wardpair_clone_started', 'wardpair_clone_check_count', 'wardpair_clone_backup_file', 'wardpair_clone_job_key', 'wardpair_clone_complete', 'wardpair_clone_files_started', 'wardpair_clone_file_check_count', 'wardpair_clone_file_backup_file', 'wardpair_clone_file_job_key', 'wardpair_clone_files_complete', 'wardpair_clone_restore_file', 'wardpair_clone_restore_running', 'wardpair_clone_restore_heartbeat', 'wardpair_clone_db_started', 'wardpair_clone_db_check_count', 'wardpair_clone_files_extracted', 'wardpair_clone_db_ready', 'wardpair_clone_db_only', 'wardpair_clone_extract_key', 'wardpair_clone_extract_pending', 'wardpair_chunk_size', 'wardpair_clone_adaptive_sleep', 'wardpair_clone_progress', 'wardpair_clone_negotiated_at', ); foreach ( $reset_keys as $key ) { delete_option( $key ); } // Verify/repair run state is pairing-scoped too. Sentinel_Clone_Verify::reset_state(); // Start every run on the direct origin route with binary chunks enabled. Sentinel_Transport::reset_state(); // Bust the in-process cache so next get_chunk_size() re-reads from DB. self::$chunk_size_cache = null; Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_OK, "Queue reset: {$killed} pending/stuck Wardpair action(s) cleared. State reset." ); } // ========================================================================= // DISPATCH (called from Sentinel_Sync::apply_payload) // ========================================================================= public static function handle_clone_payload( array $payload ): void { switch ( $payload['type'] ?? '' ) { case 'clone_chunk': Sentinel_Clone_Secondary::receive_chunk( $payload ); break; case 'clone_file_chunk': Sentinel_Clone_Secondary::receive_file_chunk( $payload ); break; case 'clone_status': Sentinel_Clone_Secondary::report_status( $payload ); break; case 'clone_verify_page': Sentinel_Clone_Verify::secondary_verify_page( $payload ); break; case 'clone_db_fingerprint': Sentinel_Clone_Verify::secondary_db_fingerprint( $payload ); break; case 'clone_repair_files': Sentinel_Clone_Verify::secondary_repair_files( $payload ); break; case 'clone_repair_pull': Sentinel_Clone_Verify::secondary_repair_pull( $payload ); break; case 'clone_mark_complete': // Primary saw real content (most/all of its tables exist // here) but this secondary's own bookkeeping still says // "never cloned" — usually a disconnect/reconnect that // reset the flag without touching actual content. Correct // it so ordinary sync (push_to_secondary()'s bootstrap // guard) and future repairs stop treating this site as // unseeded. if ( ! get_option( 'wardpair_clone_complete', false ) ) { update_option( 'wardpair_clone_complete', true ); update_option( 'wardpair_bootstrap_complete', true ); Sentinel_Sync::log( Sentinel_Sync::DIR_PULL, 'clone', Sentinel_Sync::STATUS_OK, 'Clone-complete flag corrected: primary confirmed this secondary already has seeded content.' ); } break; case 'chunk_size_probe': // Secondary just accepts — ok=true is returned automatically // by handle_incoming_payload(). Log for visibility. Sentinel_Sync::log( Sentinel_Sync::DIR_PULL, 'clone', Sentinel_Sync::STATUS_OK, 'Chunk size probe accepted: ' . size_format( (int) ( $payload['probe_size'] ?? 0 ) ) ); break; } } // ========================================================================= // PROXY — keeps Sentinel_Backup's call site unchanged after the split // ========================================================================= /** * Called by Sentinel_Backup::zip_restore_phase_complete() after a queued * ZIP restore finishes. Proxies to Sentinel_Clone_Secondary. */ public static function trigger_maybe_start_db_restore(): void { Sentinel_Clone_Secondary::trigger_maybe_start_db_restore(); } // ========================================================================= // SHARED HELPERS (called by both Primary and Secondary) // ========================================================================= /** * Returns the negotiated chunk size for this clone run. * Falls back to 512 KB — safe on any shared host. */ /** * The chunk size a transfer was queued with, recovered from its file size and * chunk count (total = ceil(size / chunk)), so every job of one transfer uses the * same boundaries whatever happens to the stored setting meanwhile. */ public static function frozen_chunk_size( int $filesize, int $total_chunks ): int { // Back-calculating from size / count is wrong whenever the last chunk is // short, so this now trusts the progress record and only falls back to the setting. $c = (int) ( self::progress_get()['chunk_size'] ?? 0 ); if ( $c > 0 && $filesize > 0 && $total_chunks > 0 && (int) ceil( $filesize / $c ) === $total_chunks ) { return $c; } return self::get_chunk_size(); } public static function get_chunk_size(): int { if ( self::$chunk_size_cache !== null ) { return self::$chunk_size_cache; } $stored = (int) get_option( 'wardpair_chunk_size', 0 ); if ( $stored >= 524288 && $stored <= 33554432 ) { // 512 KB – 32 MB self::$chunk_size_cache = $stored; return $stored; } self::$chunk_size_cache = 524288; // 512 KB fallback return self::$chunk_size_cache; } /** * True when a send failure looks like a temporary host-side throttle or * outage (rate limit, WAF, or an overloaded/unavailable backend) rather * than a real application error. These deserve a slower pace and a retry. */ public static function is_transient_block( string $reason ): bool { return (bool) preg_match( '/HTTP\s+(403|408|413|425|429|500|502|503|504|520|521|522|523|524)\b/', $reason ); } /** * Retry delay in seconds: 30s doubling per attempt, capped at 5 minutes, * with up to 20% jitter so retries don't line up with the host's limiter. */ public static function retry_backoff( int $retry_count ): int { $base = min( 300, 30 * (int) pow( 2, $retry_count ) ); return $base + random_int( 0, (int) ( $base * 0.2 ) ); } /** * Returns the current inter-chunk sleep in microseconds. * Reads the adaptive value if a rate-limit event has occurred this run, * otherwise returns the baseline constant. */ public static function get_adaptive_sleep(): int { $stored = (int) get_option( 'wardpair_clone_adaptive_sleep', 0 ); return $stored > 0 ? $stored : self::CHUNK_SLEEP_US; } /** * Called after every successful chunk send. * Decays the adaptive sleep 25 % toward the baseline. */ public static function on_chunk_success(): void { $current = (int) get_option( 'wardpair_clone_adaptive_sleep', 0 ); if ( $current <= self::CHUNK_SLEEP_US ) { return; } update_option( 'wardpair_clone_adaptive_sleep', max( self::CHUNK_SLEEP_US, (int) ( $current * 0.75 ) ) ); } /** * Called when a 403/429/5xx throttle is detected mid-transfer. * Doubles the inter-chunk sleep up to MAX_CHUNK_SLEEP. */ public static function on_chunk_rate_limited(): void { $new = min( self::get_adaptive_sleep() * 2, self::MAX_CHUNK_SLEEP ); update_option( 'wardpair_clone_adaptive_sleep', $new ); Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_OK, 'Rate-limit detected — inter-chunk sleep increased to ' . number_format( $new / 1_000_000, 1 ) . 's.' ); } /** * Converts a PHP ini size string (e.g. '128M') to bytes. */ public static function parse_ini_size( string $val ): int { $val = trim( $val ); $last = strtolower( $val[ strlen( $val ) - 1 ] ?? '' ); $num = (int) $val; switch ( $last ) { case 'g': return $num * 1024 * 1024 * 1024; case 'm': return $num * 1024 * 1024; case 'k': return $num * 1024; default: return $num; } } // ========================================================================= // PROGRESS RECORD, STALL DETECTION, RESUME // ========================================================================= /** One option holds where the clone is, so a dead job never takes the position with it. */ const PROGRESS_OPTION = 'wardpair_clone_progress'; /** A phase counts as stalled once nothing has moved for this long AND no queued job exists for it. */ const STALL_SECONDS = 240; /** Automatic resumes allowed without the cursor moving before the clone fails visibly. */ const MAX_IDLE_RESUMES = 3; const PHASE_ZIP_BUILD = 'zip_build'; const PHASE_FILE_TRANSFER = 'file_transfer'; const PHASE_DB_DUMP = 'db_dump'; const PHASE_DB_TRANSFER = 'db_transfer'; /** Every hook that can carry a phase forward. A phase with none pending or running is stalled. */ const PHASE_HOOKS = array( 'zip_build' => array( 'wardpair_clone_start', 'wardpair_file_backup_inventory', 'wardpair_file_backup_chunk', 'wardpair_file_backup_seal', 'wardpair_clone_snapshot_diff', 'wardpair_clone_snapshot_apply', 'wardpair_clone_snapshot_seal', ), 'file_transfer' => array( 'wardpair_clone_transfer_file_chunk' ), 'db_dump' => array( 'wardpair_clone_db_start', 'wardpair_db_backup_inventory', 'wardpair_db_backup_chunk', 'wardpair_db_backup_seal', ), 'db_transfer' => array( 'wardpair_clone_transfer_chunk' ), ); /** * The polls that watch a build finish. They are NOT counted as proof the phase is alive (they keep * firing after the build job itself has died), but a pause must still remove them. */ const POLL_HOOKS = array( 'wardpair_clone_check_file_backup', 'wardpair_clone_check_backup' ); public static function progress_get(): array { $p = get_option( self::PROGRESS_OPTION, array() ); return is_array( $p ) ? $p : array(); } /** Merge fields into the record and stamp updated_at. */ public static function progress_set( array $fields ): void { $p = array_merge( self::progress_get(), $fields ); $p['updated_at'] = time(); update_option( self::PROGRESS_OPTION, $p, false ); } /** Mark the saved position as failed (kept, so the admin can still resume it). */ public static function progress_mark_failed( string $note ): void { if ( ! empty( self::progress_get() ) ) { self::progress_set( array( 'status' => 'failed', 'note' => $note ) ); } } /** Status of the saved position: '' (none), active, paused, failed or complete. */ public static function progress_status(): string { return (string) ( self::progress_get()['status'] ?? '' ); } /** Record a build's position only when it moved, so a poll that sees no change does not look like progress. */ public static function progress_touch_cursor( int $cursor, int $total = 0 ): void { $p = self::progress_get(); if ( (int) ( $p['cursor'] ?? -1 ) !== $cursor || ( $total > 0 && (int) ( $p['total_chunks'] ?? 0 ) !== $total ) ) { self::progress_set( array( 'cursor' => $cursor, 'total_chunks' => $total ) ); } } public static function progress_clear(): void { delete_option( self::PROGRESS_OPTION ); } /** A transfer phase begins: freeze the file, its size and the chunk size it will use. */ public static function progress_begin_transfer( string $phase, string $filename, int $filesize, int $chunk_size, int $total_chunks ): void { self::progress_set( array( 'phase' => $phase, 'status' => 'active', 'filename' => $filename, 'filesize' => $filesize, 'chunk_size' => $chunk_size, 'total_chunks' => $total_chunks, 'cursor' => 0, 'idle_resumes' => 0, 'last_resume_cursor'=> -1, 'note' => '', ) ); } /** One chunk acknowledged: the next one to send is $next_cursor. */ public static function progress_chunk_done( int $next_cursor ): void { self::progress_set( array( 'cursor' => $next_cursor, 'status' => 'active', 'idle_resumes' => 0, 'last_resume_cursor' => -1, ) ); } /** * The chunk size a transfer was started with. Every job of one transfer must * cut the file at the same boundaries; re-reading the stored setting per job * (it can change between jobs) moves those boundaries and corrupts the file. */ public static function transfer_chunk_size( string $phase, string $filename, int $filesize, int $total_chunks ): int { $p = self::progress_get(); if ( ( $p['phase'] ?? '' ) === $phase && ( $p['filename'] ?? '' ) === $filename && (int) ( $p['filesize'] ?? 0 ) === $filesize ) { $c = (int) ( $p['chunk_size'] ?? 0 ); if ( $c > 0 && (int) ceil( $filesize / $c ) === $total_chunks ) { return $c; } } return self::get_chunk_size(); } /** True if any action for this phase is pending or running (any group, any args). */ private static function phase_has_live_action( string $phase ): bool { global $wpdb; $hooks = self::PHASE_HOOKS[ $phase ] ?? array(); if ( empty( $hooks ) ) { return false; } $in = implode( ',', array_fill( 0, count( $hooks ), '%s' ) ); // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- Table or column name taken from the plugin's own table list or the database's own metadata; identifiers cannot be bound as placeholders before WordPress 6.2. // phpcs:disable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare -- Table or column name taken from the plugin's own table list or the database's own metadata; identifiers cannot be bound as placeholders before WordPress 6.2. $n = (int) $wpdb->get_var( $wpdb->prepare( "SELECT COUNT(*) FROM %i WHERE status IN ('pending','in-progress') AND hook IN ($in)", ...array_merge( array( $wpdb->prefix . 'actionscheduler_actions' ), $hooks ) ) ); // phpcs:enable WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared return $n > 0; } /** Called from the 5-minute watchdog and from the admin status poll. */ public static function maybe_resume_stalled(): void { if ( get_option( 'wardpair_role' ) !== 'primary' || ! function_exists( 'as_enqueue_async_action' ) ) { return; } $p = self::progress_get(); if ( empty( $p['phase'] ) || 'active' !== ( $p['status'] ?? '' ) ) { return; // Nothing running, or paused/failed on purpose: never auto-resume those. } if ( ( time() - (int) ( $p['updated_at'] ?? 0 ) ) < self::STALL_SECONDS ) { return; } if ( self::phase_has_live_action( $p['phase'] ) ) { return; } // Throttle: several pollers may arrive together. if ( get_transient( 'wardpair_clone_resume_lock' ) ) { return; } set_transient( 'wardpair_clone_resume_lock', 1, 60 ); self::resume( 'auto' ); } /** Pause: stop the queue but keep every position, so the admin can resume later. */ public static function pause_keep_progress(): int { global $wpdb; self::new_run_id(); // Stops a batch that is already mid-loop. $cleared = 0; foreach ( array_merge( array( self::POLL_HOOKS ), array_values( self::PHASE_HOOKS ) ) as $hooks ) { foreach ( $hooks as $hook ) { if ( function_exists( 'as_unschedule_all_actions' ) ) { as_unschedule_all_actions( $hook ); $cleared++; } } } if ( ! empty( self::progress_get() ) ) { self::progress_set( array( 'status' => 'paused' ) ); } Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_OK, 'Clone paused by admin. Progress kept; use "Resume from last progress" to continue.' ); return $cleared; } private static function clear_retry_counters(): void { global $wpdb; $wpdb->query( $wpdb->prepare( "DELETE FROM %i WHERE option_name LIKE %s OR option_name LIKE %s", $wpdb->options, 'wardpair\\_clone\\_file\\_retry\\_%', 'wardpair\\_clone\\_retry\\_%' ) ); } /** Fail the clone with the exact place it stopped, instead of looping. */ private static function fail_progress( array $p, string $why ): array { self::progress_set( array( 'status' => 'failed', 'note' => $why ) ); update_option( 'wardpair_clone_files_started', false ); update_option( 'wardpair_clone_db_started', false ); Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_FAILED, 'Clone stopped: ' . $why, wp_json_encode( array( 'phase' => $p['phase'] ?? '', 'cursor' => $p['cursor'] ?? 0, 'total' => $p['total_chunks'] ?? 0, 'file' => $p['filename'] ?? '', ) ) ); Sentinel_Core::log_event( 'clone_error', 'Clone stopped: ' . $why ); return array( 'ok' => false, 'message' => $why ); } /** * Continue from the saved position. $source is 'auto' (stall detector) or 'admin'. * Returns array( ok, message ). */ public static function resume( string $source = 'admin' ): array { if ( get_option( 'wardpair_role' ) !== 'primary' ) { return array( 'ok' => false, 'message' => __( 'Only the primary can resume a clone.', 'wardpair' ) ); } $p = self::progress_get(); if ( empty( $p['phase'] ) || 'complete' === ( $p['status'] ?? '' ) ) { return array( 'ok' => false, 'message' => __( 'There is no saved clone progress to resume. Start a new clone instead.', 'wardpair' ) ); } $cursor = (int) ( $p['cursor'] ?? 0 ); $idle = ( $cursor === (int) ( $p['last_resume_cursor'] ?? -1 ) ) ? (int) ( $p['idle_resumes'] ?? 0 ) + 1 : 1; if ( 'admin' === $source ) { $idle = 1; // The admin chose to try again: give it a fresh allowance. } elseif ( $idle > self::MAX_IDLE_RESUMES ) { return self::fail_progress( $p, sprintf( /* translators: 1: phase, 2: chunk number, 3: total chunks */ 'no progress after %1$d automatic resumes (phase %2$s, chunk %3$d of %4$d). Check the Activity log for the cause, then use "Resume from last progress".', self::MAX_IDLE_RESUMES, $p['phase'], $cursor + 1, (int) ( $p['total_chunks'] ?? 0 ) ) ); } // Database phases: first confirm with the secondary that the files really arrived, so a // database failure never re-sends a ZIP that is already there. $db_note = ''; if ( in_array( $p['phase'], array( self::PHASE_DB_DUMP, self::PHASE_DB_TRANSFER ), true ) && ! get_option( 'wardpair_clone_db_only', false ) ) { $zip_name = (string) get_option( 'wardpair_clone_file_backup_file', '' ); if ( '' !== $zip_name ) { $zip_path = WARDPAIR_BACKUP_DIR . '/' . basename( $zip_name ); $st = self::confirm_files_on_secondary( $zip_name, is_file( $zip_path ) ? (int) filesize( $zip_path ) : 0 ); if ( null === $st ) { Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_OK, 'Resume: could not ask the secondary whether the files arrived; trusting this site\'s record and continuing with the database.' ); } elseif ( ! $st['confirmed'] ) { if ( 'auto' === $source ) { return self::fail_progress( $p, 'the secondary no longer has the files ZIP, so the database cannot be used yet. Use "Resume from last progress" to send the files again first.' ); } Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_FAILED, 'Resume: the secondary does not have the files ZIP — sending the files again before the database.' ); self::progress_clear(); delete_option( 'wardpair_clone_files_complete' ); update_option( 'wardpair_clone_files_started', false ); update_option( 'wardpair_clone_db_started', false ); as_enqueue_async_action( 'wardpair_clone_start', array(), 'wardpair' ); Sentinel_Cron::kick(); return array( 'ok' => true, 'message' => __( 'The secondary no longer has the files, so they are being sent again first, then the database.', 'wardpair' ) ); } else { $db_note = ' ' . __( 'The files are already on the secondary (confirmed), so only the database is sent.', 'wardpair' ); } } } self::progress_set( array( 'status' => 'active', 'idle_resumes' => $idle, 'last_resume_cursor' => $cursor ) ); self::clear_retry_counters(); Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_OK, sprintf( 'Resuming clone (%s) at phase %s, chunk %d.', $source, $p['phase'], $cursor + 1 ) ); $live = self::phase_has_live_action( $p['phase'] ); $ok = array( 'ok' => true, 'message' => __( 'Clone resumed from the last saved position.', 'wardpair' ) ); switch ( $p['phase'] ) { case self::PHASE_FILE_TRANSFER: case self::PHASE_DB_TRANSFER: $file = WARDPAIR_BACKUP_DIR . '/' . basename( (string) ( $p['filename'] ?? '' ) ); $size = (int) ( $p['filesize'] ?? 0 ); $csz = (int) ( $p['chunk_size'] ?? 0 ); $tot = (int) ( $p['total_chunks'] ?? 0 ); $file_phase = ( self::PHASE_FILE_TRANSFER === $p['phase'] ); // The file, its size or the chunking changed: boundaries no longer line up, so this phase starts over. if ( ! is_file( $file ) || (int) filesize( $file ) !== $size || $csz < 1 || (int) ceil( $size / $csz ) !== $tot ) { if ( 'auto' === $source ) { return self::fail_progress( $p, 'the transfer file or chunk size no longer matches the saved record, so it cannot be continued safely. Use "Resume from last progress" to restart this phase, or "Start over".' ); } Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_FAILED, 'Resume: the transfer file or chunk size no longer matches the saved record — restarting this phase from the beginning.' ); self::progress_clear(); if ( $file_phase ) { delete_option( 'wardpair_clone_files_complete' ); update_option( 'wardpair_clone_files_started', false ); as_enqueue_async_action( 'wardpair_clone_start', array(), 'wardpair' ); } else { update_option( 'wardpair_clone_db_started', false ); as_enqueue_async_action( 'wardpair_clone_db_start', array(), 'wardpair' ); } Sentinel_Cron::kick(); return array( 'ok' => true, 'message' => __( 'The saved file no longer matches, so this phase restarted from the beginning.', 'wardpair' ) ); } if ( $file_phase ) { update_option( 'wardpair_clone_files_started', true ); } else { update_option( 'wardpair_clone_db_started', true ); } if ( ! $live ) { as_enqueue_async_action( $file_phase ? 'wardpair_clone_transfer_file_chunk' : 'wardpair_clone_transfer_chunk', array( $p['filename'], min( $cursor, $tot - 1 ), $tot ), 'wardpair' ); } break; case self::PHASE_ZIP_BUILD: case self::PHASE_DB_DUMP: $zip = ( self::PHASE_ZIP_BUILD === $p['phase'] ); $job_key = (string) get_option( $zip ? 'wardpair_clone_file_job_key' : 'wardpair_clone_job_key', '' ); $job = '' !== $job_key ? get_option( $job_key ) : false; if ( ! is_array( $job ) || 'failed' === ( $job['status'] ?? '' ) ) { if ( 'auto' === $source ) { return self::fail_progress( $p, 'the build job failed or vanished (' . ( is_array( $job ) ? (string) ( $job['error'] ?? 'no detail' ) : 'record missing' ) . '). Use "Resume from last progress" to rebuild it, or "Start over".' ); } // The job itself is gone or failed: only a new build can continue. Nothing was transferred yet. self::progress_clear(); if ( $zip ) { update_option( 'wardpair_clone_files_started', false ); as_enqueue_async_action( 'wardpair_clone_start', array(), 'wardpair' ); } else { update_option( 'wardpair_clone_db_started', false ); as_enqueue_async_action( 'wardpair_clone_db_start', array(), 'wardpair' ); } $ok['message'] = __( 'The build job was gone or had failed, so it was queued again from its start.', 'wardpair' ); break; } if ( ! $live ) { $hook = self::next_build_hook( $job_key, $job, $zip ); as_enqueue_async_action( $hook, array( 'job_key' => $job_key ), 'wardpair' ); } // The poller that hands the finished file to the transfer step. update_option( $zip ? 'wardpair_clone_file_check_count' : 'wardpair_clone_db_check_count', 0 ); if ( ! as_next_scheduled_action( $zip ? 'wardpair_clone_check_file_backup' : 'wardpair_clone_check_backup' ) ) { as_schedule_single_action( time() + self::CHECK_INTERVAL, $zip ? 'wardpair_clone_check_file_backup' : 'wardpair_clone_check_backup', array(), 'wardpair' ); } break; } Sentinel_Cron::kick(); $ok['message'] .= $db_note; return $ok; } /** * Ask the secondary whether the files ZIP is already there. Returns null when it cannot be * confirmed either way (unreachable, or an older secondary that does not know the question), * otherwise array( confirmed, extracted, zip_size ). */ public static function confirm_files_on_secondary( string $filename, int $filesize = 0 ): ?array { $url = get_option( 'wardpair_partner_url', '' ); $ip = get_option( 'wardpair_partner_ip', '' ); $slug = get_option( 'wardpair_endpoint_slug', '' ); $secret = get_option( 'wardpair_shared_secret', '' ); if ( empty( $url ) || empty( $slug ) || empty( $secret ) || '' === $filename ) { return null; } $body = Sentinel_Clone_Primary::request_secondary( array( 'type' => 'clone_status', 'file_name' => $filename ), $url, $ip, $slug, $secret ); if ( ! is_array( $body ) || empty( $body['ok'] ) || ! array_key_exists( 'zip_present', $body ) ) { return null; } $extracted = (string) ( $body['zip_extracted'] ?? 'no' ); $present = ! empty( $body['zip_present'] ) && ( $filesize < 1 || (int) ( $body['zip_size'] ?? 0 ) === $filesize ); return array( 'confirmed' => ( $present || 'yes' === $extracted ), 'extracted' => $extracted, 'zip_size' => (int) ( $body['zip_size'] ?? 0 ), ); } /** * The files made it, the database did not: check with the secondary that the ZIP really is there, * then run only the database phase. Works even when the saved progress record is gone. */ public static function continue_db_only(): array { if ( get_option( 'wardpair_role' ) !== 'primary' || ! function_exists( 'as_enqueue_async_action' ) ) { return array( 'ok' => false, 'message' => __( 'Only a paired primary can do this.', 'wardpair' ) ); } $filename = (string) get_option( 'wardpair_clone_file_backup_file', '' ); if ( '' === $filename ) { return array( 'ok' => false, 'message' => __( 'No file transfer is on record, so there is nothing to continue from. Start a full clone.', 'wardpair' ) ); } $path = WARDPAIR_BACKUP_DIR . '/' . basename( $filename ); $st = self::confirm_files_on_secondary( $filename, is_file( $path ) ? (int) filesize( $path ) : 0 ); if ( null === $st ) { return array( 'ok' => false, 'message' => __( 'The secondary did not answer, so I cannot confirm the files arrived. Try again in a minute; nothing was changed.', 'wardpair' ) ); } if ( ! $st['confirmed'] ) { return array( 'ok' => false, 'message' => __( 'The secondary does not have the files ZIP, so the files have to be sent first. Use "Resume from last progress" or "Start over". Nothing was changed.', 'wardpair' ) ); } // Remove only database-phase leftovers; the files phase is left alone. foreach ( array_merge( self::PHASE_HOOKS['db_dump'], self::PHASE_HOOKS['db_transfer'], array( 'wardpair_clone_check_backup' ) ) as $hook ) { if ( function_exists( 'as_unschedule_all_actions' ) ) { as_unschedule_all_actions( $hook ); } } self::clear_retry_counters(); delete_option( 'wardpair_clone_db_started' ); delete_option( 'wardpair_clone_db_check_count' ); delete_option( 'wardpair_clone_backup_file' ); delete_option( 'wardpair_clone_job_key' ); delete_option( 'wardpair_clone_db_only' ); // The secondary still has to wait for the ZIP extraction. update_option( 'wardpair_clone_complete', false ); update_option( 'wardpair_clone_files_complete', true ); update_option( 'wardpair_clone_files_started', true ); self::progress_clear(); self::progress_set( array( 'phase' => self::PHASE_DB_DUMP, 'status' => 'active', 'job_key' => '', 'filename' => '', 'filesize' => 0, 'chunk_size' => 0, 'total_chunks' => 0, 'cursor' => 0, 'idle_resumes' => 0, 'last_resume_cursor' => -1, 'note' => '', ) ); Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_OK, 'Confirmed with the secondary: the files ZIP is already there (extraction: ' . $st['extracted'] . '). Continuing with the database only.' ); Sentinel_Core::log_event( 'clone_db_continue', 'Database-only continuation started after confirming the files ZIP on the secondary.' ); as_enqueue_async_action( 'wardpair_clone_db_start', array(), 'wardpair' ); Sentinel_Cron::kick(); $msg = __( 'Confirmed: the secondary already has the files. Only the database is being sent.', 'wardpair' ); if ( 'failed' === $st['extracted'] ) { $msg .= ' ' . __( 'Note: the secondary reported that extracting the ZIP failed. If the site looks incomplete afterwards, run "Check and repair" or a full clone.', 'wardpair' ); } return array( 'ok' => true, 'message' => $msg ); } /** Which build hook continues this job, given the status it stopped in. */ private static function next_build_hook( string $job_key, array $job, bool $zip ): string { $status = (string) ( $job['status'] ?? '' ); $index = (int) ( $job['chunk_index'] ?? 0 ); $count = isset( $job['chunks'] ) && is_array( $job['chunks'] ) ? count( $job['chunks'] ) : 0; if ( $zip ) { if ( 0 === strpos( $job_key, 'wardpair_clonesnap_' ) ) { if ( 'diffing' === $status ) { return 'wardpair_clone_snapshot_diff'; } return $index < $count ? 'wardpair_clone_snapshot_apply' : 'wardpair_clone_snapshot_seal'; } if ( 'inventory' === $status ) { return 'wardpair_file_backup_inventory'; } return $index < $count ? 'wardpair_file_backup_chunk' : 'wardpair_file_backup_seal'; } if ( 'inventory' === $status ) { return 'wardpair_db_backup_inventory'; } return $index < $count ? 'wardpair_db_backup_chunk' : 'wardpair_db_backup_seal'; } public static function release_stuck_claims(): void { global $wpdb; // Transfer jobs: release if stuck in-progress > 3 minutes. $released = (int) $wpdb->query( $wpdb->prepare( "UPDATE %i a JOIN %i c ON a.claim_id = c.claim_id SET a.status = 'pending', a.claim_id = 0 WHERE a.hook IN ( 'wardpair_clone_transfer_file_chunk', 'wardpair_clone_transfer_chunk' ) AND a.status = 'in-progress' AND c.date_created_gmt < DATE_SUB(UTC_TIMESTAMP(), INTERVAL 3 MINUTE)", $wpdb->prefix . 'actionscheduler_actions', $wpdb->prefix . 'actionscheduler_claims' ) ); // Backup chunks run with set_time_limit(300) and append to one dump file, so // releasing a LIVE run at 3 min (claim age, not action age) starts a second // writer on the same file. Wait past the PHP limit. $released += (int) $wpdb->query( $wpdb->prepare( "UPDATE %i a JOIN %i c ON a.claim_id = c.claim_id SET a.status = 'pending', a.claim_id = 0 WHERE a.hook IN ( 'wardpair_db_backup_chunk', 'wardpair_file_backup_chunk' ) AND a.status = 'in-progress' AND c.date_created_gmt < DATE_SUB(UTC_TIMESTAMP(), INTERVAL 7 MINUTE)", $wpdb->prefix . 'actionscheduler_actions', $wpdb->prefix . 'actionscheduler_claims' ) ); // Clone snapshot phases: if one of these dies silently (host timeout/kill), Action // Scheduler marks it failed/timed-out but nothing queues the next phase, so the // clone polls forever at the same chunk. Release only after the PHP limit // (set_time_limit 300) has certainly passed, so a live run is never doubled. $released += (int) $wpdb->query( $wpdb->prepare( "UPDATE %i a JOIN %i c ON a.claim_id = c.claim_id SET a.status = 'pending', a.claim_id = 0 WHERE a.hook IN ( 'wardpair_clone_snapshot_diff', 'wardpair_clone_snapshot_apply', 'wardpair_clone_snapshot_seal' ) AND a.status = 'in-progress' AND c.date_created_gmt < DATE_SUB(UTC_TIMESTAMP(), INTERVAL 7 MINUTE)", $wpdb->prefix . 'actionscheduler_actions', $wpdb->prefix . 'actionscheduler_claims' ) ); // Delete the orphaned claim records. $wpdb->query( $wpdb->prepare( "DELETE c FROM %i c WHERE NOT EXISTS ( SELECT 1 FROM %i a WHERE a.claim_id = c.claim_id AND a.status = 'in-progress' )", $wpdb->prefix . 'actionscheduler_claims', $wpdb->prefix . 'actionscheduler_actions' ) ); if ( $released ) { Sentinel_Sync::log( Sentinel_Sync::DIR_PUSH, 'clone', Sentinel_Sync::STATUS_OK, "Watchdog: released {$released} stuck clone job(s) back to pending." ); } } }