column index), options, mode (import_diff|preflight). * @return int Job id. * @throws JobRefused With a code. */ public static function createDryRun( array $in ): int { $p = self::base( $in ); $mode = isset( $in['mode'] ) && 'preflight' === $in['mode'] ? 'preflight' : 'import_diff'; $p = array_merge( array( 'v' => JobParams::VERSION, 'mode' => $mode, ), $p ); return JobEngine::enqueue( DryRunJobType::TYPE, $p, array( 'inline_s' => (int) ( $in['inline_s'] ?? 0 ) ) ); } /** * Create an import job. * * @param array $in upload_ref, mapping, options, plan_id, plan_hash, require_plan (bool, default true), inline_s. * @return array {job_id, existing} * @throws JobRefused With a code. */ public static function createImport( array $in ): array { $d = Container::requireCurrent(); $base = self::base( $in ); $p = new JobParams( $base ); $blocked = false; if ( ! isset( $in['require_plan'] ) || $in['require_plan'] ) { $plan_id = (int) ( $in['plan_id'] ?? 0 ); if ( $plan_id < 1 || ! PlanPreview::verifyFor( $d, $plan_id, (string) ( $in['plan_hash'] ?? '' ) ) ) { throw new JobRefused( 'preview_required', __( 'Run the check and preview first: the import needs a current preview of this exact file and mapping.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $summary = PlanPreview::summaryFor( $d, $plan_id ); $meta = (array) ( $summary['meta'] ?? array() ); // The delimiter splits the same bytes into other cells: a preview read with another one does not count. if ( ( $meta['sha256'] ?? '' ) !== $p->sha256() || ( $meta['config_hash'] ?? '' ) !== $p->configHash() || (string) ( $meta['delimiter'] ?? $p->delimiter() ) !== $p->delimiter() ) { throw new JobRefused( 'preview_mismatch', __( 'The preview was made for a different file, mapping or options. Run the check again.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $errors = array_sum( (array) ( $summary['summary']['errors'] ?? array() ) ); if ( 'block_all' === $p->option( 'on_error' ) && $errors > 0 ) { $blocked = true; } } elseif ( 'block_all' === $p->option( 'on_error' ) ) { throw new JobRefused( 'preview_required', __( 'Blocking the import on errors needs a preview.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $idem = sha1( 'import|' . $p->sha256() . '|' . $p->configHash() . '|' . $p->delimiter() ); $lock = 'catalogrelay_lock_' . $idem; $old = self::active( ImportJobType::TYPE, $idem ); if ( $old > 0 ) { return array( 'job_id' => $old, 'existing' => true, ); } if ( ! self::lock( $lock ) ) { $again = self::active( ImportJobType::TYPE, $idem ); if ( $again > 0 ) { return array( 'job_id' => $again, 'existing' => true, ); } throw new JobRefused( 'busy', __( 'The same import is being created. Wait a moment.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } try { // The request that held the lock before may have queued the job after the check above. $again = self::active( ImportJobType::TYPE, $idem ); if ( $again > 0 ) { return array( 'job_id' => $again, 'existing' => true, ); } self::checkCap(); $batch_id = 0; $rollbackable = false; if ( ! $blocked ) { $batch = self::openBatch( 'catalogrelay.import', $p, ! empty( $in['no_rollback_point'] ) ); $batch_id = $batch['id']; $rollbackable = $batch['rollbackable']; } $params = array_merge( $base, array( 'idem' => $idem, 'retry_of' => 0, 'batch_id' => $batch_id, 'rollbackable' => $rollbackable, 'root_started' => gmdate( 'Y-m-d H:i:s' ), 'root_key' => wp_generate_uuid4(), 'blocked' => $blocked, ) ); $job_id = self::enqueue( ImportJobType::TYPE, $params, array( 'inline_s' => (int) ( $in['inline_s'] ?? 0 ) ) ); $orig = FileStore::path( $p->ref(), 'original.upload' ); if ( is_file( $orig ) ) { wp_delete_file( $orig ); } return array( 'job_id' => $job_id, 'existing' => false, ); } finally { delete_option( $lock ); } } /** * Create a job that retries the failed rows of an import. * * @param int $jobId Import (or earlier retry) job. * @param array $opts include_codes (string[]), inline_s, no_rollback_point (confirmed: the rollback storage is full). * @return array {job_id, existing} * @throws JobRefused With a code. */ public static function createRetry( int $jobId, array $opts = array() ): array { $d = Container::requireCurrent(); $job = JobStore::find( $d, $jobId ); if ( ! $job || ! in_array( $job->type, array( ImportJobType::TYPE, RetryJobType::TYPE ), true ) ) { throw new JobRefused( 'not_found', __( 'The import job does not exist.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $root_id = (int) ( $job->params['retry_of'] ?? 0 ); $root_id = $root_id > 0 ? $root_id : $job->id; $root = JobStore::find( $d, $root_id ); if ( ! $root || ! $root->isTerminal() ) { throw new JobRefused( 'not_finished', __( 'Wait until the import has finished.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $codes = array_values( array_filter( array_map( 'sanitize_key', (array) ( $opts['include_codes'] ?? array() ) ) ) ); $idem = sha1( 'retry|' . $root_id . '|' . implode( ',', $codes ) ); $lock = self::chainLock( $root_id ); if ( ! self::lock( $lock ) ) { $again = self::active( RetryJobType::TYPE, $idem ); if ( $again > 0 ) { return array( 'job_id' => $again, 'existing' => true, ); } throw new JobRefused( 'busy', __( 'A retry or a rollback of this import is being started. Wait a moment.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } try { return self::queueRetry( $root, $idem, $codes, $opts ); } finally { delete_option( $lock ); } } /** * Queue a retry under the lock of the import's chain (no second retry, and no rollback confirmation in between). * * @param Job $root Import job. * @param string $idem Idempotency key. * @param string[] $codes Codes to retry (empty = every failed row). * @param array $opts createRetry() options. * @return array {job_id, existing} * @throws JobRefused With a code. */ private static function queueRetry( Job $root, string $idem, array $codes, array $opts ): array { $root_id = $root->id; $old = self::active( RetryJobType::TYPE, $idem ); if ( $old > 0 ) { return array( 'job_id' => $old, 'existing' => true, ); } if ( '' !== StatusPresenter::rollbackState( $root ) ) { throw new JobRefused( 'rolled_back', __( 'This import has been rolled back, so its failed rows are not retried.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $rp = JobParams::of( $root ); if ( ! FileStore::hasSource( $rp->ref() ) ) { throw new JobRefused( 'source_expired', __( 'The import file has been deleted after its retention period, so failed rows cannot be retried. The failed-rows CSV can still be downloaded and imported again.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $count = JobRowsRepo::failedCount( $root_id, $codes ); if ( 0 === $count ) { throw new JobRefused( 'nothing_to_retry', __( 'There are no failed rows to retry.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } self::checkCap(); // The chain is rolled back as a whole: when it can no longer be (no rollback point, or its snapshots expired), the // retry's snapshots could never be used. $no_point = ! empty( $opts['no_rollback_point'] ) || in_array( RollbackService::blocker( $root ), array( 'no_rollback_point', 'expired' ), true ); $batch = self::openBatch( 'catalogrelay.retry', $rp, $no_point ); $params = array( 'v' => JobParams::VERSION, 'idem' => $idem, 'retry_of' => $root_id, 'batch_id' => $batch['id'], 'rollbackable' => $batch['rollbackable'], 'root_started' => $rp->rootStarted(), 'root_key' => $rp->rootKey(), 'failed_total' => $count, 'include_codes' => $codes, 'file' => $rp->data['file'], 'mapping' => $rp->data['mapping'], 'options' => $rp->data['options'], ); // The retry writes the import's rows as the import's user, not as whoever asks for it (DEVIATIONS 33, 77). $job_id = self::enqueue( RetryJobType::TYPE, $params, array( 'inline_s' => (int) ( $opts['inline_s'] ?? 0 ), 'creator' => $root->createdBy, ) ); return array( 'job_id' => $job_id, 'existing' => false, ); } /** * Validate the common input and build the frozen part of the job parameters. * * @param array $in Input. * @return array * @throws JobRefused With a code. */ private static function base( array $in ): array { $ref = (string) ( $in['upload_ref'] ?? '' ); $meta = FileStore::meta( $ref ); if ( ! $meta || ! FileStore::hasSource( $ref ) ) { throw new JobRefused( 'upload_missing', __( 'The uploaded file no longer exists. Upload it again.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $map = array(); foreach ( (array) ( $in['mapping'] ?? array() ) as $field => $index ) { if ( FieldRegistry::isField( (string) $field ) && is_numeric( $index ) && (int) $index >= 0 && (int) $index < count( (array) $meta['header'] ) ) { $map[ (string) $field ] = (int) $index; } } if ( ! AutoMapper::keyOk( $map ) ) { throw new JobRefused( 'no_match_key', __( 'Map at least one matching column: ID or SKU.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $o = (array) ( $in['options'] ?? array() ); $options = JobParams::defaultOptions(); $options['mode'] = in_array( $o['mode'] ?? '', array( 'create_update', 'update_only', 'create_only' ), true ) ? $o['mode'] : 'create_update'; $options['on_error'] = 'block_all' === ( $o['on_error'] ?? '' ) ? 'block_all' : 'import_valid'; $options['only_fields'] = array(); foreach ( array_map( 'strval', (array) ( $o['only_fields'] ?? array() ) ) as $f ) { if ( 0 === strpos( $f, FieldRegistry::ATTR_PREFIX ) && FieldRegistry::isField( $f ) ) { $f = 'attributes'; // The planner writes a row's attributes as a whole: any attribute column selects them. } if ( FieldRegistry::isField( $f ) || 'attributes' === $f ) { $options['only_fields'][ $f ] = $f; } } if ( ! $options['only_fields'] && ! empty( $o['only_fields'] ) ) { // An empty list means "every field": a choice that names no known field must not turn into that. throw new JobRefused( 'no_fields', __( 'None of the fields chosen for the update is known. Choose the fields again.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } $options['only_fields'] = array_values( $options['only_fields'] ); $dec = isset( $o['decimal'] ) && in_array( $o['decimal'], array( '.', ',' ), true ) ? (string) $o['decimal'] : (string) wc_get_price_decimal_separator(); $th = isset( $o['thousand'] ) && in_array( $o['thousand'], array( '.', ',', ' ', '' ), true ) ? (string) $o['thousand'] : (string) wc_get_price_thousand_separator(); if ( $th === $dec ) { $th = '.' === $dec ? ',' : '.'; } $options['decimal'] = $dec; $options['thousand'] = $th; return array( 'file' => array( 'ref' => $ref, 'sha256_norm' => (string) $meta['sha256_norm'], 'rows' => (int) $meta['rows'], 'header_end' => (int) $meta['header_end'], 'encoding' => (string) $meta['encoding'], 'delimiter' => (string) $meta['delimiter'], 'header' => array_map( 'strval', (array) $meta['header'] ), 'name' => (string) ( $meta['name'] ?? '' ), ), 'mapping' => array( 'map' => $map ), 'options' => $options, ); } /** * Open the ledger batch of a job. * * @param string $type Batch type. * @param JobParams $p Params (for the label). * @param bool $noPoint Continue without a rollback point when the quota is full. * @return array {id, rollbackable} * @throws JobRefused 'snapshot_quota' when the quota is full and no confirmation was given. */ private static function openBatch( string $type, JobParams $p, bool $noPoint ): array { $d = Container::requireCurrent(); $days = max( 1, min( 30, (int) SettingsStore::get( $d, 'retention.snapshot_days', 7 ) ) ); try { if ( $noPoint ) { $b = Ledger::beginBatchFor( $d, $type, array( 'label' => (string) ( $p->data['file']['name'] ?? '' ), 'rollbackable' => false, 'snapshots' => false, ) ); return array( 'id' => $b->id, 'rollbackable' => false, ); } $b = Ledger::beginBatchFor( $d, $type, array( 'label' => (string) ( $p->data['file']['name'] ?? '' ), 'expires_at' => gmdate( 'Y-m-d H:i:s', time() + $days * DAY_IN_SECONDS ), ) ); return array( 'id' => $b->id, 'rollbackable' => true, ); } catch ( SnapshotQuotaExceeded $e ) { throw new JobRefused( 'snapshot_quota', __( 'The rollback storage is full until older rollback points expire. Confirm to continue without a rollback point.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } } /** * Lock name shared by the creation of a retry and the confirmation of a rollback of one import. * * @param int $rootId Import job. * @return string */ public static function chainLock( int $rootId ): string { return 'catalogrelay_lock_chain_' . $rootId; } /** * Take a short lock (an option; one left behind by a request that died is taken over after 10 s). * * @param string $name Option name. * @return bool Whether the lock was taken. */ public static function lock( string $name ): bool { if ( add_option( $name, time(), '', false ) ) { return true; } if ( (int) get_option( $name, 0 ) > time() - 10 ) { return false; } update_option( $name, time(), false ); return true; } /** * Queue a writing job; when the job could not be created (Action Scheduler cannot be used, or the job row could not be * stored) the ledger batch just opened for it is expired, so no open batch without a job is left behind. A job that * was created before a later step threw keeps its batch. * * @param string $type Job type. * @param array $params Params with batch_id and idem (callers hold the lock of that key). * @param array $opts Engine options. * @return int Job id. * @throws JobsUnavailable When Action Scheduler cannot be used. * @throws \Throwable Whatever else the engine throws. */ private static function enqueue( string $type, array $params, array $opts ): int { try { return JobEngine::enqueue( $type, $params, $opts ); } catch ( \Throwable $e ) { if ( (int) $params['batch_id'] > 0 && ( $e instanceof JobsUnavailable || 0 === self::active( $type, (string) $params['idem'] ) ) ) { Ledger::expireBatch( (int) $params['batch_id'] ); } throw $e; } } /** * Refuse a job that would push the row logs and snapshots over the site cap (P09-NFR-012). * * @return void * @throws JobRefused 'row_cap'. */ private static function checkCap(): void { global $wpdb; $tables = array( $wpdb->prefix . 'catalogrelay_job_rows', $wpdb->prefix . 'catalogrelay_ledger' ); $expiry = $wpdb->get_var( "SHOW SESSION VARIABLES LIKE 'information_schema_stats_expiry'", 1 ); if ( null !== $expiry && '0' !== (string) $expiry ) { // MySQL 8 caches TABLE_ROWS for a day by default: read the storage engine's current estimate instead. $wpdb->query( 'SET SESSION information_schema_stats_expiry = 0' ); } $count = (int) $wpdb->get_var( $wpdb->prepare( 'SELECT COALESCE(SUM(TABLE_ROWS), 0) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (%s, %s)', $tables[0], $tables[1] ) ); $count = (int) apply_filters( 'catalogrelay_row_count', $count ); $cap = (int) apply_filters( 'catalogrelay_row_cap', self::ROW_CAP ); if ( $count >= $cap ) { throw new JobRefused( 'row_cap', __( 'The site keeps too many import log rows and rollback snapshots (over 1,000,000). They are deleted when their retention period ends; start the import again later.', 'catalogrelay' ) ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- the message is returned as JSON or escaped on render, never echoed raw. } } /** * An unfinished job of a type with an idempotency key (also one waiting for attention: it can still be resumed). * * @param string $type Job type. * @param string $idem Key. * @return int Job id or 0. */ public static function active( string $type, string $idem ): int { global $wpdb; $done = "'" . implode( "', '", Job::TERMINAL ) . "'"; // Constants, not user input. return (int) $wpdb->get_var( $wpdb->prepare( "SELECT id FROM {$wpdb->prefix}catalogrelay_jobs WHERE job_type = %s AND status NOT IN ({$done}) AND params_json LIKE %s ORDER BY id DESC LIMIT 1", $type, '%"idem":"' . $wpdb->esc_like( $idem ) . '"%' ) ); // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name from the prefix, fixed status list. } }