8, 'max_chunk' => 100, 'max_retries' => 3, 'backoff_s' => array( 60, 300, 900 ), ); } /** * Read the next records of the file. While the scan is not complete the chunk is empty and the scan advances. * * @param Job $job Job. * @param array $cursor Cursor data. * @param int $max Engine suggestion. * @return Chunk * @throws \RuntimeException When the source file is gone. */ protected function readChunk( Job $job, array $cursor, int $max ): Chunk { $p = JobParams::of( $job ); if ( ! FileStore::hasSource( $p->ref() ) ) { throw new \RuntimeException( 'The import file no longer exists.' ); } $row = (int) ( $cursor['row'] ?? 1 ); $off = (int) ( $cursor['off'] ?? $p->headerEnd() ); if ( ! ScanBuilder::ready( $p ) ) { ScanBuilder::advance( $p, 4.0 ); return new Chunk( array(), array( 'row' => $row, 'off' => $off, ), false, $p->rows() ); } $size = ChunkTuner::next( $job->id, $max ); // A worker may run many chunks: do not let the in-memory object cache of earlier chunks pile up (P09-FR-013). // WordPress defines wp_cache_flush_runtime() for every cache; an object cache without the feature makes it complain. if ( function_exists( 'wp_cache_supports' ) && wp_cache_supports( 'flush_runtime' ) ) { wp_cache_flush_runtime(); } $reader = new CsvReader( $p->path(), $p->delimiter() ); $reader->seek( $off ); $items = array(); $done = false; $n_items = 0; while ( $n_items < $size ) { $rec = $reader->read(); if ( null === $rec ) { $done = true; break; } ++$row; $off = $rec['next']; if ( $rec['blank'] ) { continue; } ++$n_items; $items[] = new ItemRef( 'r' . $row, array( 'row' => $row, 'off' => $rec['off'], 'cells' => $rec['cells'], ) ); } if ( ! $done ) { $done = null === $reader->read(); // Peek: the file may end right after this chunk. } return new Chunk( $items, array( 'row' => $row, 'off' => $off, ), $done, $p->rows() ); } /** * Store the outcome of a row in job_rows (failures, skips and warnings only) and map it to an engine result. * * @param Job $job Job. * @param RowContext $c Row. * @param RowResult $r Result. * @return ItemResult */ protected function recordResult( Job $job, RowContext $c, RowResult $r ): ItemResult { if ( $c->replay ) { JobRowsRepo::forget( $job->id, $c->row ); // The outcome an interrupted attempt recorded is replaced by this one. } foreach ( $r->warnings as $w ) { if ( $w instanceof Issue ) { JobRowsRepo::record( $job->id, $c->row, $c->offset, 'warning', $w->code, $w->detail, $r->productId ); } } switch ( $r->kind ) { case RowResult::FAILED: JobRowsRepo::record( $job->id, $c->row, $c->offset, 'failed', $r->code, $r->detail, $r->productId ); return ItemResult::failed( $r->code, $r->detail ); case RowResult::SKIPPED: JobRowsRepo::record( $job->id, $c->row, $c->offset, 'skipped', $r->code, $r->detail ); return ItemResult::skipped( $r->code, $r->detail ); } return ItemResult::done( $r->ref() ); } /** * A row the job engine failed on its own (interrupted repeatedly, undecidable after an interruption, retries used up, * memory, retry data too large or unreadable) never reaches recordResult(): record it like any failed row, so it is * listed, in the failed-rows CSV and can be retried. Hooked to catalogrelay_core_job_item_failed. No message: it could * carry cell values. * * @param int $jobId Job. * @param string $jobType Job type. * @param array $payload Item payload {row, off, cells}. * @param string $code Engine code. * @param string $key Item key ("r" and the row number for a file row). * @return void */ public static function engineFailed( int $jobId, string $jobType, array $payload, string $code, string $key = '' ): void { if ( ! in_array( $jobType, array( ImportJobType::TYPE, RetryJobType::TYPE, DryRunJobType::TYPE ), true ) ) { return; } if ( ! isset( $payload['row'], $payload['off'] ) ) { $payload = self::locate( $jobId, $key ); // The stored data of the item could not be read (payload_lost). if ( ! $payload ) { return; // Not a file row (rollback preview), or its position cannot be found. } } $row = (int) $payload['row']; $code = 'needs_review' === $code ? 'reconcile_uncertain' : $code; if ( RetryJobType::TYPE === $jobType ) { $job = JobStore::find( Container::requireCurrent(), $jobId ); if ( $job ) { JobRowsRepo::refail( JobParams::of( $job )->retryOf(), $row, $code, '' ); // The root job keeps the authoritative failed row. } } JobRowsRepo::record( $jobId, $row, (int) $payload['off'], 'failed', $code ); } /** * Position of a file row found again from its item key: the row's record is read from the start of the file (rows are * numbered as readChunk() counts them). Only for an item whose stored data was lost, which is rare. * * @param int $jobId Job. * @param string $key Item key. * @return array {row, off} or empty. */ private static function locate( int $jobId, string $key ): array { if ( ! preg_match( '/^r([0-9]+)$/', $key, $m ) ) { return array(); } $job = JobStore::find( Container::requireCurrent(), $jobId ); $p = $job ? JobParams::of( $job ) : null; if ( ! $p || 'rollback_preview' === $p->mode() || ! FileStore::hasSource( $p->ref() ) ) { return array(); } $want = (int) $m[1]; $reader = new CsvReader( $p->path(), $p->delimiter() ); $reader->seek( $p->headerEnd() ); for ( $row = 2; $row <= $want; $row++ ) { $rec = $reader->read(); if ( null === $rec ) { return array(); } if ( $row === $want ) { return $rec['blank'] ? array() : array( 'row' => $want, 'off' => (int) $rec['off'], ); } } return array(); } /** * Whether a row item was started (and interrupted) before: a crash replay or a retry attempt. * * @param int $jobId Job. * @param string $key Item key. * @return bool */ protected static function isReplay( int $jobId, string $key ): bool { global $wpdb; return (int) $wpdb->get_var( $wpdb->prepare( "SELECT crash_count + attempt - 1 FROM {$wpdb->prefix}catalogrelay_job_items WHERE job_id = %d AND item_key = %s", $jobId, $key ) ) > 0; // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name from the prefix. } /** * Map a WooCommerce exception to a row code. * * @param \WC_Data_Exception $e Exception. * @return string */ protected static function wcCode( \WC_Data_Exception $e ): string { switch ( $e->getErrorCode() ) { case 'product_invalid_sku': return 'sku_exists_other'; case 'product_invalid_global_unique_id': return 'gtin_conflict'; } return 'write_failed'; } /** * Count rows per receipt kind from job_items. * * @param int $jobId Job. * @return array c, u, n, recovered counts. */ public static function receiptCounts( int $jobId ): array { global $wpdb; $out = array( 'c' => 0, 'u' => 0, 'n' => 0, 'recovered' => 0, ); $rows = (array) $wpdb->get_results( $wpdb->prepare( "SELECT LEFT(result_ref, 1) AS k, COUNT(*) AS n FROM {$wpdb->prefix}catalogrelay_job_items WHERE job_id = %d AND status = 'done' GROUP BY LEFT(result_ref, 1)", $jobId ), ARRAY_A ); // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name from the prefix. foreach ( $rows as $r ) { $k = (string) $r['k']; if ( isset( $out[ $k ] ) ) { $out[ $k ] += (int) $r['n']; } elseif ( '' === $k ) { $out['recovered'] += (int) $r['n']; } } return $out; } }