array( $jobId, $row, $offset, $productId, $slot ), 'file' => '', ); try { return self::download( $jobId, $row, $offset, $productId, $slot, $url, $attempt ); } catch ( \Throwable $e ) { self::giveUp( $jobId, $row, $offset, $productId, $slot, 'image_failed', 'error' ); return 'failed'; } finally { self::dropFile(); self::$running = null; } } /** * The download itself. * * @param int $jobId Job. * @param int $row Row number. * @param int $offset Byte offset of the CSV record. * @param int $productId Product. * @param int $slot Slot. * @param string $url URL. * @param int $attempt Attempt number. * @return string */ private static function download( int $jobId, int $row, int $offset, int $productId, int $slot, string $url, int $attempt ): string { if ( ! self::productExists( $productId ) ) { JobRowsRepo::bump( $jobId, 'images_done' ); return 'skipped'; } $job = JobStore::find( Plugin::descriptor(), $jobId ); $hash = UrlNormalizer::hash( $url ); $created = 0; // This site's own media (a WooCommerce product export lists it) and a re-import of the same URL reuse the attachment // instead of downloading a duplicate. if ( 0 === self::localAttachment( $url ) && 0 === self::existing( $productId, $hash ) ) { $res = ( new SafeFetcher() )->fetch( $url ); if ( empty( $res['ok'] ) ) { if ( 'image_failed' === $res['code'] && $attempt < 3 && ImageQueue::retryLater( array( $jobId, $row, $offset, $productId, $slot, $url, $attempt ), 1 === $attempt ? 120 : 600 ) ) { return 'retry'; } self::giveUp( $jobId, $row, $offset, $productId, $slot, (string) $res['code'], (string) $res['detail'] ); return 'image_blocked' === $res['code'] ? 'blocked' : 'failed'; } self::$running['file'] = (string) $res['file']; $author = $job && $job->createdBy > 0 && false !== get_userdata( $job->createdBy ) ? $job->createdBy : 0; $created = self::sideload( (string) $res['file'], (string) $res['mime'], $url, $productId, $hash, $author ); if ( is_wp_error( $created ) ) { self::giveUp( $jobId, $row, $offset, $productId, $slot, 'image_failed', 'sideload' ); return 'failed'; } } $urls = $job ? self::rowUrls( JobParams::of( $job ), $offset ) : array(); $full = isset( $urls[ $slot ] ) && UrlNormalizer::hash( $urls[ $slot ] ) === $hash; if ( ! self::attach( $productId, $hash, (int) $created, $full ? $urls : array( $slot => $url ), $full ) ) { // Another download of this product holds its lock: place the image a minute later (not a new attempt). if ( ImageQueue::add( array( $jobId, $row, $offset, $productId, $slot, $url, $attempt ), time() + 60 ) ) { return 'retry'; } self::giveUp( $jobId, $row, $offset, $productId, $slot, 'image_failed', 'locked' ); return 'failed'; } global $wpdb; // A retried image that now works clears its earlier warning (a warning resolved before is replaced: one row per key). $wpdb->query( $wpdb->prepare( "DELETE FROM {$wpdb->prefix}catalogrelay_job_rows WHERE job_id = %d AND row_no = %d AND status = 'resolved' AND code IN ('image_failed', 'image_blocked') AND fields = %s", $jobId, $row, 'images:' . $slot ) ); // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name from the prefix. $wpdb->query( $wpdb->prepare( "UPDATE {$wpdb->prefix}catalogrelay_job_rows SET status = 'resolved' WHERE job_id = %d AND row_no = %d AND status = 'warning' AND code IN ('image_failed', 'image_blocked') AND fields = %s", $jobId, $row, 'images:' . $slot ) ); // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name from the prefix. JobRowsRepo::bump( $jobId, 'images_done' ); return 'ok'; } /** * End a download that failed: a warning for its slot ("retry images" offers it) and one more done download. * * @param int $jobId Job. * @param int $row Row number. * @param int $offset Byte offset of the CSV record. * @param int $productId Product. * @param int $slot Slot. * @param string $code image_failed|image_blocked. * @param string $detail Detail. * @return void */ public static function giveUp( int $jobId, int $row, int $offset, int $productId, int $slot, string $code, string $detail ): void { JobRowsRepo::record( $jobId, $row, $offset, 'warning', $code, 'images:' . $slot . ( '' !== $detail ? ' ' . $detail : '' ), $productId, 'images:' . $slot ); JobRowsRepo::bump( $jobId, 'images_done' ); } /** * Action Scheduler gave up on a download that never returned: its process died (fatal error, time limit: * action_scheduler_unexpected_shutdown, in that process) or was killed (action_scheduler_failed_action, once the claim * timed out). The download ends with its warning, so the job no longer waits for it and "retry images" offers it. * * @param int|string $actionId Action. * @return void */ public static function abandoned( $actionId ): void { if ( ! class_exists( 'ActionScheduler' ) ) { return; } $action = \ActionScheduler::store()->fetch_action( (int) $actionId ); if ( ImageQueue::HOOK !== $action->get_hook() ) { return; } $a = array_values( (array) $action->get_args() ); if ( count( $a ) < 5 ) { return; } if ( self::$running && array( (int) $a[0], (int) $a[1], (int) $a[2], (int) $a[3], (int) $a[4] ) === self::$running['args'] ) { self::dropFile(); self::$running = null; } self::giveUp( (int) $a[0], (int) $a[1], (int) $a[2], (int) $a[3], (int) $a[4], 'image_failed', 'error' ); } /** * Whether the product still exists (not trashed): a cheap check, the product itself is loaded under the lock. * * @param int $id Product. * @return bool */ private static function productExists( int $id ): bool { if ( $id < 1 ) { return false; } wp_cache_delete( $id, 'posts' ); $status = get_post_status( $id ); return false !== $status && 'trash' !== $status && in_array( get_post_type( $id ), array( 'product', 'product_variation' ), true ); } /** * Delete the temporary file of the running download (moved into the uploads when it was stored). * * @return void */ private static function dropFile(): void { $file = self::$running ? (string) self::$running['file'] : ''; if ( '' !== $file && file_exists( $file ) ) { wp_delete_file( $file ); } } /** * The image URLs of the row (slot = index), read again from the import's file; none when it is gone. * * @param JobParams $p Params of the job. * @param int $offset Byte offset of the CSV record. * @return string[] */ private static function rowUrls( JobParams $p, int $offset ): array { $idx = $p->mapping()['images'] ?? null; $path = $p->path(); if ( null === $idx || ! is_readable( $path ) ) { return array(); } $cells = ( new CsvReader( $path, $p->delimiter() ) )->at( $offset ); return null !== $cells && isset( $cells[ $idx ] ) ? RowParser::splitList( (string) $cells[ $idx ] ) : array(); } /** * Image attachment of this site that a URL points to (a URL under the uploads directory), or 0. * * @param string $url URL. * @return int */ private static function localAttachment( string $url ): int { $base = (string) preg_replace( '#^https?:#i', '', (string) wp_get_upload_dir()['baseurl'] ); $plain = (string) preg_replace( '#^https?:#i', '', trim( $url ) ); if ( '' === $base || 0 !== stripos( $plain, trailingslashit( $base ) ) ) { return 0; } $id = (int) attachment_url_to_postid( trim( $url ) ); return $id > 0 && wp_attachment_is_image( $id ) ? $id : 0; } /** * Attachments this importer added to the product: URL hash => attachment ids, oldest first. * * @param int $productId Product. * @return array */ private static function own( int $productId ): array { global $wpdb; $rows = (array) $wpdb->get_results( $wpdb->prepare( "SELECT p.ID, m.meta_value FROM {$wpdb->posts} p INNER JOIN {$wpdb->postmeta} m ON m.post_id = p.ID AND m.meta_key = %s WHERE p.post_parent = %d AND p.post_type = 'attachment' AND p.post_status = 'inherit' ORDER BY p.ID", self::META_URL_HASH, $productId ), ARRAY_A ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery -- one product's attachments; bulk mode keeps stale caches. $out = array(); foreach ( $rows as $r ) { $out[ (string) $r['meta_value'] ][] = (int) $r['ID']; } return $out; } /** * Attachment of the product that was downloaded from the same (normalized) URL (the oldest one), or 0. * * @param int $productId Product. * @param string $hash URL hash. * @return int */ private static function existing( int $productId, string $hash ): int { return self::own( $productId )[ $hash ][0] ?? 0; } /** * Whether the product already shows these images as an import would lay them out (nothing to download or save). * * @param \WC_Product $product Product. * @param string[] $urls URLs of the row (slot = index). * @return bool */ public static function inPlace( \WC_Product $product, array $urls ): bool { $want = self::layout( $product, array_values( $urls ), true ); return ! $want['missing'] && (int) $product->get_image_id( 'edit' ) === $want['image'] && array_map( 'intval', (array) $product->get_gallery_image_ids( 'edit' ) ) === $want['gallery']; } /** * Featured image and gallery of the product for a row's images (see the class description). * * @param \WC_Product $product Product. * @param string[] $urls URLs by slot. * @param bool $full $urls is the whole row (else only the action's slot: the image is appended to the * gallery and nothing is taken out). * @return array {image: int, gallery: int[], missing: bool} */ private static function layout( \WC_Product $product, array $urls, bool $full ): array { $own = self::own( $product->get_id() ); $ids = array(); $missing = false; foreach ( $urls as $slot => $url ) { $id = self::localAttachment( (string) $url ); if ( 0 === $id ) { $id = $own[ UrlNormalizer::hash( (string) $url ) ][0] ?? 0; } $ids[ (int) $slot ] = $id; $missing = $missing || 0 === $id; } $current = array_map( 'intval', (array) $product->get_gallery_image_ids( 'edit' ) ); $image = ! empty( $ids[0] ) ? (int) $ids[0] : (int) $product->get_image_id( 'edit' ); if ( $product->is_type( 'variation' ) ) { // A variation has one image and no gallery. return array( 'image' => $image, 'gallery' => $current, 'missing' => $missing, ); } $added = $full ? array_merge( array(), ...array_values( $own ) ) : array(); $want = array(); foreach ( $ids as $slot => $id ) { if ( $slot > 0 && $id > 0 && $id !== $image && ! in_array( $id, $want, true ) ) { $want[] = $id; } } $keep = array(); foreach ( $current as $id ) { if ( $id !== $image && ! in_array( $id, $want, true ) && ! in_array( $id, $added, true ) && ! in_array( $id, $keep, true ) ) { $keep[] = $id; } } return array( 'image' => $image, 'gallery' => $full ? array_merge( $want, $keep ) : array_merge( $keep, $want ), 'missing' => $missing, ); } /** * Add the file to the media library, as the user who started the import (core D23), with its URL hash from the start * (an attachment whose sub-sizes never got made is still found by the next import). * * @param string $file Temporary file. * @param string $mime Content type found by SafeFetcher. * @param string $url Source URL. * @param int $productId Product. * @param string $hash URL hash. * @param int $author User the attachment belongs to. * @return int|\WP_Error */ private static function sideload( string $file, string $mime, string $url, int $productId, string $hash, int $author ) { require_once ABSPATH . 'wp-admin/includes/file.php'; require_once ABSPATH . 'wp-admin/includes/media.php'; require_once ABSPATH . 'wp-admin/includes/image.php'; $path = (string) wp_parse_url( $url, PHP_URL_PATH ); $name = sanitize_file_name( basename( $path ) ); $ext = strtolower( pathinfo( $name, PATHINFO_EXTENSION ) ); if ( '' === $name || ! in_array( $ext, array( 'jpg', 'jpeg', 'png', 'gif', 'webp', 'avif' ), true ) ) { $map = array( 'image/png' => 'png', 'image/gif' => 'gif', 'image/webp' => 'webp', 'image/avif' => 'avif', ); $name = 'image-' . substr( $hash, 0, 8 ) . '.' . ( $map[ $mime ] ?? 'jpg' ); } $user = get_current_user_id(); wp_set_current_user( $author ); try { $id = media_handle_sideload( array( 'name' => $name, 'tmp_name' => $file, ), $productId, null, array( 'meta_input' => array( self::META_URL_HASH => $hash ) ) ); } finally { wp_set_current_user( $user ); } return is_wp_error( $id ) ? $id : (int) $id; } /** * Lay out the product's images (serialised per product, so parallel downloads neither lose updates nor keep two * attachments of one URL). Saves the product only when its images change. * * @param int $productId Product. * @param string $hash URL hash of the action's image. * @param int $created Attachment this action just created, 0 for none. * @param string[] $urls URLs by slot. * @param bool $full $urls is the whole row. * @return bool False when the lock could not be taken. */ private static function attach( int $productId, string $hash, int $created, array $urls, bool $full ): bool { global $wpdb; $lock = 'catalogrelay_img_' . $productId; if ( 1 !== (int) $wpdb->get_var( $wpdb->prepare( 'SELECT GET_LOCK(%s, 10)', $lock ) ) ) { return false; } try { if ( $created > 0 ) { // Another worker stored the same URL for this product meanwhile: keep the older attachment. $first = self::existing( $productId, $hash ); if ( $first > 0 && $first !== $created ) { wp_delete_attachment( $created, true ); } } $product = Fresh::product( $productId ); if ( ! $product ) { return true; } $want = self::layout( $product, $urls, $full ); $changed = false; if ( (int) $product->get_image_id( 'edit' ) !== $want['image'] ) { $product->set_image_id( $want['image'] ); $changed = true; } if ( array_map( 'intval', (array) $product->get_gallery_image_ids( 'edit' ) ) !== $want['gallery'] ) { $product->set_gallery_image_ids( $want['gallery'] ); $changed = true; } if ( $changed ) { $product->save(); } return true; } finally { $wpdb->query( $wpdb->prepare( 'SELECT RELEASE_LOCK(%s)', $lock ) ); } } }