prefix . 'afsd_im_' . $name; } /** * Create or update Image Manager's tables and record the schema version. */ public static function install(): void { $im_reviewed_owner_lock = Reviewed_Store::enter(); try { global $wpdb; require_once ABSPATH . 'wp-admin/includes/upgrade.php'; $cc = $wpdb->get_charset_collate(); $s = self::table( 'sources' ); $p = self::table( 'pages' ); $o = self::table( 'owners' ); $l = self::table( 'locks' ); $u = self::table( 'urls' ); dbDelta( "CREATE TABLE $s ( id bigint(20) unsigned NOT NULL AUTO_INCREMENT, source_key char(64) NOT NULL, attachment_id bigint(20) unsigned NOT NULL DEFAULT 0, priority tinyint unsigned NOT NULL DEFAULT 2, state varchar(24) NOT NULL DEFAULT 'discovered', lane varchar(12) NOT NULL DEFAULT 'backlog', revision bigint(20) unsigned NOT NULL DEFAULT 1, requested tinyint unsigned NOT NULL DEFAULT 0, excluded tinyint unsigned NOT NULL DEFAULT 0, delivery tinyint unsigned NOT NULL DEFAULT 1, attempts tinyint unsigned NOT NULL DEFAULT 0, available_at bigint(20) unsigned NOT NULL DEFAULT 0, updated_at bigint(20) unsigned NOT NULL DEFAULT 0, details longtext NOT NULL, manifest longtext NOT NULL, reason varchar(190) NOT NULL DEFAULT '', saved_bytes bigint(20) unsigned NOT NULL DEFAULT 0, copy_bytes bigint(20) unsigned NOT NULL DEFAULT 0, copy_count int unsigned NOT NULL DEFAULT 0, PRIMARY KEY (id), UNIQUE KEY source_key (source_key), KEY attachment (attachment_id), KEY work (state,available_at,lane,priority), KEY display_order (priority,id) ) $cc;" ); dbDelta( "CREATE TABLE $p ( id bigint(20) unsigned NOT NULL AUTO_INCREMENT, url_key char(64) NOT NULL, url text NOT NULL, depth tinyint unsigned NOT NULL DEFAULT 1, roots tinyint unsigned NOT NULL DEFAULT 0, state varchar(24) NOT NULL DEFAULT 'waiting', reason varchar(190) NOT NULL DEFAULT '', images int unsigned NOT NULL DEFAULT 0, attempts tinyint unsigned NOT NULL DEFAULT 0, available_at bigint(20) unsigned NOT NULL DEFAULT 0, PRIMARY KEY (id), UNIQUE KEY url_key (url_key), KEY pending (state,depth,id) ) $cc;" ); dbDelta( "CREATE TABLE $o ( id bigint(20) unsigned NOT NULL AUTO_INCREMENT, source_id bigint(20) unsigned NOT NULL, owner_type varchar(12) NOT NULL, owner_id bigint(20) unsigned NOT NULL, PRIMARY KEY (id), UNIQUE KEY relation (source_id,owner_type,owner_id), KEY reverse_owner (owner_type,owner_id) ) $cc;" ); dbDelta( "CREATE TABLE $l ( name varchar(32) NOT NULL, token varchar(64) NOT NULL, expires bigint(20) unsigned NOT NULL DEFAULT 0, PRIMARY KEY (name) ) $cc;" ); dbDelta( "CREATE TABLE $u ( url_key char(64) NOT NULL, attachment_id bigint(20) unsigned NOT NULL, source_id bigint(20) unsigned NOT NULL, PRIMARY KEY (url_key), KEY source (source_id) ) $cc;" ); $b = self::table( 'batch' ); dbDelta( "CREATE TABLE $b ( batch_id char(32) NOT NULL, source_id bigint(20) unsigned NOT NULL, state varchar(20) NOT NULL DEFAULT 'waiting', draft_id char(32) NOT NULL DEFAULT '', reason varchar(190) NOT NULL DEFAULT '', PRIMARY KEY (batch_id,source_id), KEY work (batch_id,state,source_id) ) $cc;" ); // Do not report a successful schema migration when dbDelta failed to add required columns. // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $source_columns = $wpdb->get_col( $wpdb->prepare( 'SHOW COLUMNS FROM %i', $s ), 0 ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $page_columns = $wpdb->get_col( $wpdb->prepare( 'SHOW COLUMNS FROM %i', $p ), 0 ); if ( array_diff( array( 'copy_count', 'copy_bytes', 'saved_bytes' ), $source_columns ) || array_diff( array( 'attempts', 'available_at' ), $page_columns ) ) { return; } // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. if ( array_diff( array( 'batch_id', 'source_id', 'state', 'draft_id', 'reason' ), $wpdb->get_col( $wpdb->prepare( 'SHOW COLUMNS FROM %i', $b ), 0 ) ) ) { return; } Reviewed_Store::update_option( 'afsd_im_schema', SCHEMA, false ); } finally { Reviewed_Store::leave( $im_reviewed_owner_lock ); } } /** * A source row, decoded. * * @param int $id The row. * @return array|null The row, or null. */ public static function row( int $id ): ?array { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $r = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id=%d', self::table( 'sources' ), $id ), ARRAY_A ); return $r ? self::decode( $r ) : null; } /** * The source row of an attachment, decoded. * * @param int $id The attachment. * @return array|null The row, or null. */ public static function attachment( int $id ): ?array { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $r = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE source_key=%s', self::table( 'sources' ), hash( 'sha256', 'attachment:' . $id ) ), ARRAY_A ); return $r ? self::decode( $r ) : null; } /** * A source row with its details and manifest decoded from JSON. * * @param array $row The row as read. * @return array The row. */ public static function decode( array $row ): array { foreach ( array( 'details', 'manifest' ) as $key ) { $row[ $key ] = is_array( $row[ $key ] ) ? $row[ $key ] : ( json_decode( $row[ $key ], true ) ?: array() ); } return $row; } /** * Record an attachment found on a page as a source (made once, updated after), with where it was found. * * @param int $attachment The attachment. * @param int $priority Its priority. * @param array $finding Where it was found: page, address, surface and mapping. * @param string $lane backlog or upload. * @return array The source row. * @throws \RuntimeException With a reason code as its message when the step cannot be done. */ public static function discover( int $attachment, int $priority, array $finding, string $lane = 'backlog' ): array { $im_reviewed_owner_lock = Reviewed_Store::enter(); try { global $wpdb; $url = $finding['url'] ?? ''; $key = hash( 'sha256', $attachment ? 'attachment:' . $attachment : 'unmapped:' . $url ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $row = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE source_key=%s', self::table( 'sources' ), $key ), ARRAY_A ); if ( ! $row ) { // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. Reviewed_Store::database_query( $wpdb->prepare( 'INSERT IGNORE INTO %i (source_key,attachment_id,priority,lane,state,details,manifest,updated_at) VALUES (%s,%d,%d,%s,%s,%s,%s,%d)', self::table( 'sources' ), $key, $attachment, $priority, $lane, $attachment ? 'discovered' : 'unmapped', '{}', '{}', time() ) ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $row = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE source_key=%s', self::table( 'sources' ), $key ), ARRAY_A ); } if ( ! $row ) { throw new \RuntimeException( 'database_write_failed' ); } $row = self::decode( $row ); $d = $row['details']; $d['pages'] = array_slice( array_values( array_unique( array_filter( array_merge( $d['pages'] ?? array(), array( $finding['page'] ?? '' ) ) ) ) ), 0, 12 ); $d['urls'] = array_slice( array_values( array_unique( array_filter( array_merge( $d['urls'] ?? array(), array( $url ) ) ) ) ), 0, 16 ); $d['surfaces'] = array_slice( array_values( array_unique( array_filter( array_merge( (array) ( $d['surfaces'] ?? array() ), array( $d['surface'] ?? '', $finding['surface'] ?? '' ) ) ) ) ), 0, 12 ); $d['surface'] = $finding['surface'] ?? ( $d['surface'] ?? 'public_html' ); $d['mapping'] = $finding['mapping'] ?? ( $d['mapping'] ?? 'unresolved' ); $d['cdn'] = ! empty( $finding['cdn'] ) || ! empty( $d['cdn'] ); $d['html_width'] = $finding['width'] ?? ( $d['html_width'] ?? 0 ); $d['display_width'] = 'unknown'; self::update( (int) $row['id'], array( 'priority' => min( $priority, (int) $row['priority'] ), 'details' => $d, ) ); if ( $attachment ) { Sources::index( $attachment, (int) $row['id'] ); } return self::row( (int) $row['id'] ); } finally { Reviewed_Store::leave( $im_reviewed_owner_lock ); } } /** * Save fields of a source row (only its allowed fields), when its revision still matches. * * @param int $id The row. * @param array $values The fields to save. * @param int|null $revision The revision it must still have, or null for any. * @return bool True when it was saved. */ public static function update( int $id, array $values, ?int $revision = null ): bool { $im_reviewed_owner_lock = Reviewed_Store::enter(); try { global $wpdb; $allowed = array( 'priority', 'state', 'lane', 'revision', 'requested', 'excluded', 'delivery', 'attempts', 'available_at', 'updated_at', 'details', 'manifest', 'reason' ); $values = array_intersect_key( $values, array_flip( $allowed ) ); if ( isset( $values['manifest'] ) && is_array( $values['manifest'] ) ) { $values = array_merge( $values, self::manifest_totals( $values['manifest'] ) ); } foreach ( array( 'details', 'manifest' ) as $k ) { if ( isset( $values[ $k ] ) ) { $values[ $k ] = wp_json_encode( $values[ $k ] ); } } $values['updated_at'] = time(); $where = array( 'id' => $id ); if ( null !== $revision ) { $where['revision'] = $revision; } // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $result = Reviewed_Store::database_update( self::table( 'sources' ), $values, $where ); return false !== $result && ( null === $revision || $result > 0 ); } finally { Reviewed_Store::leave( $im_reviewed_owner_lock ); } } /** * One prepared file is counted once even if multiple native entries share it. * * @param array $manifest The manifest. * @return array Bytes saved, bytes of copies and the number of copies. */ public static function manifest_totals( array $manifest ): array { $out = array( 'saved_bytes' => 0, 'copy_bytes' => 0, 'copy_count' => 0, ); $seen = array(); foreach ( $manifest['variants'] ?? array() as $v ) { if ( ! is_array( $v ) || empty( $v['file'] ) || isset( $seen[ $v['file'] ] ) || ! isset( $v['bytes'], $v['output_bytes'] ) ) { continue; }$seen[ $v['file'] ] = true; $out['saved_bytes'] += max( 0, (int) $v['bytes'] - (int) $v['output_bytes'] ); } foreach ( Variants::flatten( $manifest ) as $v ) { $out['copy_bytes'] += max( 0, (int) $v['output_bytes'] ); ++$out['copy_count']; } return $out; } /** * Record that a post or comment owns a source. * * @param int $source The source row. * @param string $type post or comment. * @param int $id Its ID. */ public static function owner( int $source, string $type, int $id ): void { $im_reviewed_owner_lock = Reviewed_Store::enter(); try { global $wpdb; if ( ! in_array( $type, array( 'post', 'comment' ), true ) || ! $id ) { return; } // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. Reviewed_Store::database_query( $wpdb->prepare( 'INSERT IGNORE INTO %i (source_id,owner_type,owner_id) VALUES (%d,%s,%d)', self::table( 'owners' ), $source, $type, $id ) ); } finally { Reviewed_Store::leave( $im_reviewed_owner_lock ); } } /** * The posts and comments that own a source (at most 64). * * @param int $source The source row. * @return array Owner types and IDs. */ public static function owners( int $source ): array { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. return $wpdb->get_results( $wpdb->prepare( 'SELECT owner_type,owner_id FROM %i WHERE source_id=%d LIMIT 64', self::table( 'owners' ), $source ), ARRAY_A ); } /** * The sources a post or comment owns. * * @param string $type post or comment. * @param int $id Its ID. * @return array Source row IDs. */ public static function affected( string $type, int $id ): array { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. return array_map( 'intval', $wpdb->get_col( $wpdb->prepare( 'SELECT source_id FROM %i WHERE owner_type=%s AND owner_id=%d', self::table( 'owners' ), $type, $id ) ) ); } /** * Queue a source for its copies (an excluded source or one already queued stays as it is). * * @param int $id The source row. * @param string $lane upload for new uploads, backlog otherwise. */ public static function request( int $id, string $lane = 'backlog' ): void { $im_reviewed_owner_lock = Reviewed_Store::enter(); try { $r = self::row( $id ); if ( ! $r || ( ! $r['attachment_id'] && ! Theme::is_source( $r ) ) || $r['excluded'] ) { return; } // Duplicate discovery/upload events must not reset backoff or repeatedly queue a ready source. if ( 'queued' === $r['state'] && $r['requested'] ) { Scheduler::needed(); return; } self::update( $id, array( 'state' => 'queued', 'requested' => 1, 'lane' => 'upload' === $lane ? 'upload' : 'backlog', 'attempts' => 0, 'available_at' => time(), 'reason' => '', ) ); Scheduler::needed(); } finally { Reviewed_Store::leave( $im_reviewed_owner_lock ); } } /** * The next queued source that is due: its lane first, then by priority and age. * * @param string $lane The lane to prefer. * @return array|null The source row, or null. */ public static function next_source( string $lane ): ?array { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $r = $wpdb->get_row( $wpdb->prepare( "SELECT * FROM %i WHERE state='queued' AND excluded=0 AND requested=1 AND available_at<=%d ORDER BY (lane=%s) DESC,priority ASC,updated_at ASC,id ASC LIMIT 1", self::table( 'sources' ), time(), $lane ), ARRAY_A ); return $r ? self::decode( $r ) : null; } /** * A page of source rows by priority. * * @param int $page The page, from 1. * @param int $limit Rows per page. * @return array The rows, decoded. */ public static function listing( int $page, int $limit = 20 ): array { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $rows = $wpdb->get_results( $wpdb->prepare( 'SELECT * FROM %i ORDER BY priority ASC,id ASC LIMIT %d OFFSET %d', self::table( 'sources' ), $limit, max( 0, $page - 1 ) * $limit ), ARRAY_A ); return array_map( array( self::class, 'decode' ), $rows ); } /** * Sources that could be queued first: attachments not excluded that are found, held or failed. * * @return array The rows, decoded. */ public static function priority_candidates(): array { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $rows = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM %i WHERE attachment_id>0 AND excluded=0 AND state IN ('discovered','held','failed') ORDER BY priority ASC,id ASC LIMIT 100", self::table( 'sources' ) ), ARRAY_A ); return array_map( array( self::class, 'decode' ), $rows ); } /** * How many rows of a table are in each state. * * @param string $table sources or pages. * @return array State => count. */ public static function counts( string $table = 'sources' ): array { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. $r = $wpdb->get_results( $wpdb->prepare( "SELECT state,COUNT(*) AS n FROM %i WHERE state<>'grouped' GROUP BY state", self::table( $table ) ), ARRAY_A ); $out = array(); foreach ( $r as $v ) { $out[ $v['state'] ] = (int) $v['n']; } return $out; } /** * Atomic owner-aware leases; every publication additionally checks source revision. * * @param string $name The lease's name (worker by default). * @param int $seconds How long it lasts, 15 to 600 seconds. * @return string|null The lease's token, or null when another holds it. */ public static function acquire( string $name = 'worker', int $seconds = 600 ): ?string { global $wpdb; $token = bin2hex( random_bytes( 24 ) ); $expires = time() + max( 15, min( 600, $seconds ) ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. Reviewed_Store::database_query( $wpdb->prepare( 'INSERT IGNORE INTO %i (name,token,expires) VALUES (%s,%s,%d)', self::table( 'locks' ), $name, $token, $expires ) ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. Reviewed_Store::database_query( $wpdb->prepare( 'UPDATE %i SET token=%s,expires=%d WHERE name=%s AND expires<%d', self::table( 'locks' ), $token, $expires, $name, time() ) ); return self::owns( $token, $name ) ? $token : null; } /** * Whether a lease is still held with this token. * * @param string $token The token. * @param string $name The lease's name. * @return bool True when it is. */ public static function owns( string $token, string $name = 'worker' ): bool { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. return (bool) $wpdb->get_var( $wpdb->prepare( 'SELECT COUNT(*) FROM %i WHERE name=%s AND token=%s AND expires>=%d', self::table( 'locks' ), $name, $token, time() ) ); } /** * Release a lease held with this token. * * @param string $token The token. * @param string $name The lease's name. */ public static function release( string $token, string $name = 'worker' ): void { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching -- Plugin-owned tables, schema checks and atomic worker leases have no equivalent WP object API; current revisions/locks must come directly from the database. Reviewed_Store::database_delete( self::table( 'locks' ), array( 'name' => $name, 'token' => $token, ) ); } }