/: the folder name is an HMAC * of the job ID with the site's auth salt, so knowing the ID (it appears in * REST URLs and server logs) is not enough to fetch a file directly, even * on servers that ignore the .htaccess deny. A step saves its progress after * every batch (state written to a temp file, then renamed), and truncates * the spool back to the saved size before appending, so a step killed by a * time limit is simply done again. A lock stops two steps of one job (a * browser retry while the first is still running) from writing at once. */ final class Job { // Seconds of work per step, at most a third of max_execution_time. public const SECONDS = 5.0; // Orders loaded per wc_get_orders() call. public const BATCH = 50; // IDs per snapshot query. public const PAGE = 2000; // Each snapshot page re-reads this many positions of the previous one, // so up to this many orders leaving the export between two pages (trashed, // status changed) don't shift unread orders past us. private const OVERLAP = 200; // A job nobody stepped for this long is abandoned; its files are removed. public const MAX_IDLE = 3600; // Cron hook that removes abandoned jobs. public const SWEEP_HOOK = 'modulary_export_sweep_jobs'; // Build in the same step that collected the last rows, when the file is this small. private const BUILD_INLINE = 20000; // Bytes per ID in the snapshot file: zero-padded, then a newline. private const ID_WIDTH = 20; private const FOLDER = 'modulary-export'; private const VERSION = 1; public const SNAPSHOT = 'snapshot'; public const ROWS = 'rows'; public const BUILD = 'build'; public const READY = 'ready'; /** * The request, rebuilt from the saved state once per instance. * * @var Request|null */ private ?Request $request = null; /** * Constructor. * * @param string $id Public job ID (32 hex characters). * @param string $dir Job folder. * @param array $state Saved state (see start()). */ private function __construct( public readonly string $id, private readonly string $dir, private array $state ) {} /** * Create a job for this request and user. Fast: no orders are read yet. * * @param Request $request What to export. * @param int $user Owner; only they can step, cancel or download it. * @throws JobException When the uploads folder is not writable. */ public static function start( Request $request, int $user ): self { self::sweep(); $id = bin2hex( random_bytes( 16 ) ); $dir = self::dir_for( $id, true ); if ( ! wp_mkdir_p( $dir ) ) { throw JobException::storage(); } self::put( $dir . '/index.html', '' ); self::put( $dir . '/ids.txt', '' ); self::put( $dir . '/rows.jsonl', '' ); $job = new self( $id, $dir, [ 'version' => self::VERSION, 'user' => $user, 'request' => json_decode( (string) wp_json_encode( $request->to_array() ), true ), 'headers' => array_values( array_map( 'strval', $request->headers() ) ), 'phase' => self::SNAPSHOT, // Shown as the total until the snapshot has counted the real one. 'estimate' => ( new Query( $request->filters ) )->count(), 'total' => 0, 'read' => 0, 'recent' => [], 'ids_bytes' => 0, 'cursor' => 0, 'rows' => 0, 'spool_bytes' => 0, 'created' => time(), ] ); $job->save(); if ( ! wp_next_scheduled( self::SWEEP_HOOK ) ) { wp_schedule_single_event( time() + self::MAX_IDLE + 60, self::SWEEP_HOOK ); } return $job; } /** * This user's job, or null: unknown, malformed, cancelled, cleaned up or * someone else's all look the same, so IDs can't be probed. * * @param string $id Job ID. * @param int $user Current user. */ public static function find( string $id, int $user ): ?self { if ( $user <= 0 || 1 !== preg_match( '/^[0-9a-f]{32}$/', $id ) ) { return null; } $dir = self::dir_for( $id, false ); $state = self::read_state( $dir ); if ( null === $state || $state['user'] !== $user ) { return null; } return new self( $id, $dir, $state ); } /** * The saved state of a job folder, or null when there is none (or it * was cancelled). * * @param string $dir Job folder. * @return array|null */ private static function read_state( string $dir ): ?array { $file = $dir . '/job.json'; if ( ! is_file( $file ) || is_file( $dir . '/cancelled' ) ) { return null; } return self::validate( json_decode( (string) file_get_contents( $file ), true ) ); // phpcs:ignore WordPress.WP.AlternativeFunctions -- Local file in our own folder. } /** * Do up to $seconds of work, then save. Call again until finished(). * * Always does at least one unit of work (a snapshot page or a batch of * orders), so 0 means "one unit per step". * * @param float $seconds Time budget. * @param int $batch Orders per batch. * @param int $page IDs per snapshot page. * @throws JobException When another step is running, the job is gone or files can't be written. */ public function step( float $seconds = self::SECONDS, int $batch = self::BATCH, int $page = self::PAGE ): void { $lock = $this->lock(); try { // Whatever the step before this one saved, now that we hold the lock. $state = self::read_state( $this->dir ); if ( null === $state || $state['user'] !== $this->state['user'] ) { throw JobException::gone(); } $this->state = $state; if ( self::READY === $this->state['phase'] ) { return; } wp_raise_memory_limit( 'admin' ); $start = microtime( true ); $worked = false; while ( true ) { if ( self::BUILD === $this->state['phase'] ) { // The build is one pass: give a big file a step of its own. $late = microtime( true ) - $start >= $seconds / 2 || $this->state['rows'] > self::BUILD_INLINE; if ( ! $worked || ! $late ) { $this->build(); } return; } if ( self::SNAPSHOT === $this->state['phase'] ) { $this->snapshot( max( 1, $page ) ); } else { $this->collect( max( 1, $batch ) ); } $this->save(); $worked = true; if ( microtime( true ) - $start >= $seconds || $this->low_memory() || is_file( $this->dir . '/cancelled' ) ) { return; } } } finally { flock( $lock, LOCK_UN ); // phpcs:ignore WordPress.WP.AlternativeFunctions fclose( $lock ); // phpcs:ignore WordPress.WP.AlternativeFunctions } } /** * Progress for the browser. * * @return array{id: string, total: int, done: int, rows: int, finished: bool} */ public function progress(): array { $total = (int) $this->state['total']; if ( self::SNAPSHOT === $this->state['phase'] ) { $total = max( $total, (int) $this->state['estimate'] ); } return [ 'id' => $this->id, 'total' => $total, 'done' => (int) $this->state['cursor'], 'rows' => (int) $this->state['rows'], 'finished' => self::READY === $this->state['phase'], ]; } /** * Whether the file is complete. */ public function finished(): bool { return self::READY === $this->state['phase']; } /** * Rows in the file (excluding the header). */ public function rows(): int { return (int) $this->state['rows']; } /** * Orders in this export, in file order: up to `$limit` IDs from position * `$offset` of the snapshot taken when the job started. Pro marks exactly * these orders as sent once a scheduled file was delivered. An order * deleted before its batch was read is listed here but not in the file. * A snapshot file that can't be read throws JobException, as a step does. * * @param int $offset Position of the first ID, from 0. * @param int $limit At most this many IDs. * @return int[] * @throws \LogicException Before the snapshot is complete. */ public function order_ids( int $offset = 0, int $limit = 500 ): array { if ( self::SNAPSHOT === $this->state['phase'] ) { throw new \LogicException( 'The order list is complete once the snapshot is.' ); } $offset = max( 0, $offset ); $count = min( max( 0, $limit ), (int) $this->state['total'] - $offset ); return $count > 0 ? $this->read_ids( $offset, $count ) : []; } /** * What this job exports, re-sanitised from the saved state. */ public function request(): Request { if ( null === $this->request ) { $this->request = Request::from_array( (array) $this->state['request'] ); } return $this->request; } /** * Path of the finished file. * * @throws JobException Before the file is ready. */ public function file(): string { $file = $this->dir . '/export'; if ( ! $this->finished() || ! is_file( $file ) ) { throw JobException::gone(); } return $file; } /** * Stop a running job and remove its files. Waits for a step that is * running to finish its current batch (it checks for the cancel mark). */ public function cancel(): void { self::put( $this->dir . '/cancelled', '' ); $lock = fopen( $this->dir . '/lock', 'c' ); // phpcs:ignore WordPress.WP.AlternativeFunctions if ( false !== $lock ) { flock( $lock, LOCK_EX ); // phpcs:ignore WordPress.WP.AlternativeFunctions flock( $lock, LOCK_UN ); // phpcs:ignore WordPress.WP.AlternativeFunctions fclose( $lock ); // phpcs:ignore WordPress.WP.AlternativeFunctions } self::remove( $this->dir ); } /** * Remove the job's files (after the download). */ public function delete(): void { self::remove( $this->dir ); } /** * Remove jobs nobody has stepped for $max_idle seconds: abandoned * (the merchant left the page), or finished and never downloaded. Runs * when a job starts and from a single cron event, which reschedules * itself while jobs remain. * * @param int $max_idle Seconds since the last step. * @return int Jobs removed. */ public static function sweep( int $max_idle = self::MAX_IDLE ): int { $base = self::base( false ); if ( ! is_dir( $base ) ) { return 0; } clearstatcache(); $removed = 0; $left = 0; foreach ( (array) scandir( $base ) as $name ) { if ( 1 !== preg_match( '/^[0-9a-f]{32}$/', (string) $name ) ) { continue; } $dir = $base . '/' . $name; $state = $dir . '/job.json'; $touched = (int) ( is_file( $state ) ? filemtime( $state ) : filemtime( $dir ) ); if ( time() - $touched > $max_idle ) { self::remove( $dir ); ++$removed; } else { ++$left; } } if ( $left && ! wp_next_scheduled( self::SWEEP_HOOK ) ) { wp_schedule_single_event( time() + self::MAX_IDLE + 60, self::SWEEP_HOOK ); } return $removed; } /** * Snapshot one page of matching IDs into ids.txt. * * Paging is by offset, newest first. A new order lands on top and shifts * the rest down: the overlap re-reads what we already have and drops it, * so nothing repeats. An order leaving the export shifts the rest up: the * overlap picks up what slid back into pages already read, so nothing is * skipped. Orders created once the snapshot is complete are not in it. * * @param int $page IDs per page. */ private function snapshot( int $page ): void { $read = (int) $this->state['read']; $from = max( 0, $read - self::OVERLAP ); $limit = $page + ( $read - $from ); $ids = ( new Query( $this->request()->filters ) )->id_page( $from, $limit ); $seen = array_flip( $this->state['recent'] ); $new = ''; $add = 0; foreach ( $ids as $id ) { if ( ! isset( $seen[ $id ] ) ) { $new .= str_pad( (string) $id, self::ID_WIDTH - 1, '0', STR_PAD_LEFT ) . "\n"; ++$add; } } $this->state['ids_bytes'] = $this->append( 'ids.txt', (int) $this->state['ids_bytes'], $new ); $this->state['total'] += $add; $this->state['read'] = $from + $limit; $this->state['recent'] = $ids; if ( count( $ids ) < $limit ) { $this->state['recent'] = []; $this->state['phase'] = self::ROWS; } } /** * Load the next batch of orders and spool their rows. * * @param int $batch Orders per batch. * @throws JobException When the snapshot can't be read. */ private function collect( int $batch ): void { $cursor = (int) $this->state['cursor']; $total = (int) $this->state['total']; $count = min( $batch, $total - $cursor ); if ( $count <= 0 ) { $this->state['phase'] = self::BUILD; return; } $ids = $this->read_ids( $cursor, $count ); $exporter = new Exporter( $this->request() ); $orders = ( new Query( $this->request()->filters ) )->load( $ids ); $lines = ''; $rows = 0; // Snapshot order, not the order the query returned them in. foreach ( $ids as $id ) { if ( ! isset( $orders[ $id ] ) ) { continue; } foreach ( $exporter->rows( $orders[ $id ] ) as $row ) { $lines .= self::encode( $row ) . "\n"; ++$rows; } } $this->state['spool_bytes'] = $this->append( 'rows.jsonl', (int) $this->state['spool_bytes'], $lines ); $this->state['cursor'] = $cursor + $count; $this->state['rows'] += $rows; if ( $this->state['cursor'] >= $total ) { $this->state['phase'] = self::BUILD; } } /** * `$count` IDs from position `$offset` of the snapshot (fixed-width lines, * so this is one seek). * * @param int $offset First position, from 0. * @param int $count How many; the caller keeps it within the total. * @return int[] * @throws JobException When the snapshot can't be read or is short. */ private function read_ids( int $offset, int $count ): array { $handle = fopen( $this->dir . '/ids.txt', 'rb' ); // phpcs:ignore WordPress.WP.AlternativeFunctions if ( false === $handle ) { throw JobException::storage(); } $raw = (string) stream_get_contents( $handle, $count * self::ID_WIDTH, $offset * self::ID_WIDTH ); fclose( $handle ); // phpcs:ignore WordPress.WP.AlternativeFunctions $ids = array_map( 'intval', array_filter( explode( "\n", $raw ), static fn( string $line ) => '' !== $line ) ); if ( count( $ids ) !== $count ) { throw JobException::storage(); } return $ids; } /** * Write the file from the spool in one pass, then fire * `modulary_export_completed` exactly as a one-pass export does. * * @throws JobException When files can't be read or written. */ private function build(): void { // A cushion where the host allows it; the pass itself is short. wc_set_time_limit( 0 ); $spool = fopen( $this->dir . '/rows.jsonl', 'rb' ); // phpcs:ignore WordPress.WP.AlternativeFunctions $out = fopen( $this->dir . '/export', 'wb' ); // phpcs:ignore WordPress.WP.AlternativeFunctions if ( false === $spool || false === $out ) { throw JobException::storage(); } $request = $this->request(); $bytes = (int) $this->state['spool_bytes']; $writer = Format::writer( $request, $out ); $writer->row( $this->state['headers'] ); $count = 0; while ( ftell( $spool ) < $bytes ) { $line = fgets( $spool ); if ( false === $line ) { break; } $writer->row( self::decode( rtrim( $line, "\n" ) ) ); ++$count; if ( 0 === $count % 200 ) { $writer->flush(); } } $writer->close(); fclose( $spool ); // phpcs:ignore WordPress.WP.AlternativeFunctions fclose( $out ); // phpcs:ignore WordPress.WP.AlternativeFunctions if ( $count !== (int) $this->state['rows'] ) { throw JobException::storage(); } wp_delete_file( $this->dir . '/rows.jsonl' ); // ids.txt stays (20 bytes per order) for order_ids(); it goes with the folder. $this->state['phase'] = self::READY; $this->save(); /** This action is documented in src/Export/Exporter.php */ do_action( 'modulary_export_completed', $request, $count ); } /** * One row as one spool line. JSON, unless a cell is not valid UTF-8 * (JSON can't hold that): then hex, so the file gets exactly the bytes a * one-pass export writes. * * @param array $row Cells. */ private static function encode( array $row ): string { $cells = array_map( static fn( $v ) => null === $v || false === $v ? '' : (string) $v, array_values( $row ) ); $json = json_encode( $cells, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES ); // phpcs:ignore WordPress.WP.AlternativeFunctions -- wp_json_encode() would silently re-encode invalid UTF-8; we want to know. if ( false !== $json ) { return $json; } return 'H' . implode( ',', array_map( 'bin2hex', $cells ) ); } /** * One spool line back to a row. * * @param string $line Line without its newline. * @return array * @throws JobException On a line this class did not write. */ private static function decode( string $line ): array { if ( str_starts_with( $line, 'H' ) ) { return array_map( static fn( string $hex ) => (string) hex2bin( $hex ), explode( ',', substr( $line, 1 ) ) ); } $cells = json_decode( $line, true ); if ( ! is_array( $cells ) ) { throw JobException::storage(); } return array_map( 'strval', $cells ); } /** * Append to a job file after its last saved byte. Anything after that * byte was written by a step that died before saving, and is dropped. * * @param string $name File in the job folder. * @param int $saved Saved size. * @param string $data Bytes to add. * @return int New size. * @throws JobException When the file can't be written. */ private function append( string $name, int $saved, string $data ): int { $handle = fopen( $this->dir . '/' . $name, 'c+b' ); // phpcs:ignore WordPress.WP.AlternativeFunctions if ( false === $handle ) { throw JobException::storage(); } $ok = ftruncate( $handle, $saved ) && 0 === fseek( $handle, $saved ); if ( $ok && '' !== $data ) { $ok = strlen( $data ) === fwrite( $handle, $data ); // phpcs:ignore WordPress.WP.AlternativeFunctions } $ok = $ok && fflush( $handle ); // phpcs:ignore WordPress.WP.AlternativeFunctions fclose( $handle ); // phpcs:ignore WordPress.WP.AlternativeFunctions if ( ! $ok ) { throw JobException::storage(); } return $saved + strlen( $data ); } /** * Save the state: a temp file renamed over the old one, so a step * killed mid-write leaves the previous state intact. * * @throws JobException When the state can't be written. */ private function save(): void { $file = $this->dir . '/job.json'; if ( ! self::put( $file . '.tmp', (string) wp_json_encode( $this->state ) ) || ! rename( $file . '.tmp', $file ) ) { // phpcs:ignore WordPress.WP.AlternativeFunctions throw JobException::storage(); } } /** * Take the job's lock, or fail fast when another step holds it. * * @return resource * @throws JobException When busy or the lock file can't be opened. */ private function lock() { $lock = fopen( $this->dir . '/lock', 'c' ); // phpcs:ignore WordPress.WP.AlternativeFunctions if ( false === $lock ) { throw JobException::storage(); } $would_block = 0; if ( ! flock( $lock, LOCK_EX | LOCK_NB, $would_block ) && $would_block ) { // phpcs:ignore WordPress.WP.AlternativeFunctions fclose( $lock ); // phpcs:ignore WordPress.WP.AlternativeFunctions throw JobException::busy(); } // A filesystem without locks (some network mounts) runs unlocked: the // browser sends one step at a time, so this only matters on a retry. return $lock; } /** * Near the memory limit: stop the step early and let the next one go on. */ private function low_memory(): bool { $limit = wp_convert_hr_to_bytes( (string) ini_get( 'memory_limit' ) ); return $limit > 0 && memory_get_usage( true ) > $limit * 0.8; } /** * Saved state, or null if it is not a state this version wrote. * * @param mixed $state Decoded job.json. * @return array|null */ private static function validate( mixed $state ): ?array { if ( ! is_array( $state ) || self::VERSION !== ( $state['version'] ?? null ) || ! is_array( $state['request'] ?? null ) ) { return null; } if ( ! in_array( $state['phase'] ?? '', [ self::SNAPSHOT, self::ROWS, self::BUILD, self::READY ], true ) ) { return null; } foreach ( [ 'user', 'estimate', 'total', 'read', 'ids_bytes', 'cursor', 'rows', 'spool_bytes', 'created' ] as $key ) { if ( ! is_int( $state[ $key ] ?? null ) || $state[ $key ] < 0 ) { return null; } } $state['headers'] = array_values( array_map( 'strval', array_filter( (array) ( $state['headers'] ?? [] ), 'is_scalar' ) ) ); $state['recent'] = array_values( array_map( 'intval', array_filter( (array) ( $state['recent'] ?? [] ), 'is_int' ) ) ); return $state; } /** * Folder for a job ID: an HMAC, so the ID alone doesn't reveal it. * * @param string $id Job ID. * @param bool $create Create and protect the base folder. */ private static function dir_for( string $id, bool $create ): string { return self::base( $create ) . '/' . substr( hash_hmac( 'sha256', 'job|' . $id, wp_salt( 'auth' ) ), 0, 32 ); } /** * The uploads/modulary-export folder, with the same protection WooCommerce gives * its log folder: deny for Apache, an index file against listings. * * @param bool $create Create and protect it. * @throws JobException When it can't be created. */ private static function base( bool $create ): string { $uploads = wp_upload_dir( null, false ); $base = untrailingslashit( (string) $uploads['basedir'] ) . '/' . self::FOLDER; if ( ! $create || is_file( $base . '/.htaccess' ) ) { return $base; } if ( ! wp_mkdir_p( $base ) ) { throw JobException::storage(); } $deny = "# Temporary export files. Never served.\n" . "\n\tRequire all denied\n\n" . "\n\tDeny from all\n\n"; if ( ! self::put( $base . '/index.php', "