'POST', 'callback' => [ $this, 'handle' ], 'permission_callback' => [ $this->permission, 'require_pull' ], ] ); // WordPress serializes a returned response; streaming keeps the encoded batch out of memory. add_filter( 'rest_pre_serve_request', [ $this, 'stream_batch' ], 10, 4 ); } /** `rest_pre_serve_request` hook: takes over only for a batch response, otherwise passes $served through. */ public function stream_batch( $served, $result, $server = null, $request = null ): bool { $batch = $result instanceof FileBatchBodyResponse ? $result : null; if ( null === $batch ) { return (bool) $served; } // phpcs:disable WordPress.WP.AlternativeFunctions -- Streaming needs the native output functions. while ( ob_get_level() > 0 ) { ob_end_flush(); } header( 'Content-Type: application/json; charset=utf-8' ); self::no_transform_headers(); // The length is unknown until the batch is written, and a wrong one truncates it. header_remove( 'Content-Length' ); // echo on a Generator is a fatal, so walk it. foreach ( $batch->stream_body() as $fragment ) { echo $fragment; // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped -- A fragment of the JSON response body, not HTML. flush(); } // phpcs:enable return true; } /** * `no-store` keeps a CDN from serving one site's chunks to another. `no-transform` * stops a proxy re-compressing the body, which changes the byte count the * destination verifies against and fails a correct file. */ public static function no_transform_headers(): void { header( 'Cache-Control: no-store, no-transform' ); } /** * Budget is reserved before a file is read: its encoded cost is projected from the * raw size, so the response can't overshoot the cap by one file. Oversized, * unreadable and changed-while-reading files go to `skipped_large`. */ public function handle( WP_REST_Request $request ): WP_REST_Response|WP_Error { $body = $request->get_json_params(); $stage = isset( $body['stage'] ) ? sanitize_text_field( (string) $body['stage'] ) : 'uploads'; $raw_paths = isset( $body['paths'] ) && is_array( $body['paths'] ) ? $body['paths'] : []; if ( empty( $raw_paths ) ) { return new WP_Error( 'shiftora_invalid_params', 'paths array is required.', [ 'status' => 400 ] ); } // Unknown stages fall back to uploads. if ( ! FileStages::has( $stage ) ) { $stage = FileStages::UPLOADS; } // Not sanitized as text: that would serve a different file than the one asked for. $paths = array_slice( array_values( array_filter( array_map( static fn( $p ) => is_string( $p ) ? VirtualPath::is_usable( $p ) : null, $raw_paths ) ) ), 0, self::BATCH_MAX_FILES ); // Recomputed on a cache miss, so it always bounds what a request may read. $connection = $request->get_param( '_shiftora_connection' ); $allowed_paths = null; if ( $connection && isset( $connection->id ) ) { $allowed_paths = StageExportScope::allowed_paths( (int) $connection->id, $stage ); } $exporter = StageExportScope::exporter( $stage ); $skipped_large = []; $bytes_used = self::JSON_ENVELOPE_BASE; // The shell counts against the budget. $bytes_served = 0; // Fragments, so the batch is never resident in one piece. $this->body = [ '{"files":[' ]; $first_entry = true; foreach ( $paths as $path ) { if ( '' === $path ) { continue; } if ( null !== $allowed_paths && ! isset( $allowed_paths[ $path ] ) ) { $skipped_large[] = $path; continue; } try { // Size is probed first so the cap and reservation apply before any read. $raw_size = $exporter->file_size( $path ); } catch ( \RuntimeException $e ) { ServerLog::write( "[Shiftora] file-batch: skipping {$path}: {$e->getMessage()}" ); $skipped_large[] = $path; continue; } if ( $raw_size > self::BATCH_FILE_MAX ) { $skipped_large[] = $path; continue; } // Reserve before reading. $remaining = $this->response_budget - $bytes_used; if ( $this->encoded_entry_size( $raw_size ) > $remaining ) { $skipped_large[] = $path; continue; } try { $compressed = $exporter->is_compressible( $path ); // Payload and envelope are built separately: one array held the file, its // gzip, base64 and a serialized copy at once, enough to exhaust a 128 MB host. $data = $exporter->encoded_payload( $path, 0, (int) $raw_size, $compressed ); // A file that grew since the probe was encoded from a stale length, so its // digest covers a prefix. The chunked route reads to EOF and handles it. if ( $exporter->file_size( $path ) !== (int) $raw_size ) { $skipped_large[] = $path; continue; } $entry = wp_json_encode( [ 'path' => $path, 'chunk_hash' => $exporter->chunk_hash( $path, 0, (int) $raw_size ), 'compressed' => $compressed, 'size' => $raw_size, ] ); if ( ! is_string( $entry ) ) { $skipped_large[] = $path; continue; } $this->body[] = ( $first_entry ? '' : ',' ) . substr( $entry, 0, -1 ) . ',"data":"' . $data . '"}'; $first_entry = false; ++$this->served; // Charge actual encoded bytes; the projection was an upper bound, so this never overshoots. $bytes_used += strlen( $data ) + self::PER_FILE_OVERHEAD; $bytes_served += $raw_size; unset( $data ); } catch ( \RuntimeException $e ) { ServerLog::write( "[Shiftora] file-batch: skipping {$path}: {$e->getMessage()}" ); $skipped_large[] = $path; } } $this->body[] = '],"skipped_large":' . wp_json_encode( array_values( $skipped_large ) ) . ',"count":' . (int) $this->served . ',"bytes_expanded":' . $bytes_served . ',"reserved":' . $bytes_used . ',"remaining":' . max( 0, $this->response_budget - $bytes_used ) . '}'; // Cleared so the next request starts from zero and the batch isn't held. $body = $this->body; $this->body = []; $this->served = 0; return new FileBatchBodyResponse( $body ); } /** Upper bound: base64 of the worst-case gzip size plus the envelope. */ private function encoded_entry_size( int $raw_size ): int { $expanded = $raw_size + self::GZIP_EXPANSION; return (int) ( 4 * ceil( $expanded / 3 ) ) + self::PER_FILE_OVERHEAD; } }