exclusions = new ExclusionMatcher(); } /** The resolved base directory. */ public function get_uploads_base(): string { return $this->uploads_base; } /** * Build a sorted manifest of every file under the base. * * @param string[] $exclusions Shell-style patterns (fnmatch) to skip. * * @return array */ public function build_manifest( array $exclusions = [] ): array { // Resolve symlinks so the str_starts_with() checks against getRealPath() hold when the base sits behind one. $base = rtrim( realpath( $this->uploads_base ) ?: $this->uploads_base, DIRECTORY_SEPARATOR ); if ( ! is_dir( $base ) ) { return []; } $iterator = new \RecursiveIteratorIterator( new \RecursiveDirectoryIterator( $base, \FilesystemIterator::SKIP_DOTS ), \RecursiveIteratorIterator::LEAVES_ONLY ); $entries = []; /** @var \SplFileInfo $file_info */ foreach ( $iterator as $file_info ) { if ( ! $file_info->isFile() ) { continue; } $abs_path = $file_info->getRealPath(); if ( false === $abs_path ) { continue; } // Symlink guard: the resolved path must stay inside the base. if ( ! str_starts_with( $abs_path, $base . DIRECTORY_SEPARATOR ) ) { continue; } if ( $this->is_protected( $abs_path ) ) { continue; } // Manifest paths cross machines, so they are POSIX and relative regardless of this host's separator. $relative = VirtualPath::relative( substr( $abs_path, strlen( $base ) ) ); // A half-finished write FileImporter keeps beside its target, not site content. if ( TransferTempFile::is_temp( $relative ) ) { continue; } // Shared with the reconciler, so a pattern that skips a file here also protects it from deletion on the destination. if ( $this->exclusions->any_match( $exclusions, $relative ) ) { continue; } $size = $file_info->getSize(); $mtime = $file_info->getMTime(); // A content digest, since mtime|size can't tell a changed same-size file from an intact one. $hash = hash_file( 'sha256', $abs_path ); // An unreadable file would fail verification on the destination and take the whole stage down. if ( false === $size || ! is_string( $hash ) || '' === $hash ) { ServerLog::write( "[Shiftora] manifest: skipping unreadable file {$relative}" ); continue; } $entries[] = [ 'path' => $relative, 'size' => (int) $size, 'mtime' => $mtime, 'hash' => $hash, ]; } // Deterministic order so both sides iterate alike. usort( $entries, fn( $a, $b ) => strcmp( $a['path'], $b['path'] ) ); return $entries; } /** * Size in bytes of a file under the base, without reading it. * * Lets the batch exporter project a file's encoded cost, so one that can't fit the response budget is never read. * * @throws RuntimeException If the path is unsafe or the file cannot be stat'ed. */ public function file_size( string $relative_path ): int { $abs_path = $this->safe_absolute_path( $relative_path ); if ( ! is_file( $abs_path ) ) { throw new RuntimeException( "File not found: {$relative_path}" ); } $file_size = filesize( $abs_path ); if ( false === $file_size ) { throw new RuntimeException( "Cannot stat file: {$relative_path}" ); } return (int) $file_size; } /** * Read a single chunk and encode it whole. * * Reference for the wire format. describe_chunk() with encoded_payload() produces * the same fields without holding the chunk, its gzip and its base64 at once. * * @return array{data: string, bytes_read: int, has_more: bool, chunk_hash: string, compressed: bool} `chunk_hash` is over the raw bytes, before gzip and base64. * * @throws RuntimeException If the path is unsafe, the file is not found, or the read fails. */ public function stream_chunk( string $relative_path, int $offset, int $chunk_size = self::DEFAULT_CHUNK_SIZE ): array { $described = $this->describe_chunk( $relative_path, $offset, $chunk_size ); return [ 'data' => $this->encoded_payload( $relative_path, $offset, $described['bytes_read'], $described['compressed'] ), 'bytes_read' => $described['bytes_read'], 'has_more' => $described['has_more'], 'chunk_hash' => $described['chunk_hash'], 'compressed' => $described['compressed'], ]; } /** * Everything about a chunk except its bytes. * * The receiver verifies a chunk before writing it, so the hash is needed first. * Reading in bounded pieces here lets the payload be streamed afterwards. The * cost is a second read of a range likely in the page cache; holding the chunk, * gzip, base64 and JSON at once can be fatal on a modest host. * * @return array{bytes_read: int, has_more: bool, chunk_hash: string, compressed: bool} * * @throws RuntimeException If the path is unsafe, missing, or unreadable. */ public function describe_chunk( string $relative_path, int $offset, int $chunk_size ): array { if ( $offset < 0 || $chunk_size < 1 ) { throw new RuntimeException( 'Invalid file chunk bounds.' ); } $file_size = $this->file_size( $relative_path ); // Past the end is empty, not an error, so a client re-requesting a completed offset gets an answer. if ( $offset >= $file_size ) { return [ 'bytes_read' => 0, 'has_more' => false, 'chunk_hash' => hash( 'sha256', '' ), 'compressed' => $this->is_compressible( $relative_path ), ]; } // Clamp at EOF so the next file's bytes are never read. $bytes_read = min( $chunk_size, $file_size - $offset ); return [ 'bytes_read' => $bytes_read, 'has_more' => ( $offset + $bytes_read ) < $file_size, 'chunk_hash' => $this->chunk_hash( $relative_path, $offset, $bytes_read ), 'compressed' => $this->is_compressible( $relative_path ), ]; } /** * The base64 payload of one chunk, as one string. * * The batch route needs a value because the puller reads it from inside a JSON * body. It is still built from bounded reads, so the raw chunk and its gzip are never resident beside the base64. * * @throws RuntimeException If the path is unsafe or the read fails. */ public function encoded_payload( string $relative_path, int $offset, int $bytes_read, bool $compressed ): string { // An empty compressed chunk must still be a real gzip stream, since the receiver inflates anything marked compressed. if ( $bytes_read < 1 && ! $compressed ) { return ''; } $payload = ''; foreach ( $this->encoded_payload_fragments( $relative_path, $offset, $bytes_read, $compressed ) as $fragment ) { $payload .= $fragment; } return $payload; } /** * SHA-256 of one chunk's raw bytes, read in bounded pieces. * * @return string Hex digest. * * @throws RuntimeException If the path is unsafe or the read fails. */ public function chunk_hash( string $relative_path, int $offset, int $chunk_size ): string { $handle = $this->open_at( $relative_path, $offset ); try { $ctx = hash_init( 'sha256' ); $left = $chunk_size; while ( $left > 0 && ! feof( $handle ) ) { $piece = fread( $handle, min( self::READ_PIECE_BYTES, $left ) ); if ( false === $piece ) { throw new RuntimeException( "Read error at offset {$offset} in: {$relative_path}" ); } if ( '' === $piece ) { break; } hash_update( $ctx, $piece ); $left -= strlen( $piece ); } } finally { fclose( $handle ); } return hash_final( $ctx ); } /** * The JSON body of one chunk as one string. * * For callers that need the finished body first, such as one that signs its hash. * Built by appending fragments so the peak is the body plus the pieces. * * @return string JSON with data, bytes_read, has_more, chunk_hash, compressed. * * @throws RuntimeException If the path is unsafe or the read fails. */ public function chunk_body( string $relative_path, int $offset, int $chunk_size ): string { $body = ''; foreach ( $this->chunk_body_fragments( $relative_path, $offset, $chunk_size, $this->describe_chunk( $relative_path, $offset, $chunk_size ) ) as $fragment ) { $body .= $fragment; } return $body; } /** * The JSON body of one chunk, yielded in pieces so the caller writes it out instead of holding it. * * The envelope comes first and the payload last, so everything needed to size the * transfer is known before the first payload byte. Compression is incremental * (deflate_add()). Base64 is emitted in 3-byte groups, with a partial group carried into the next piece. * * @param array{bytes_read:int, has_more:bool, chunk_hash:string, compressed:bool} $described From describe_chunk(). * * @return \Generator JSON body fragments, in order. * * @throws RuntimeException If the path is unsafe or the read fails. */ public function chunk_body_fragments( string $relative_path, int $offset, int $chunk_size, array $described ): \Generator { $bytes_read = (int) ( $described['bytes_read'] ?? 0 ); yield '{"chunk_hash":' . wp_json_encode( (string) ( $described['chunk_hash'] ?? '' ) ) . ',"compressed":' . ( empty( $described['compressed'] ) ? 'false' : 'true' ) . ',"bytes_read":' . $bytes_read . ',"has_more":' . ( empty( $described['has_more'] ) ? 'false' : 'true' ) . ',"data":"'; // Read the described range, not the requested size, so a short final chunk doesn't pull in later bytes. if ( $bytes_read > 0 || ! empty( $described['compressed'] ) ) { yield from $this->encoded_payload_fragments( $relative_path, $offset, $bytes_read, ! empty( $described['compressed'] ) ); } yield '"}'; } /** * The base64 payload of one chunk, compressed and encoded in bounded pieces. * * @return \Generator base64 fragments, in order. * * @throws RuntimeException If the path is unsafe or the read fails. */ private function encoded_payload_fragments( string $relative_path, int $offset, int $chunk_size, bool $compressed ): \Generator { $handle = $this->open_at( $relative_path, $offset ); // Base64 works on 3-byte groups; the remainder (at most two bytes) is carried to the next piece, since encoding it alone would corrupt the stream. $carry = ''; $deflate = $compressed ? deflate_init( ZLIB_ENCODING_GZIP, [ 'level' => 6 ] ) : null; if ( $compressed && false === $deflate ) { fclose( $handle ); throw new RuntimeException( "Cannot start compression for: {$relative_path}" ); } try { $left = $chunk_size; while ( $left > 0 && ! feof( $handle ) ) { $piece = fread( $handle, min( self::READ_PIECE_BYTES, $left ) ); if ( false === $piece ) { throw new RuntimeException( "Read error at offset {$offset} in: {$relative_path}" ); } if ( '' === $piece ) { break; } $left -= strlen( $piece ); if ( null !== $deflate ) { $out = deflate_add( $deflate, $piece, ZLIB_NO_FLUSH ); if ( false === $out ) { throw new RuntimeException( "Compression failed for: {$relative_path}" ); } $piece = $out; } $grouped = $carry . $piece; $whole = intdiv( strlen( $grouped ), 3 ) * 3; if ( $whole > 0 ) { yield base64_encode( substr( $grouped, 0, $whole ) ); } $carry = substr( $grouped, $whole ); } if ( null !== $deflate ) { // Finishing the stream flushes whatever the window held. $out = deflate_add( $deflate, '', ZLIB_FINISH ); if ( false === $out ) { throw new RuntimeException( "Compression failed for: {$relative_path}" ); } $carry .= $out; } if ( '' !== $carry ) { yield base64_encode( $carry ); } } finally { fclose( $handle ); } } /** * Open a file positioned at a byte offset. * * @throws RuntimeException If the path is unsafe or the seek fails. */ private function open_at( string $relative_path, int $offset ) { if ( $offset < 0 ) { throw new RuntimeException( 'Invalid file chunk bounds.' ); } $handle = fopen( $this->safe_absolute_path( $relative_path ), 'rb' ); if ( false === $handle ) { throw new RuntimeException( "Cannot open file for reading: {$relative_path}" ); } if ( $offset > 0 && fseek( $handle, $offset ) !== 0 ) { fclose( $handle ); throw new RuntimeException( "Cannot seek to offset {$offset} in: {$relative_path}" ); } return $handle; } /** * Bytes read from disk at a time while encoding, which is the peak the encoder * holds. Small enough for any shared host, large enough that the loop isn't the bottleneck. */ private const READ_PIECE_BYTES = 131_072; /** * Whether the file type benefits from gzip. Images, audio, video and archives are * skipped because they are already compressed. */ public function is_compressible( string $relative_path ): bool { static $compressible = [ 'php', 'js', 'css', 'html', 'htm', 'xml', 'json', 'txt', 'csv', 'svg', 'po', 'pot', 'map', 'md', 'log', 'ini', 'conf', 'htaccess', 'sql', ]; $ext = strtolower( pathinfo( $relative_path, PATHINFO_EXTENSION ) ); return in_array( $ext, $compressible, true ); } /** * The absolute path for $relative_path, asserted to be inside the base. * * Protected files are refused here instead of in each reader, so the batch * route (sizes before reading) and the chunk route (reads) share one check. * * @throws RuntimeException If the path would escape the base, or is a protected file. */ private function safe_absolute_path( string $relative_path ): string { // Reject empty or dangerous input early. if ( '' === $relative_path || str_contains( $relative_path, "\0" ) ) { throw new RuntimeException( 'Invalid file path.' ); } // Resolve the base through realpath so comparisons hold when it is reached via a symlink. $base = rtrim( realpath( $this->uploads_base ) ?: $this->uploads_base, DIRECTORY_SEPARATOR ); $joined = $base . DIRECTORY_SEPARATOR . VirtualPath::to_local( $relative_path ); // realpath only works for existing files; otherwise normalise by hand. $real = realpath( $joined ); $normalised = false !== $real ? $real : $this->normalise_path( $joined ); if ( ! str_starts_with( $normalised, $base . DIRECTORY_SEPARATOR ) ) { throw new RuntimeException( "Path traversal attempt blocked: {$relative_path}" ); } if ( $this->is_protected( $normalised ) ) { throw new RuntimeException( "Refused: `{$relative_path}` is the migration engine's own code and is never served." ); } return $normalised; } private function is_protected( string $absolute_path ): bool { return null !== $this->protected && (bool) ( $this->protected )( $absolute_path ); } /** * Resolve `.` and `..` segments without requiring the target to exist. */ private function normalise_path( string $path ): string { $parts = explode( DIRECTORY_SEPARATOR, $path ); $result = []; foreach ( $parts as $part ) { if ( '' === $part || '.' === $part ) { continue; } if ( '..' === $part ) { array_pop( $result ); } else { $result[] = $part; } } return DIRECTORY_SEPARATOR . implode( DIRECTORY_SEPARATOR, $result ); } // phpcs:enable WordPress.Security.EscapeOutput.ExceptionNotEscaped, WordPress.WP.AlternativeFunctions.file_system_operations_fclose, WordPress.WP.AlternativeFunctions.file_system_operations_fopen, WordPress.WP.AlternativeFunctions.file_system_operations_fread }