cursor ) { self::remove_db_dir( $job ); // The database step's last work files; its result is saved by now. } $c = $job->cursor + [ 'pos' => 0, 'rel' => '', 'add' => [], 'start' => 0, 'files' => 0, 'bytes' => 0 ]; $list = self::list_path( $job ); $fh = is_file( $list ) ? fopen( $list, 'rb' ) : false; if ( false === $fh ) { throw new Mench_Migrate_Storage_Exception( 'Cannot read the file list.', 'write-files' ); } $total = max( 1, (int) fstat( $fh )['size'] ); $base = self::content_dir(); $writer = self::writer( $job ); $beat = microtime( true ); $done = false; $state = null; // Set when a partial entry is rolled back; replaces the writer's own state. $files = (int) ( $job->data['scan']['files'] ?? 0 ); $bytes = (int) ( $job->data['scan']['bytes'] ?? 0 ); $report = static function ( int $partial ) use ( $job, &$c, $files, $bytes, $total ): void { $done_bytes = (int) $c['bytes'] + $partial; Mench_Migrate_Progress::report( $job, [ 'counts' => [ 'files_done' => (int) $c['files'], 'files_total' => max( $files, (int) $c['files'] ), 'bytes_done' => $done_bytes, 'bytes_total' => max( $bytes, $done_bytes ) ], 'fraction' => $bytes > 0 ? $done_bytes / $bytes : (int) $c['pos'] / $total, ] ); }; $report( (int) ( $c['add']['offset'] ?? 0 ) ); try { fseek( $fh, (int) $c['pos'] ); while ( true ) { if ( [] === $c['add'] ) { $line = fgets( $fh ); if ( false === $line ) { $done = true; break; } $c['pos'] = (int) ftell( $fh ); $parts = explode( "\t", rtrim( $line, "\n" ) ); $c['rel'] = $parts[0]; $c['start'] = $writer->length(); if ( 3 !== count( $parts ) || 1 !== preg_match( '//u', $c['rel'] ) ) { $this->skip( $job, $c, 'invalid_name' ); if ( microtime( true ) >= $deadline ) { break; } continue; } } try { $res = $writer->add_file( $base . '/' . $c['rel'], $c['rel'], $c['add'], Mench_Migrate_Progress::slice( $deadline ) ); } catch ( Mench_Migrate_Archive_Exception $e ) { $code = $e->getMessage(); if ( in_array( $code, self::BAD_SOURCE, true ) ) { $state = $writer->state(); $state['length'] = (int) $c['start']; $state['current'] = null; $this->skip( $job, $c, $code ); break; // End the slice so the rolled-back state is saved before anything else is written. } if ( ! in_array( $code, self::BAD_NAME, true ) ) { throw $e; } $this->skip( $job, $c, $code ); if ( microtime( true ) >= $deadline ) { break; } continue; } if ( ! $res['done'] ) { $c['add'] = $res['cursor']; $report( (int) $c['add']['offset'] ); Mench_Migrate_Job_Runner::heartbeat( $job ); if ( microtime( true ) >= $deadline ) { break; } continue; // The same large file, in another short sub-slice. } $c['files']++; $c['bytes'] += (int) $res['cursor']['size']; $c['add'] = []; $c['rel'] = ''; $report( 0 ); if ( microtime( true ) - $beat > 5 ) { Mench_Migrate_Job_Runner::heartbeat( $job ); $beat = microtime( true ); } if ( microtime( true ) >= $deadline ) { break; } } } finally { fclose( $fh ); } $job->data['writer'] = $state ?? $writer->state(); unset( $writer ); if ( ! $done ) { return Mench_Migrate_Step_Result::more( $c, $c['pos'] / $total ); } $job->log( 'info', 'Added ' . $c['files'] . ' files (' . size_format( $c['bytes'] ) . ').' ); return Mench_Migrate_Step_Result::done( [ 'files' => [ 'count' => $c['files'], 'bytes' => $c['bytes'] ] ] ); } /** Counts and logs a skipped file, then clears the in-progress entry from the cursor. */ private function skip( Mench_Migrate_Job $job, array &$c, string $why ): void { $rel = (string) $c['rel']; $shown = function_exists( 'mb_scrub' ) ? (string) preg_replace( '/[\x00-\x1f\x7f]/', '?', mb_scrub( $rel, 'UTF-8' ) ) : (string) preg_replace( '/[^\x20-\x7e]/', '?', $rel ); $job->log( 'warning', 'Skipped ' . $shown . ' (' . $why . ').' ); $job->data['skipped_files'] = (int) ( $job->data['skipped_files'] ?? 0 ) + 1; $c['add'] = []; $c['rel'] = ''; } }