job = $job; } /** * Close file handles and undo per-dump session settings (end of tick). */ public function close() { if ( $this->writer ) { $this->writer->close(); $this->writer = null; } Adapter_Factory::end_sessions(); } /** * @return string Path of part N. */ private function part_path( $n ) { return $this->job->work_dir() . '/' . sprintf( 'part-%03d.sql', $n ); } /** * Open the current part, truncating to the committed size, writing the header on a fresh part. */ private function writer() { $s = &$this->job->s; if ( $this->writer ) { return $this->writer; } $path = $this->part_path( $s['part_index'] ); if ( 0 === (int) $s['part_bytes'] ) { $remaining = max( MB_IN_BYTES, (int) $s['bytes_est_total'] - (int) $s['sql_bytes_closed'] ); Diskspace::guard( $this->job->work_dir(), min( (int) $s['part_size'], $remaining ) ); } $this->writer = new Part_Writer( $path, (int) $s['part_bytes'] ); if ( 0 === (int) $s['part_bytes'] ) { $this->writer->write( $this->header_block() ); if ( ! empty( $s['relock'] ) && $this->use_lock_tables() ) { $t = $this->job->table( $s['table_index'] ); if ( $t ) { $this->writer->write( 'LOCK TABLES ' . Adapter_Mysql::quote_name( $this->out_name( $t ) ) . " WRITE;\n" ); } } $s['relock'] = false; } return $this->writer; } /** * @return bool LOCK TABLES around inserts (filter dbmg_use_lock_tables). */ private function use_lock_tables() { return (bool) apply_filters( 'dbmg_use_lock_tables', true ); } /** * Name written into the SQL (prefix conversion may rename). * * @param array $t Work entry. * @return string */ private function out_name( array $t ) { return ! empty( $t['rename_to'] ) ? $t['rename_to'] : $t['name']; } /** * @param array $t Work entry. * @return Adapter */ private function adapter( array $t ) { return Adapter_Factory::get( $t['dataset'] ); } /** * Make text safe inside a `-- ` comment line. A table name may legally contain a newline; written raw it * would end the comment and the rest would run as SQL on import (mysqldump's "Bad Dump", CVE-2016-5483). * * @param string $text Text. * @return string */ public static function comment( $text ) { return str_replace( array( "\r\n", "\r", "\n", "\0" ), array( ' ', ' ', ' ', '' ), (string) $text ); } /** * Per-part header (§9.2). Every part imports on its own. * * @return string */ private function header_block() { global $wpdb; $s = $this->job->s; $env = $s['env']; $charset = ( 'sqlite' === $s['backend']['adapter'] || empty( $wpdb->charset ) ) ? 'utf8mb4' : $wpdb->charset; $lines = array( '-- Database Export Genie ' . DBMG_VERSION . ' SQL dump', '-- Site: ' . self::comment( $env['site_url'] . ' | WordPress ' . $env['wp'] . ' | Table prefix: ' . $env['table_prefix'] ), '-- Server: ' . self::comment( $s['backend']['server'] . ' | PHP ' . $env['php'] ) . ' | Generated (UTC): ' . gmdate( 'Y-m-d H:i:s', (int) $s['id'] ), '-- Part ' . sprintf( '%03d', $s['part_index'] ) . ' (import parts in numeric order, see IMPORT-README.txt)', '', '/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;', '/*!40101 SET NAMES ' . preg_replace( '/[^a-z0-9_]/i', '', $charset ) . ' */;', '/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;', "/*!40103 SET TIME_ZONE='+00:00' */;", '/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;', '/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;', "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;", '', ); return implode( "\n", $lines ) . "\n"; } /** * Restore block at the end of every part. * * @return string */ private function restore_block() { return "\n/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;\n" . "/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;\n" . "/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;\n" . "/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;\n" . "/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;\n"; } /** * One unit of work. * * @return string continue | part_closed | done */ public function step() { $s = &$this->job->s; if ( ! empty( $s['dump_done'] ) ) { return 'done'; } $t = $this->job->table( $s['table_index'] ); if ( null === $t ) { $this->close_part( true ); return 'part_closed'; } $writer = $this->writer(); $before = $writer->size(); switch ( $s['table_phase'] ) { case 'rows': $this->rows( $t ); break; case 'footer': $this->footer( $t ); break; default: $this->header( $t ); } $writer->flush(); $s['part_bytes'] = $writer->size(); $s['table_sql_bytes'] += $writer->size() - $before; $s['sql_bytes'] = (int) $s['sql_bytes_closed'] + (int) $s['part_bytes']; $more = null !== $this->job->table( $s['table_index'] ); if ( $more && (int) $s['part_bytes'] >= (int) $s['part_size'] ) { $this->close_part( false ); return 'part_closed'; } return 'continue'; } /** * DROP + CREATE, and LOCK TABLES for base tables. * * @param array $t Work entry. */ private function header( array $t ) { $s = &$this->job->s; $adapter = $this->adapter( $t ); $name = $this->out_name( $t ); $q = Adapter_Mysql::quote_name( $name ); $is_view = 'VIEW' === $t['type']; $create = $adapter->create_statement( $t['name'], $is_view ); if ( $name !== $t['name'] ) { $create = self::rename_in_create( $create, $t['name'], $name ); } $s['table_sql_bytes'] = 0; $s['rows_done'] = 0; $s['rows_written'] = 0; $s['cursor'] = array(); $s['table_huge'] = false; $s['table_started'] = time(); if ( $is_view ) { $this->writer->write( "\n--\n-- View structure for view " . self::comment( $q ) . "\n--\n\nDROP VIEW IF EXISTS " . $q . ";\n" . rtrim( $create, "; \n" ) . ";\n" ); $this->job->update_table( $s['table_index'], array( 'state' => 'done', 'rows_done' => 0, ) ); $this->advance(); return; } $key = $adapter->key_columns( $t['name'] ); $cols = array_values( array_filter( $adapter->columns( $t['name'] ), static function ( $c ) { return empty( $c['generated'] ); } ) ); if ( 'none' === $key['kind'] ) { $this->job->warn( 'no_key', $t['name'] ); } $this->job->update_table( $s['table_index'], array( 'state' => 'exporting', 'key' => $key, ) ); $this->writer->write( "\n--\n-- Table structure for table " . self::comment( $q ) . "\n--\n\nDROP TABLE IF EXISTS " . $q . ";\n" . rtrim( $create, "; \n" ) . ";\n" ); $this->writer->write( "\n--\n-- Dumping data for table " . self::comment( $q ) . "\n--\n\n" ); if ( $this->use_lock_tables() ) { $this->writer->write( 'LOCK TABLES ' . $q . " WRITE;\n" ); } $s['table_phase'] = $cols ? 'rows' : 'footer'; } /** * Rename the table in its CREATE statement (prefix conversion, §9B.3). * * @param string $create CREATE statement. * @param string $from Old name. * @param string $to New name. * @return string */ public static function rename_in_create( $create, $from, $to ) { $pattern = '/^(CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?)`' . preg_quote( str_replace( '`', '``', $from ), '/' ) . '`/i'; return preg_replace( $pattern, '${1}' . Adapter_Mysql::quote_name( $to ), $create, 1 ); } /** * One batch of rows → multi-row INSERTs capped at insert_cap (§9.3). * * @param array $t Work entry. */ private function rows( array $t ) { $s = &$this->job->s; $adapter = $this->adapter( $t ); $cols = array_values( array_filter( $adapter->columns( $t['name'] ), static function ( $c ) { return empty( $c['generated'] ); } ) ); $key = ! empty( $t['key'] ) ? $t['key'] : $adapter->key_columns( $t['name'] ); $limit = ! empty( $s['table_huge'] ) ? 1 : max( 1, (int) $s['batch_rows'] ); $limit = (int) apply_filters( 'dbmg_batch_rows', $limit, $t['name'] ); $cap = $this->insert_cap( $adapter ); $name = Adapter_Mysql::quote_name( $this->out_name( $t ) ); $head = 'INSERT INTO ' . $name . ' (' . implode( ',', array_map( array( Adapter_Mysql::class, 'quote_name' ), wp_list_pluck( $cols, 'name' ) ) ) . ") VALUES\n"; $kinds = wp_list_pluck( $cols, 'kind', 'name' ); $writer = $this->writer; $buf = ''; $written = 0; $biggest = 0; $bytes = 0; $fail_at = (int) apply_filters( 'dbmg_test_fail_after_rows', 0 ); $done = (int) $s['rows_done']; $on_row = function ( array $row ) use ( &$buf, &$written, &$biggest, &$bytes, &$done, $head, $cap, $kinds, $adapter, $writer, $t, $fail_at ) { ++$done; if ( $fail_at > 0 && $done >= $fail_at ) { // Test hook (§19.2 #2): the process dies mid-batch after uncommitted bytes hit the disk. if ( '' !== $buf ) { $writer->write( $buf ); $writer->flush(); } exit( 255 ); // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped } $row = Multisite_Scope::transform_row( $row, $t['transform'] ); if ( null === $row ) { return; } $values = array(); foreach ( $row as $col => $value ) { $values[] = Dumper::format_value( $value, isset( $kinds[ $col ] ) ? $kinds[ $col ] : 'string', $adapter ); } $tuple = '(' . implode( ',', $values ) . ')'; $len = strlen( $tuple ); $biggest = max( $biggest, $len ); if ( '' === $buf ) { $buf = $head . $tuple; } elseif ( strlen( $buf ) + 2 + $len > $cap ) { $writer->write( $buf . ";\n" ); $bytes += strlen( $buf ) + 2; $buf = $head . $tuple; } else { $buf .= ",\n" . $tuple; } ++$written; }; $started = microtime( true ); $result = $adapter->select_batch( $t['name'], $cols, $key, (array) $s['cursor'], $limit, (string) $t['row_filter'], $on_row ); $elapsed = microtime( true ) - $started; if ( '' !== $buf ) { $writer->write( $buf . ";\n" ); $bytes += strlen( $buf ) + 2; } $s['cursor'] = $result['cursor']; $s['rows_done'] += $result['count']; $s['rows_written'] += $written; // Adaptive batch size: aim for ~2 MB and ≤ 2 s per batch (§9.3). if ( $biggest > self::HUGE_ROW ) { $s['table_huge'] = true; } elseif ( $result['count'] > 0 && empty( $s['table_huge'] ) ) { $factor = min( self::TARGET_BYTES / max( 1, $bytes ), self::TARGET_SECS / max( 0.001, $elapsed ), 4.0 ); $s['batch_rows'] = (int) max( 50, min( 10000, round( $s['batch_rows'] * $factor ) ) ); if ( ! empty( $s['batch_cap'] ) ) { $s['batch_rows'] = min( $s['batch_rows'], (int) $s['batch_cap'] ); } } if ( $result['count'] < $limit ) { $s['table_phase'] = 'footer'; } } /** * Largest INSERT statement: min(1 MB, max_allowed_packet / 2) (§9.3). * * @param Adapter $adapter Adapter. * @return int */ private function insert_cap( Adapter $adapter ) { static $caps = array(); $id = $adapter->dataset(); if ( ! isset( $caps[ $id ] ) ) { $caps[ $id ] = (int) max( 16384, min( MB_IN_BYTES, floor( $adapter->max_allowed_packet() / 2 ) ) ); } return $caps[ $id ]; } /** * SQL literal for one value (§9.3 "Values"). * * @param mixed $value Raw value. * @param string $kind numeric|binary|bit|string. * @param Adapter $adapter Adapter (escaping). * @return string */ public static function format_value( $value, $kind, Adapter $adapter ) { if ( null === $value ) { return 'NULL'; } $value = (string) $value; if ( 'numeric' === $kind && is_numeric( $value ) && '' === trim( $value, '0123456789.-+eE' ) ) { return $value; } if ( 'binary' === $kind ) { return '' === $value ? "''" : '0x' . $value; } if ( 'bit' === $kind ) { return "b'" . preg_replace( '/[^01]/', '', $value ) . "'"; } return "'" . $adapter->escape_string( $value ) . "'"; } /** * UNLOCK, optional triggers, sanity check, next table. * * @param array $t Work entry. */ private function footer( array $t ) { $s = &$this->job->s; $adapter = $this->adapter( $t ); if ( $this->use_lock_tables() ) { $this->writer->write( "UNLOCK TABLES;\n" ); } if ( apply_filters( 'dbmg_include_triggers', false ) ) { foreach ( $adapter->triggers( $t['name'] ) as $trigger ) { $this->writer->write( "\nDELIMITER ;;\n" . rtrim( $trigger, "; \n" ) . ";;\nDELIMITER ;\n" ); } } // §10.3: an estimate > 0 with nothing exported must be double-checked. if ( (int) $t['rows_est'] > 0 && 0 === (int) $s['rows_done'] ) { $real = $adapter->count_rows( $t['name'], (string) $t['row_filter'] ); if ( $real > 0 ) { throw new Backup_Exception( 'table_empty_mismatch', sprintf( '%s: exported 0 of %d rows', $t['name'], $real ), array( 'table' => $t['name'] ) ); } } $this->job->update_table( $s['table_index'], array( 'state' => 'done', 'rows_done' => (int) $s['rows_written'], 'rows_read' => (int) $s['rows_done'], 'sql_bytes' => (int) $s['table_sql_bytes'], ) ); $this->advance(); } /** * Move to the next table. */ private function advance() { $s = &$this->job->s; ++$s['table_index']; $s['table_phase'] = 'header'; $s['cursor'] = array(); $s['rows_done'] = 0; $s['rows_written'] = 0; $s['table_huge'] = false; } /** * Close the current part at a statement boundary and queue it for zipping. * * @param bool $final Last part of the dump. */ private function close_part( $final ) { $s = &$this->job->s; $writer = $this->writer(); if ( ! $final && in_array( $s['table_phase'], array( 'rows', 'footer' ), true ) && $this->use_lock_tables() ) { $writer->write( "UNLOCK TABLES;\n" ); $s['relock'] = true; } $writer->write( $this->restore_block() ); if ( $final ) { $writer->write( self::MARKER . ' ' . gmdate( 'Y-m-d\TH:i:s\Z' ) . "\n" ); } $writer->flush(); $bytes = $writer->size(); $path = $writer->path(); $writer->close(); $this->writer = null; if ( ! $final ) { $s['multi'] = true; } $n = (int) $s['part_index']; $entry = ( $final && 1 === $n && empty( $s['multi'] ) ) ? 'database.sql' : sprintf( 'database.part-%03d.sql', $n ); if ( $final ) { $s['footer_found'] = self::has_marker( $path ); $s['dump_done'] = true; $this->job->end_snapshot(); } $s['pending_parts'][] = array( 'index' => $n, 'file' => basename( $path ), 'entry' => $entry, 'bytes' => $bytes, 'crc' => hash_file( 'crc32b', $path ), ); $s['sql_bytes_closed'] = (int) $s['sql_bytes_closed'] + $bytes; $s['sql_bytes'] = (int) $s['sql_bytes_closed']; $s['part_index'] = $n + 1; $s['part_bytes'] = 0; } /** * Does the file end with the completion marker? (§10.3, read before the part is deleted.) * * @param string $path Part file. * @return bool */ public static function has_marker( $path ) { // DECISION: PHP < 8.4 keeps the size Part_Writer's constructor stat'ed; without this a part opened and // closed in one tick was read at its old length and the marker looked missing (D32). clearstatcache( true, $path ); $size = (int) filesize( $path ); $fh = fopen( $path, 'rb' ); // phpcs:ignore WordPress.WP.AlternativeFunctions.file_system_operations_fopen if ( ! $fh ) { return false; } fseek( $fh, max( 0, $size - 256 ) ); $tail = (string) fread( $fh, 256 ); // phpcs:ignore WordPress.WP.AlternativeFunctions.file_system_operations_fread fclose( $fh ); // phpcs:ignore WordPress.WP.AlternativeFunctions.file_system_operations_fclose return false !== strpos( $tail, self::MARKER ); } }